@fangorn-network/westmarch 0.0.1-dev → 0.0.2-dev

package/consume/view-tools.js

@@ -0,0 +1,202 @@
+// An app's views, served as tools, with no browser.
+//
+// A Fangorn app's card names its views: static, content-addressed shards any
+// HTTP client can read. The page's WebMCP tools are one way in, but they need a
+// browser, and a browser costs seconds and gigabytes before the first answer.
+// These tools read the views directly with westmarch's own search, browse,
+// count, get and similar, so the only code that runs is ours. The app's data is
+// verified by its manifest's sha256, not trusted.
+//
+// Time to first answer is the point, so:
+//   - a search goes to the 3 views whose coverage sketch best matches the query,
+//     not to all of them (Kingsfoil: ~3 of 23 views, ~92% of the true top 10 with
+//     a 32-vector sketch). The result names what was searched and what was not.
+//   - `warm()` starts the embedder the moment an app is opened, and a search
+//     never waits on a first-time model download.
+//
+// `get`/`similar`/`count` answer over what is loaded, and say so.
+
+import { configure, loadShard, trimView } from "./shard.js";
+import { rolesFrom, textOf } from "./roles.js";
+import { browse, facet, getRow, neighbors, search } from "./tools.js";
+import { lexScore, rankDomains } from "./rank.js";
+import { existsSync } from "node:fs";
+import { EMBED_MODEL, embedQueryDirect, warmDirect } from "./embed.js";
+
+const ROUTE_TOP = 3;
+
+// One roles table per view, set from its manifests before its first row parses.
+const rolesByView = new Map();
+const DEFAULT_ROLES = rolesFrom([]);
+const rolesOf = (view) => rolesByView.get(view) ?? DEFAULT_ROLES;
+configure({
+    onManifests: (manifests, view) => rolesByView.set(view, rolesFrom(manifests)),
+    rowText: (f, view) => textOf(f, rolesOf(view)),
+});
+
+// ponytail: an embedder that fails to load is remembered as failed, and search
+// ranks by words for the rest of the process. Retry when that matters.
+let embedder = null, ready = false, downloading = false;
+/** `cacheDir`: where the model is kept. Outside node_modules, or every new
+ *  install of this package downloads the same 131MB again. */
+export const warm = ({ cacheDir } = {}) => (embedder ??= (async () => {
+    if (cacheDir) {
+        downloading = !existsSync(`${cacheDir}/${EMBED_MODEL}/onnx/model_quantized.onnx`);   // before any await
+        (await import("@huggingface/transformers")).env.cacheDir = cacheDir;
+    }
+    await warmDirect();
+})().then(() => (ready = true), (e) => { warmError = e?.message ?? String(e); return false; })
+    .finally(() => { downloading = false; }));
+/** Why the embedder did not load, if it did not. */
+export let warmError = null;
+// A model on disk loads in ~0.5s and is worth the wait. A download is 131MB, so
+// until it lands, search ranks by words and says so, rather than stall ~10s.
+const queryVector = async (q) => {
+    if (!ready && downloading) return null;
+    return (await warm()) ? embedQueryDirect(q).catch(() => null) : null;
+};
+const rankedBy = (qv) => qv ? "meaning"
+    : downloading ? "words (the embedding model is downloading; meaning once it lands)"
+    : "words (embedder unavailable)";
+
+const text = (o) => ({ content: [{ type: "text", text: JSON.stringify(o, null, 1) }] });
+const str = (description) => ({ type: "string", description });
+const num = (description) => ({ type: "number", description });
+const VIEWS = { type: "array", items: { type: "string" }, description: 'View names from describe, or ["all"]. Omit to let the query choose.' };
+const WHERE = { type: "object", description: 'Equality filters, e.g. {"phase":"PHASE3"}' };
+
+/**
+ * The tools for one app, or null when its card names no views.
+ * `app` is `{ name, desc, views }` from a verified card. `fetchCatalog(url)` may
+ * answer from a stale copy: the survey only describes and routes, and loading a
+ * view re-reads its catalog through shard.js either way.
+ */
+export function viewTools(app, { fetchCatalog = fetch } = {}) {
+    if (!app.views?.length) return null;
+    const names = new Map();   // view url → its name in every result
+    const label = (url) => names.get(url) ?? url.split("/").pop();
+    let surveyed;
+    // The catalogs: what each view holds and its coverage sketch. Small, and no rows.
+    const survey = () => (surveyed ??= Promise.all(app.views.map(async (url) => {
+        const view = trimView(url);
+        const { domains = [] } = await (await fetchCatalog(`${view}/cdn/catalog`)).json();
+        names.set(view, domains.length === 1 ? domains[0].name : label(view));
+        return {
+            view, name: label(view),
+            description: domains.map((d) => d.description).filter(Boolean).join(" "),
+            count: domains.reduce((s, d) => s + (d.count ?? 0), 0),
+            coverage: { vectors: domains.flatMap((d) => d.coverage?.vectors ?? []) },
+        };
+    })).catch((e) => { surveyed = null; throw e; }));
+
+    survey().catch(() => {});   // started now, so describe and the first search do not wait on it
+    const loaded = new Map();   // view → rows
+    const load = async (v) => { if (!loaded.has(v.view)) loaded.set(v.view, await loadShard(v.view)); return loaded.get(v.view); };
+    const pick = async (names) => {
+        const all = await survey();
+        if (names?.includes("all")) return all;
+        const bad = names.filter((n) => !all.some((v) => v.name === n || label(v.view) === n));
+        if (bad.length) throw new Error(`unknown view(s) ${bad.join(", ")}; views: ${all.map((v) => v.name).join(", ")}`);
+        return all.filter((v) => names.includes(v.name) || names.includes(label(v.view)));
+    };
+    // The query decides: coverage sketch by meaning, else the descriptions by words.
+    const route = async (query, qv) => {
+        const all = await survey();
+        return qv ? rankDomains(all, qv)
+            : all.map((v) => ({ ...v, affinity: lexScore({ text: `${v.name} ${v.description}` }, query) }))
+                 .sort((a, b) => b.affinity - a.affinity);
+    };
+    const loadedRows = () => [...loaded.entries()].flatMap(([view, rows]) => rows.map((r) => ({ r, view })));
+
+    const run = {
+        describe: async () => ({
+            app: app.name, description: app.desc,
+            views: (await survey()).map((v) => ({ view: v.name, rows: v.count, description: v.description, loaded: loaded.has(v.view) })),
+        }),
+        search: async ({ query, limit = 10, views, where: w }) => {
+            if (!query) throw new Error("query is required");
+            const qv = await queryVector(query);
+            const ranked = views?.length ? await pick(views) : (await route(query, qv)).slice(0, ROUTE_TOP);
+            const others = views?.length ? [] : (await route(query, qv)).slice(ROUTE_TOP, ROUTE_TOP + 5).map((v) => v.name);
+            const per = await Promise.all(ranked.map(async (v) =>
+                search(await load(v), query, rolesOf(v.view), { qv, limit, where: w }).map((h) => ({ ...h, view: v.name }))));
+            const hits = per.flat().sort((a, b) => (b.score ?? 0) - (a.score ?? 0)).slice(0, limit);
+            return { query, ranked_by: rankedBy(qv), searched: ranked.map((v) => v.name),
+                     next_views: others, hits };
+        },
+        get: async ({ id, view }) => {
+            if (view) for (const v of await pick([view])) await load(v);
+            for (const [v, rows] of loaded) { const r = getRow(rows, id, rolesOf(v)); if (r) return r; }
+            return { error: `${id} is not in a loaded view. Search for it, or pass its view.`, loaded: [...loaded.keys()].map(label) };
+        },
+        similar: async ({ id, limit = 10, views }) => {
+            const hit = loadedRows().find(({ r }) => r.id === id);
+            if (!hit) return { error: `${id} is not loaded. Search for it first.` };
+            // Neighbours by design live anywhere, so route on the row's own vector.
+            const targets = views?.length ? await pick(views)
+                : hit.r.vector ? (await route("", hit.r.vector)).slice(0, ROUTE_TOP) : [];
+            await Promise.all(targets.map(load));
+            const rows = [...loaded.values()].flat();
+            const { seed, near } = neighbors(rows, id, rolesOf(hit.view), { limit });
+            return { id, seed, searched: [...loaded.keys()].map(label), hits: near };
+        },
+        count: async ({ field, views, limit = 20, where: w }) => {
+            if (!field) throw new Error("field is required");
+            if (views?.length) await Promise.all((await pick(views)).map(load));
+            if (!loaded.size) return { error: 'nothing loaded yet. Pass views (or ["all"]) to count over.' };
+            return { field, over: [...loaded.keys()].map(label), ...facet([...loaded.values()].flat(), field, { limit, where: w }) };
+        },
+        browse: async ({ view, limit = 20, offset = 0, sort, where: w }) => {
+            const [v] = await pick([view]);
+            return { view: v.name, ...browse(await load(v), rolesOf(v.view), { limit, offset, sort, where: w }) };
+        },
+    };
+
+    const n = app.name;
+    const tools = [
+        { name: "describe", description: `What ${n} holds: its views, each with a description and row count. Downloads no rows; call it first.`,
+          inputSchema: { type: "object", properties: {} } },
+        { name: "search", description: `Search ${n} by meaning. Without views, searches the ${ROUTE_TOP} views whose coverage best matches the query and names the next candidates; pass views to widen.`,
+          inputSchema: { type: "object", properties: { query: str("A question in natural language"), limit: num("Max hits (default 10)"), views: VIEWS, where: WHERE }, required: ["query"] } },
+        { name: "get", description: `One record of ${n} in full, by id.`,
+          inputSchema: { type: "object", properties: { id: str("The record id, as search returns it"), view: str("Its view, if it has not been loaded") }, required: ["id"] } },
+        { name: "similar", description: `Records of ${n} nearest to one record, across the views its vector points to.`,
+          inputSchema: { type: "object", properties: { id: str("A record id from search"), limit: num("Max results (default 10)"), views: VIEWS }, required: ["id"] } },
+        { name: "count", description: `Count the distinct values of a field in ${n}: the shape of a result set before reading it.`,
+          inputSchema: { type: "object", properties: { field: str("Field name, e.g. from get"), views: VIEWS, limit: num("Max values (default 20)"), where: WHERE }, required: ["field"] } },
+        { name: "browse", description: `List one view of ${n} without a query, with filters, sorting and paging.`,
+          inputSchema: { type: "object", properties: { view: str("View name from describe"), limit: num("default 20"), offset: num("default 0"), sort: str("Field to sort by"), where: WHERE }, required: ["view"] } },
+    ];
+    // Records the app sells over x402 (its card's `paid`). Paid from this server's wallet,
+    // never above FANGORN_MCP_MAX_PRICE, and checked against the sha256 the app published
+    // for that record, so what was paid for is what was committed.
+    if (app.paid) {
+        const p = app.paid, dec = p.decimals ?? 6;
+        const price = `${(Number(p.price) / 10 ** dec).toFixed(dec).replace(/0+$/, "").replace(/\.$/, "")} ${p.symbol ?? "USDC"}`;
+        const cap = BigInt(Math.round(Number(process.env.FANGORN_MCP_MAX_PRICE ?? "0.10") * 10 ** dec));
+        run.buy = async ({ id }) => {
+            if (!id) throw new Error("id is required");
+            const hit = loadedRows().find(({ r, view }) => r.id === id || r[rolesOf(view).identity] === id);
+            const key = hit ? hit.r[rolesOf(hit.view).identity] ?? id : id;
+            const { payAndFetch } = await import("./x402.js");
+            const r = await payAndFetch(p.url.replace("{id}", encodeURIComponent(key)),
+                { privateKey: process.env.FANGORN_MCP_WALLET_KEY, maxPrice: cap });
+            if (r.status !== 200) return { id: key, error: `HTTP ${r.status}: ${r.body.slice(0, 200)}` };
+            const digest = [...new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(r.body)))]
+                .map((b) => b.toString(16).padStart(2, "0")).join("");
+            const want = hit?.r.paid_sha256;
+            return { id: key, paid: r.price ? price : "free", transaction: r.receipt?.transaction ?? null, network: r.receipt?.network ?? p.network,
+                     verified: want ? (want === digest ? "matches the published sha256" : `MISMATCH: published ${want}, got ${digest}`) : "not checked (search for the record first)",
+                     record: JSON.parse(r.body) };
+        };
+        tools.push({ name: "buy", description: `Buy one ${n} record's paid detail for ${price} (x402 on ${p.network}): ${p.description || "the structured record"}. ` +
+                "Pass an id from search. Paid from this server's wallet (FANGORN_MCP_WALLET_KEY), never above FANGORN_MCP_MAX_PRICE.",
+            inputSchema: { type: "object", properties: { id: str("A record id from search or get") }, required: ["id"] } });
+    }
+
+    return { tools, call: async (name, args = {}) => {
+        if (!run[name]) throw new Error(`no tool ${name}`);
+        return text(await run[name](args));
+    } };
+}
+

package/consume/x402.js

@@ -0,0 +1,94 @@
+// Paying for a resource over x402 (v1, "exact" scheme): the buyer's half.
+//
+//   const { body, receipt, price } = await payAndFetch(url, { privateKey, maxPrice: 50000n });
+//
+// 1. GET url → 402 with `accepts`: what it costs, in which token, on which chain, to whom.
+// 2. Sign an EIP-3009 transferWithAuthorization for exactly that (EIP-712, the token's own
+//    domain). Nothing is sent on chain by the buyer: the seller's facilitator submits it.
+// 3. GET url again with X-PAYMENT: the seller verifies, settles, and returns the resource,
+//    with the settlement (the tx hash) in X-PAYMENT-RESPONSE.
+//
+// `maxPrice` (token base units) is checked BEFORE signing: a seller cannot ask for more
+// than the caller agreed to, whatever its 402 says.
+
+const CHAIN_IDS = { "arbitrum-sepolia": 421614, arbitrum: 42161, "base-sepolia": 84532, base: 8453 };
+const b64 = (s) => (typeof btoa === "function" ? btoa(s) : Buffer.from(s, "utf8").toString("base64"));
+const unb64 = (s) => (typeof atob === "function" ? atob(s) : Buffer.from(s, "base64").toString("utf8"));
+
+/** The one requirement we can pay: exact scheme, a chain we know, within `maxPrice`. */
+export function choose(accepts = [], { maxPrice, networks = Object.keys(CHAIN_IDS) } = {}) {
+    const ok = accepts.filter((a) => a.scheme === "exact" && networks.includes(a.network));
+    if (!ok.length) throw new Error(`no payable option: ${JSON.stringify(accepts.map((a) => `${a.scheme}/${a.network}`))}`);
+    const cheapest = ok.sort((a, b) => (BigInt(a.maxAmountRequired) < BigInt(b.maxAmountRequired) ? -1 : 1))[0];
+    if (maxPrice != null && BigInt(cheapest.maxAmountRequired) > BigInt(maxPrice))
+        throw new Error(`price ${cheapest.maxAmountRequired} exceeds the cap ${maxPrice} (token base units)`);
+    return cheapest;
+}
+
+/** The X-PAYMENT header value for requirement `req`, signed by `account` (a viem account). */
+export async function paymentHeader(req, account, { now = Math.floor(Date.now() / 1000) } = {}) {
+    const chainId = CHAIN_IDS[req.network];
+    if (!chainId) throw new Error(`unknown network ${req.network}`);
+    const nonce = `0x${[...crypto.getRandomValues(new Uint8Array(32))].map((b) => b.toString(16).padStart(2, "0")).join("")}`;
+    const authorization = {
+        from: account.address, to: req.payTo, value: String(req.maxAmountRequired),
+        validAfter: String(now - 600), validBefore: String(now + (req.maxTimeoutSeconds ?? 120)), nonce,
+    };
+    const signature = await account.signTypedData({
+        domain: { name: req.extra?.name ?? "USD Coin", version: req.extra?.version ?? "2", chainId, verifyingContract: req.asset },
+        types: { TransferWithAuthorization: [
+            { name: "from", type: "address" }, { name: "to", type: "address" }, { name: "value", type: "uint256" },
+            { name: "validAfter", type: "uint256" }, { name: "validBefore", type: "uint256" }, { name: "nonce", type: "bytes32" }] },
+        primaryType: "TransferWithAuthorization",
+        message: { ...authorization, value: BigInt(authorization.value), validAfter: BigInt(authorization.validAfter), validBefore: BigInt(authorization.validBefore) },
+    });
+    return b64(JSON.stringify({ x402Version: 1, scheme: "exact", network: req.network, payload: { signature, authorization } }));
+}
+
+/** GET `url`, paying once if it answers 402. Returns { status, body, price, receipt }. */
+export async function payAndFetch(url, { privateKey, account, maxPrice, fetchImpl = fetch } = {}) {
+    const first = await fetchImpl(url);
+    if (first.status !== 402) return { status: first.status, body: await first.text(), price: null, receipt: null };
+    const offer = await first.json();
+    const req = choose(offer.accepts, { maxPrice });
+    if (!account) {
+        if (!privateKey) throw new Error(`${url} costs ${req.maxAmountRequired} (base units of ${req.asset} on ${req.network}); no wallet configured to pay`);
+        account = (await import("viem/accounts")).privateKeyToAccount(privateKey);
+    }
+    const paid = await fetchImpl(url, { headers: { "X-PAYMENT": await paymentHeader(req, account) } });
+    const receiptHeader = paid.headers.get("X-PAYMENT-RESPONSE");
+    const body = await paid.text();
+    if (paid.status === 402) throw new Error(`payment refused: ${JSON.parse(body).error ?? body}`);
+    return { status: paid.status, body, price: req.maxAmountRequired, receipt: receiptHeader ? JSON.parse(unb64(receiptHeader)) : null };
+}
+
+// ── self-check: `node consume/x402.js` — a stand-in seller that checks the signature ──
+if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/x402.js")) {
+    const { privateKeyToAccount, generatePrivateKey } = await import("viem/accounts");
+    const { verifyTypedData } = await import("viem");
+    const buyer = privateKeyToAccount(generatePrivateKey());
+    const USDC = "0x75faf114eafb1BDbe2F0316DF893fd58CE46AA4d", SELLER = `0x${"5e".repeat(20)}`;
+    const req = { scheme: "exact", network: "arbitrum-sepolia", maxAmountRequired: "10000", resource: "https://x/paid/1",
+                  payTo: SELLER, asset: USDC, maxTimeoutSeconds: 120, extra: { name: "USD Coin", version: "2" } };
+    let settled = null;
+    const seller = async (url, init = {}) => {
+        const h = init.headers?.["X-PAYMENT"];
+        if (!h) return new Response(JSON.stringify({ x402Version: 1, accepts: [req] }), { status: 402 });
+        const p = JSON.parse(unb64(h)).payload, a = p.authorization;
+        const ok = await verifyTypedData({ address: a.from, signature: p.signature,
+            domain: { name: "USD Coin", version: "2", chainId: 421614, verifyingContract: USDC },
+            types: { TransferWithAuthorization: [{ name: "from", type: "address" }, { name: "to", type: "address" }, { name: "value", type: "uint256" },
+                { name: "validAfter", type: "uint256" }, { name: "validBefore", type: "uint256" }, { name: "nonce", type: "bytes32" }] },
+            primaryType: "TransferWithAuthorization",
+            message: { ...a, value: BigInt(a.value), validAfter: BigInt(a.validAfter), validBefore: BigInt(a.validBefore) } });
+        if (!ok || a.to !== SELLER || a.value !== "10000") return new Response(JSON.stringify({ error: "bad payment" }), { status: 402 });
+        settled = a;
+        return new Response('{"secret":42}', { status: 200, headers: { "X-PAYMENT-RESPONSE": b64(JSON.stringify({ success: true, transaction: "0xabc" })) } });
+    };
+    const r = await payAndFetch("https://x/paid/1", { account: buyer, maxPrice: 10000n, fetchImpl: seller });
+    if (r.body !== '{"secret":42}' || r.receipt.transaction !== "0xabc" || settled.from !== buyer.address) throw new Error(`paid fetch: ${JSON.stringify(r)}`);
+    let refused = false;
+    try { await payAndFetch("https://x/paid/1", { account: buyer, maxPrice: 9999n, fetchImpl: seller }); } catch (e) { refused = /exceeds the cap/.test(e.message); }
+    if (!refused) throw new Error("a price over the cap must be refused before signing");
+    console.log("x402.js self-check ok — a signed EIP-3009 authorization the seller can verify, exactly the asked amount to the asked payee, the receipt read back, and a price over the cap refused before signing");
+}