@fangorn-network/westmarch 0.0.10-dev → 0.0.12-dev
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -23
- package/dist/ts/eval.js +1 -1
- package/docs/app-to-agent.md +5 -5
- package/package.json +39 -36
- package/plugin/scripts/fx.mjs +7 -7
- package/site/agent.js +2 -2
- package/site/dist/assets/embed.worker-CZXpLhAu.js +1 -0
- package/site/dist/assets/index-oyUqokZh.js +8 -0
- package/site/dist/assets/{ort-wasm-simd-threaded.asyncify-DMmc6YqF.wasm → ort-wasm-simd-threaded.asyncify-CxOG5pUO.wasm} +0 -0
- package/site/dist/assets/preload-helper-BaNbYf_w.js +1 -0
- package/site/dist/assets/transformers.web-CM54uIeJ.js +40 -0
- package/site/dist/assets/transformers.web-D8W1u_I2.js +49 -0
- package/site/dist/index.html +2 -2
- package/site/feed.js +2 -2
- package/site/store.js +3 -3
- package/{publish → src/agent}/agent-card.js +3 -3
- package/{consume → src/agent}/mcp-bin.js +1 -1
- package/{consume → src/agent}/mcp.js +3 -3
- package/{consume → src/agent}/tools.js +3 -3
- package/{consume → src/agent}/ui.js +1 -1
- package/{consume → src/agent}/view-tools.js +4 -4
- package/{consume → src/agent}/x402.js +1 -1
- package/{consume → src/core}/embed.js +33 -8
- package/{consume → src/core}/ort-preload.js +1 -1
- package/{consume → src/core}/rank.js +1 -1
- package/{consume → src/core}/roles.js +1 -1
- package/{consume → src/core}/shard.js +1 -1
- package/{consume → src/discover}/apps.js +2 -2
- package/{consume → src/discover}/corpora.js +2 -2
- package/{consume → src/discover}/directory.js +3 -3
- package/{consume → src/discover}/explore.js +56 -28
- package/{publish → src/market}/cohort.js +1 -1
- package/{publish → src/market}/demand.js +2 -2
- package/{publish → src/market}/reactions.js +4 -4
- package/{publish → src/market}/settle.js +2 -2
- package/{publish → src/market}/terms.js +1 -1
- package/{publish → src/publish}/cli.js +2 -2
- package/{publish → src/publish}/envelope.js +1 -1
- package/{publish → src/publish}/eval.ts +1 -1
- package/{consume → src/publish}/lint.js +5 -4
- package/{publish → src/publish}/manifest.js +1 -1
- package/{publish → src/publish}/ship.js +8 -8
- package/{publish → src/publish}/view.js +27 -14
- package/{consume → src/taste}/steam.js +1 -1
- package/{consume → src/taste}/taste-doc.js +2 -2
- package/{consume → src/taste}/taste.js +35 -56
- package/{publish → src/taste}/vault.js +1 -1
- package/site/dist/assets/embed.worker-Dn3fKctb.js +0 -1
- package/site/dist/assets/index-w2wfUwhm.js +0 -8
- package/site/dist/assets/preload-helper-uBIymjUX.js +0 -1
- package/site/dist/assets/transformers.web-BgUm7F_-.js +0 -46
- package/site/dist/assets/transformers.web-u9uSoh5E.js +0 -37
- /package/{consume → src/core}/embed.worker.js +0 -0
- /package/{publish → src/publish}/enrich.js +0 -0
- /package/{publish → src/publish}/graph.js +0 -0
- /package/{publish → src/publish}/view-cli.js +0 -0
- /package/{publish → src/publish}/westmarch.d.ts +0 -0
package/site/dist/index.html
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
<!doctype html>
|
|
2
2
|
<html lang="en">
|
|
3
|
-
<script type="module" crossorigin src="./assets/index-
|
|
3
|
+
<script type="module" crossorigin src="./assets/index-oyUqokZh.js"></script>
|
|
4
4
|
<link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-Dd_uD5pT.js">
|
|
5
|
-
<link rel="modulepreload" crossorigin href="./assets/preload-helper-
|
|
5
|
+
<link rel="modulepreload" crossorigin href="./assets/preload-helper-BaNbYf_w.js">
|
|
6
6
|
<link rel="modulepreload" crossorigin href="./assets/roles-C9V_uDr_.js">
|
|
7
7
|
|
|
8
8
|
<meta charset="utf-8">
|
package/site/feed.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// occasion (a meeting, an issue, an episode), from the roles the app declared. Nothing here
|
|
3
3
|
// knows what a row is: an occasion is the rows sharing a subtitle and a facet value, and
|
|
4
4
|
// its date is their temporal field.
|
|
5
|
-
import { pick, subtitleOf, titleOf, values } from "../
|
|
5
|
+
import { pick, subtitleOf, titleOf, values } from "../src/core/roles.js";
|
|
6
6
|
|
|
7
7
|
/** The field a reader narrows by first: the app's declared facet, else the first tag with
|
|
8
8
|
* a handful of values (a town, not every body in every town). */
|
|
@@ -45,7 +45,7 @@ export function detail(r, roles) {
|
|
|
45
45
|
|
|
46
46
|
// ── self-check: `node site/feed.js` ──
|
|
47
47
|
if (typeof process !== "undefined" && process.argv[1]?.endsWith("/feed.js")) {
|
|
48
|
-
const { rolesFrom } = await import("../
|
|
48
|
+
const { rolesFrom } = await import("../src/core/roles.js");
|
|
49
49
|
const roles = rolesFrom([{ role_map: { title: "heading", subtitle: "meeting", temporal: "date", tags: ["city", "body"], text: ["text"] },
|
|
50
50
|
presentation: { facets: ["city"] } }]);
|
|
51
51
|
const row = (city, date, heading, text = heading) => ({ city, date, heading, meeting: `Board · ${date}`, body: "Board", text });
|
package/site/store.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
// What a person does on the page, kept on their own device: what they saved, what they
|
|
2
2
|
// passed on, and what they searched this session. Nothing leaves the browser unless they
|
|
3
|
-
// export it. The saved items are also the likes a taste is built from (
|
|
3
|
+
// export it. The saved items are also the likes a taste is built from (src/taste/taste.js),
|
|
4
4
|
// so "For you" is theirs, and so is the file they can hand an agent.
|
|
5
5
|
//
|
|
6
6
|
// Saved items are snapshots keyed by the publisher's own id (the identity role), not the
|
|
7
7
|
// row's CID: a record re-published with a new CID must still be the item they saved.
|
|
8
|
-
import { exportTaste, taste } from "../
|
|
9
|
-
import { packVec, unpackVec } from "../
|
|
8
|
+
import { exportTaste, taste } from "../src/taste/taste.js";
|
|
9
|
+
import { packVec, unpackVec } from "../src/core/embed.js";
|
|
10
10
|
|
|
11
11
|
const read = (s, k, dflt) => { try { return JSON.parse(s?.getItem(k) ?? "null") ?? dflt; } catch { return dflt; } };
|
|
12
12
|
const write = (s, k, v) => { try { s?.setItem(k, JSON.stringify(v)); } catch { /* private mode or full: the page still works */ } };
|
|
@@ -106,13 +106,13 @@ export function fangornExtension({ app, fromBlock, namespaces = [], views = [],
|
|
|
106
106
|
};
|
|
107
107
|
}
|
|
108
108
|
|
|
109
|
-
// ── self-check: `node
|
|
109
|
+
// ── self-check: `node src/agent/agent-card.js` ────────────────────────────────
|
|
110
110
|
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/agent-card.js")) {
|
|
111
111
|
const eq = (a, b, m) => { if (JSON.stringify(a) !== JSON.stringify(b)) throw new Error(`${m}: ${JSON.stringify(a)} != ${JSON.stringify(b)}`); };
|
|
112
112
|
const throws = (f, re, m) => { try { f(); } catch (e) { if (re.test(e.message)) return; throw e; } throw new Error(`must throw: ${m}`); };
|
|
113
113
|
|
|
114
|
-
const { APP_EXTENSION } = await import("../
|
|
115
|
-
eq(APP_EXTENSION, FANGORN_APP_EXTENSION, "
|
|
114
|
+
const { APP_EXTENSION } = await import("../discover/apps.js");
|
|
115
|
+
eq(APP_EXTENSION, FANGORN_APP_EXTENSION, "src/discover/apps.js's copy of the extension uri");
|
|
116
116
|
|
|
117
117
|
let fired = 0;
|
|
118
118
|
const tools = await captureTools(() => {
|
|
@@ -2,5 +2,5 @@
|
|
|
2
2
|
// Static imports run in order, so the library is loaded before the server
|
|
3
3
|
// starts. Not `await import("./mcp.js")`: bun 1.1 then wraps each module in a
|
|
4
4
|
// function, and their top-level awaits stop parsing.
|
|
5
|
-
import "
|
|
5
|
+
import "../core/ort-preload.js";
|
|
6
6
|
import "./mcp.js";
|
|
@@ -29,7 +29,7 @@ import { createHash } from "node:crypto";
|
|
|
29
29
|
import net from "node:net";
|
|
30
30
|
import { homedir } from "node:os";
|
|
31
31
|
import { createInterface } from "node:readline";
|
|
32
|
-
import { APP_EXTENSION, listApps, toApp } from "
|
|
32
|
+
import { APP_EXTENSION, listApps, toApp } from "../discover/apps.js";
|
|
33
33
|
|
|
34
34
|
// The current AppRegistry's first event on Arbitrum Sepolia. A reader's
|
|
35
35
|
// default, not deployment config: pass --from-block for another deployment.
|
|
@@ -522,12 +522,12 @@ const deps = {
|
|
|
522
522
|
// ponytail: cached for a minute, like findApps. Two getLogs, a card and catalog per app.
|
|
523
523
|
exploreApps: async (query) => {
|
|
524
524
|
if (!explored || Date.now() - exploredAt > 60_000) {
|
|
525
|
-
const [ex, { createPublicClient, http }, { arbitrumSepolia }] = await Promise.all([import("
|
|
525
|
+
const [ex, { createPublicClient, http }, { arbitrumSepolia }] = await Promise.all([import("../discover/explore.js"), import("viem"), import("viem/chains")]);
|
|
526
526
|
explored = { ex, dir: await ex.readDirectory(createPublicClient({ chain: arbitrumSepolia, transport: http() })) };
|
|
527
527
|
exploredAt = Date.now();
|
|
528
528
|
}
|
|
529
529
|
const { ex, dir } = explored;
|
|
530
|
-
const { EMBED_MODEL } = await import("
|
|
530
|
+
const { EMBED_MODEL } = await import("../core/embed.js");
|
|
531
531
|
const qv = query?.trim() ? await vt.queryVector(query) : null;
|
|
532
532
|
const apps = ex.rankApps(dir.apps, query, { qv, model: EMBED_MODEL }).map(ex.brief);
|
|
533
533
|
return { apps, ranked_by: apps[0]?.ranked_by ?? null };
|
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
//
|
|
12
12
|
// Pure functions, so `node tools.js` checks them with no browser and no network.
|
|
13
13
|
|
|
14
|
-
import { bestPassage, cosine, lexScore, norm, zFloor } from "
|
|
15
|
-
import { briefOf, collections, linkOf, subtitleOf, titleOf, typeOf, values } from "
|
|
14
|
+
import { bestPassage, cosine, lexScore, norm, zFloor } from "../core/rank.js";
|
|
15
|
+
import { briefOf, collections, linkOf, subtitleOf, titleOf, typeOf, values } from "../core/roles.js";
|
|
16
16
|
|
|
17
17
|
/** Fields that are plumbing rather than content — never worth faceting or
|
|
18
18
|
* showing as "what this corpus holds". */
|
|
@@ -232,7 +232,7 @@ export function getRow(rows, id, roles) {
|
|
|
232
232
|
|
|
233
233
|
// ── self-check: `node tools.js` — no browser, no network ────────────────────
|
|
234
234
|
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/tools.js")) {
|
|
235
|
-
const { rolesFrom } = await import("
|
|
235
|
+
const { rolesFrom } = await import("../core/roles.js");
|
|
236
236
|
const row = (id, fields, vector) => ({
|
|
237
237
|
id, owner: "0xa", ...fields,
|
|
238
238
|
text: [fields.name, fields.desc].filter(Boolean).join(" "),
|
|
@@ -504,7 +504,7 @@ export function stage({ picks = [], heading = "", note = "" } = {}) {
|
|
|
504
504
|
<script>window.__ITEMS__ = ${json};${STAGE_JS}<\/script>`;
|
|
505
505
|
}
|
|
506
506
|
|
|
507
|
-
// ── self-check: `node
|
|
507
|
+
// ── self-check: `node src/agent/ui.js` ────────────────────────────────────────
|
|
508
508
|
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/ui.js")) {
|
|
509
509
|
const roles = { measures: ["rating", "year", "runtime", "votes"] };
|
|
510
510
|
const html = rankedList({
|
|
@@ -16,12 +16,12 @@
|
|
|
16
16
|
//
|
|
17
17
|
// `get`/`similar`/`count` answer over what is loaded, and say so.
|
|
18
18
|
|
|
19
|
-
import { configure, loadShard, trimView } from "
|
|
20
|
-
import { rolesFrom, textOf } from "
|
|
19
|
+
import { configure, loadShard, trimView } from "../core/shard.js";
|
|
20
|
+
import { rolesFrom, textOf } from "../core/roles.js";
|
|
21
21
|
import { browse, facet, getRow, neighbors, search } from "./tools.js";
|
|
22
|
-
import { lexScore, rankDomains } from "
|
|
22
|
+
import { lexScore, rankDomains } from "../core/rank.js";
|
|
23
23
|
import { existsSync } from "node:fs";
|
|
24
|
-
import { EMBED_MODEL, embedQueryDirect, warmDirect } from "
|
|
24
|
+
import { EMBED_MODEL, embedQueryDirect, warmDirect } from "../core/embed.js";
|
|
25
25
|
|
|
26
26
|
const ROUTE_TOP = 3;
|
|
27
27
|
|
|
@@ -62,7 +62,7 @@ export async function payAndFetch(url, { privateKey, account, maxPrice, fetchImp
|
|
|
62
62
|
return { status: paid.status, body, price: req.maxAmountRequired, receipt: receiptHeader ? JSON.parse(unb64(receiptHeader)) : null };
|
|
63
63
|
}
|
|
64
64
|
|
|
65
|
-
// ── self-check: `node
|
|
65
|
+
// ── self-check: `node src/agent/x402.js` — a stand-in seller that checks the signature ──
|
|
66
66
|
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/x402.js")) {
|
|
67
67
|
const { privateKeyToAccount, generatePrivateKey } = await import("viem/accounts");
|
|
68
68
|
const { verifyTypedData } = await import("viem");
|
|
@@ -13,6 +13,11 @@
|
|
|
13
13
|
|
|
14
14
|
const MODEL = "nomic-ai/nomic-embed-text-v1.5";
|
|
15
15
|
const DIM = 256;
|
|
16
|
+
// The document side's precision. q8 everywhere by default (the browser's queries are q8).
|
|
17
|
+
// WESTMARCH_EMBED_DTYPE=fp16 is a different encoder (~0.96 cosine to q8): a view records it
|
|
18
|
+
// and refuses to mix the two (view.js), so switching an app is one --rebake.
|
|
19
|
+
const DTYPE = globalThis.process?.env?.WESTMARCH_EMBED_DTYPE ?? "q8";
|
|
20
|
+
export const EMBED_DTYPE = DTYPE;
|
|
16
21
|
|
|
17
22
|
// Stamped onto every vector a publisher commits. Vectors from different models
|
|
18
23
|
// (or dims) are not comparable — cosine between them is noise, and silently so.
|
|
@@ -25,10 +30,11 @@ let _extractor = null;
|
|
|
25
30
|
const extractor = () => (_extractor ??= (async () => {
|
|
26
31
|
const { pipeline } = await import("@huggingface/transformers");
|
|
27
32
|
// WESTMARCH_EMBED_DEVICE=cuda puts the publisher's document side on a GPU (Node only; needs
|
|
28
|
-
// onnxruntime-node's CUDA binaries
|
|
29
|
-
//
|
|
33
|
+
// onnxruntime-node's CUDA binaries and the matching CUDA runtime, cuBLAS, cuRAND and cuDNN on
|
|
34
|
+
// LD_LIBRARY_PATH). q8's integer ops do not run on CUDA, so q8 there is slower than on CPU:
|
|
35
|
+
// pair it with WESTMARCH_EMBED_DTYPE=fp16, which is ~4x faster than CPU q8, batched.
|
|
30
36
|
const device = globalThis.process?.env?.WESTMARCH_EMBED_DEVICE;
|
|
31
|
-
return pipeline("feature-extraction", MODEL, { dtype:
|
|
37
|
+
return pipeline("feature-extraction", MODEL, { dtype: DTYPE, ...(device && { device }) });
|
|
32
38
|
})());
|
|
33
39
|
|
|
34
40
|
/** Standardize over the full vector, slice to `dim`, L2-normalize the slice. */
|
|
@@ -59,11 +65,30 @@ export async function embedQueryDirect(text) {
|
|
|
59
65
|
return matryoshka(output.data);
|
|
60
66
|
}
|
|
61
67
|
|
|
62
|
-
/** One document, in-process.
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
68
|
+
/** One document, in-process. Calls made together are embedded as one batch (a GPU's
|
|
69
|
+
* speed); q8 stays at batch 1, since batching moves its activation quantization. */
|
|
70
|
+
export function embedDocumentDirect(text) {
|
|
71
|
+
return new Promise((resolve, reject) => {
|
|
72
|
+
pending.push({ text, resolve, reject });
|
|
73
|
+
if (pending.length === 1) setTimeout(flushDocuments, 0);
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
const BATCH = DTYPE === "q8" ? 1 : 32;
|
|
77
|
+
let pending = [];
|
|
78
|
+
async function flushDocuments() {
|
|
79
|
+
const all = pending;
|
|
80
|
+
pending = [];
|
|
81
|
+
try {
|
|
82
|
+
const ex = await extractor();
|
|
83
|
+
for (let i = 0; i < all.length; i += BATCH) {
|
|
84
|
+
const part = all.slice(i, i + BATCH);
|
|
85
|
+
const output = await ex(part.map((p) => `search_document: ${p.text}`), { pooling: "mean", normalize: false });
|
|
86
|
+
const d = output.dims.at(-1);
|
|
87
|
+
part.forEach((p, j) => p.resolve(matryoshka(output.data.slice(j * d, (j + 1) * d))));
|
|
88
|
+
}
|
|
89
|
+
} catch (e) {
|
|
90
|
+
for (const p of all) p.reject(e);
|
|
91
|
+
}
|
|
67
92
|
}
|
|
68
93
|
|
|
69
94
|
/** Load the model in-process, and settle when it is actually ready. */
|
|
@@ -9,7 +9,7 @@ import { dlopen } from "bun:ffi";
|
|
|
9
9
|
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
10
10
|
import { homedir } from "node:os";
|
|
11
11
|
// build-bin.js puts the target's library here before it compiles.
|
|
12
|
-
import lib from "
|
|
12
|
+
import lib from "../../dist/ort/libonnxruntime.so.1" with { type: "file" };
|
|
13
13
|
|
|
14
14
|
try {
|
|
15
15
|
const dir = `${homedir()}/.cache/fangorn-mcp/lib/${lib.split("/").pop()}`; // bun names it by content hash
|
|
@@ -158,7 +158,7 @@ export function rankDomains(domains = [], q) {
|
|
|
158
158
|
.sort((a, b) => (b.affinity ?? -Infinity) - (a.affinity ?? -Infinity));
|
|
159
159
|
}
|
|
160
160
|
|
|
161
|
-
// ── self-check: `node
|
|
161
|
+
// ── self-check: `node src/core/rank.js` ──────────────────────────────────────
|
|
162
162
|
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/rank.js")) {
|
|
163
163
|
// Word boundaries: the bug this replaced ranked "intoxicated" for "cat".
|
|
164
164
|
if (lexScore({ text: "intoxicated education" }, "cat") !== 0) throw new Error("substring matched — word boundaries lost");
|
|
@@ -366,7 +366,7 @@ export function actionsOf(row, roles) {
|
|
|
366
366
|
}).filter((a) => a.locked.length || !a.missing.length);
|
|
367
367
|
}
|
|
368
368
|
|
|
369
|
-
// ── self-check: `node
|
|
369
|
+
// ── self-check: `node src/core/roles.js` ─────────────────────────────────────
|
|
370
370
|
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/roles.js")) {
|
|
371
371
|
// The real places manifest's shape.
|
|
372
372
|
const places = {
|
|
@@ -437,7 +437,7 @@ export async function suggestDomains(q, { url } = {}) {
|
|
|
437
437
|
return rankDomains(domains, q);
|
|
438
438
|
}
|
|
439
439
|
|
|
440
|
-
// ── self-check: `node
|
|
440
|
+
// ── self-check: `node src/core/shard.js` — no network, no view ────────────────
|
|
441
441
|
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/shard.js")) {
|
|
442
442
|
const { gzipSync } = await import("node:zlib");
|
|
443
443
|
const warn = console.warn, info = console.info;
|
|
@@ -30,7 +30,7 @@ function safeView(url) {
|
|
|
30
30
|
}
|
|
31
31
|
|
|
32
32
|
/** The SDK's FANGORN_APP_EXTENSION. Copied so this file needs no SDK;
|
|
33
|
-
*
|
|
33
|
+
* src/agent/agent-card.js's self-check fails if the two drift. */
|
|
34
34
|
export const APP_EXTENSION = "https://fangorn.network/a2a/app/v1";
|
|
35
35
|
|
|
36
36
|
/** A verified card → an app entry. `views` is westmarch's param; the SDK does not read it. */
|
|
@@ -70,7 +70,7 @@ export async function listApps(fangorn, { fromBlock, toBlock } = {}) {
|
|
|
70
70
|
return { apps, rejected };
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
-
// ── self-check: `node
|
|
73
|
+
// ── self-check: `node src/discover/apps.js` ──────────────────────────────────────
|
|
74
74
|
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/apps.js")) {
|
|
75
75
|
const j = (x) => JSON.stringify(x, (_, v) => (typeof v === "bigint" ? `${v}n` : v));
|
|
76
76
|
const eq = (a, b, m) => { if (j(a) !== j(b)) throw new Error(`${m}: ${j(a)} != ${j(b)}`); };
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
// each and a person opens a handful; when a session holds enough of them to
|
|
16
16
|
// matter, drop the least recently focused — the map is already keyed for it.
|
|
17
17
|
|
|
18
|
-
import { trimView } from "
|
|
18
|
+
import { trimView } from "../core/shard.js";
|
|
19
19
|
|
|
20
20
|
/**
|
|
21
21
|
* A set of open corpora, and the vocabulary for naming one.
|
|
@@ -188,7 +188,7 @@ export function follow(picks = [], corpora = [], { at = null } = {}) {
|
|
|
188
188
|
return out;
|
|
189
189
|
}
|
|
190
190
|
|
|
191
|
-
// ── self-check: `node
|
|
191
|
+
// ── self-check: `node src/discover/corpora.js` ───────────────────────────────────
|
|
192
192
|
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/corpora.js")) {
|
|
193
193
|
const s = session();
|
|
194
194
|
const add = (view, name, n) => {
|
|
@@ -22,8 +22,8 @@
|
|
|
22
22
|
// still well under a second. Past that, the directory needs its own coverage
|
|
23
23
|
// index, which is the same trick one level up.
|
|
24
24
|
|
|
25
|
-
import { trimView } from "
|
|
26
|
-
import { rankDomains, zFloor } from "
|
|
25
|
+
import { trimView } from "../core/shard.js";
|
|
26
|
+
import { rankDomains, zFloor } from "../core/rank.js";
|
|
27
27
|
import { listApps } from "./apps.js";
|
|
28
28
|
|
|
29
29
|
/** Catalog URL for a view base, tolerating either form the registry prints. */
|
|
@@ -167,7 +167,7 @@ export async function sourcesFromChain(fangorn, opts) {
|
|
|
167
167
|
return { sources: apps.flatMap((a) => a.views.map((view) => ({ view, name: a.name, appId: a.appId }))), rejected };
|
|
168
168
|
}
|
|
169
169
|
|
|
170
|
-
// ── self-check: `node
|
|
170
|
+
// ── self-check: `node src/discover/directory.js` ─────────────────────────────────
|
|
171
171
|
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/directory.js")) {
|
|
172
172
|
const warn = console.warn; console.warn = () => {};
|
|
173
173
|
const cov = (...vectors) => ({ dim: 2, sampled: 100, vectors, counts: vectors.map(() => 50) });
|
|
@@ -8,26 +8,30 @@
|
|
|
8
8
|
// ERC-8004 identity `Registered` the agent minted for the app
|
|
9
9
|
// ERC-8004 reputation feedback count and average per agent
|
|
10
10
|
//
|
|
11
|
-
// Both registries
|
|
12
|
-
//
|
|
13
|
-
//
|
|
11
|
+
// Both registries are scanned from their first event in 10M-block windows, in
|
|
12
|
+
// parallel, so the page and the MCP tool read the same chain the same way, and
|
|
13
|
+
// a new app shows up on the next read.
|
|
14
14
|
//
|
|
15
15
|
// `client` is a viem PublicClient (getLogs, readContract): passed in, so this
|
|
16
16
|
// runs in a tab and in node, and the self-check needs no chain.
|
|
17
17
|
//
|
|
18
|
-
// ponytail:
|
|
19
|
-
//
|
|
18
|
+
// ponytail: windows grow with the chain (1 + 8 at block 316M, ~0.2s). Index
|
|
19
|
+
// the logs somewhere if that ever gets slow.
|
|
20
20
|
|
|
21
21
|
import { parseAbi, parseAbiItem } from "viem";
|
|
22
22
|
import { APP_EXTENSION } from "./apps.js";
|
|
23
23
|
import { survey, comparable } from "./directory.js";
|
|
24
|
-
import { rankDomains, zFloor, lexScore } from "
|
|
24
|
+
import { rankDomains, zFloor, lexScore } from "../core/rank.js";
|
|
25
25
|
|
|
26
26
|
export const CHAIN = {
|
|
27
27
|
chainId: 421614,
|
|
28
28
|
appRegistry: "0x11d228c4774af3d9cae3b4b6874a12576a1a83ec",
|
|
29
29
|
identity: "0x8004A818BFB912233c491871b3d84c89A494BD9e",
|
|
30
30
|
reputation: "0x8004B663056A597Dffe9eCcC1965A193B7388713",
|
|
31
|
+
// Each registry's first event (2026-10-05): nothing to read before it.
|
|
32
|
+
// Scanning from 0 instead was 32 windows per registry, ~5s of a 7.7s load.
|
|
33
|
+
appRegistryFrom: 311_639_205n,
|
|
34
|
+
identityFrom: 241_557_787n,
|
|
31
35
|
scan: (agentId) => `https://8004scan.io/agents/arbitrum-sepolia/${agentId}`,
|
|
32
36
|
};
|
|
33
37
|
|
|
@@ -101,54 +105,78 @@ async function reputation(client, agentId, chain) {
|
|
|
101
105
|
}
|
|
102
106
|
}
|
|
103
107
|
|
|
108
|
+
// The public Arbitrum Sepolia RPC refuses an address+event getLogs spanning more
|
|
109
|
+
// than 10M blocks (from 2026-10; one call from 0 worked before).
|
|
110
|
+
const LOG_WINDOW = 10_000_000n;
|
|
111
|
+
|
|
112
|
+
async function getLogsSince(client, start, filter) {
|
|
113
|
+
const head = await client.getBlockNumber();
|
|
114
|
+
const windows = [];
|
|
115
|
+
for (let from = start; from <= head; from += LOG_WINDOW) {
|
|
116
|
+
const to = from + LOG_WINDOW - 1n < head ? from + LOG_WINDOW - 1n : head;
|
|
117
|
+
windows.push(client.getLogs({ ...filter, fromBlock: from, toBlock: to }));
|
|
118
|
+
}
|
|
119
|
+
return (await Promise.all(windows)).flat(); // in window order: oldest first still holds
|
|
120
|
+
}
|
|
121
|
+
|
|
104
122
|
/**
|
|
105
123
|
* Every verified app, with its 8004 agent, reputation, and catalog.
|
|
106
124
|
* Unverified apps are dropped, not listed; `dropped` says why, for debugging.
|
|
125
|
+
*
|
|
126
|
+
* `onUpdate({ apps, pending })`, if given, is called once the bindings are read
|
|
127
|
+
* and again as each app lands or drops, so a page can show apps as they arrive.
|
|
128
|
+
* `pending` counts the bound apps not yet settled.
|
|
107
129
|
*/
|
|
108
|
-
export async function readDirectory(client, { chain = CHAIN } = {}) {
|
|
130
|
+
export async function readDirectory(client, { chain = CHAIN, onUpdate = () => {} } = {}) {
|
|
109
131
|
const [bindLogs, regLogs] = await Promise.all([
|
|
110
|
-
client.
|
|
111
|
-
client.
|
|
132
|
+
getLogsSince(client, chain.appRegistryFrom ?? 0n, { address: chain.appRegistry, event: APP_AGENT_CHANGED }),
|
|
133
|
+
getLogsSince(client, chain.identityFrom ?? 0n, { address: chain.identity, event: REGISTERED }),
|
|
112
134
|
]);
|
|
113
135
|
const registered = regLogs.map((l) => ({ agentId: l.args.agentId, owner: l.args.owner, block: l.blockNumber }));
|
|
114
136
|
const latest = new Map(); // logs come oldest first: the last write wins
|
|
115
137
|
for (const l of bindLogs) latest.set(l.args.app_id, { uri: l.args.agent_uri, block: l.blockNumber });
|
|
138
|
+
const bound = [...latest].filter(([, { uri }]) => uri); // an empty uri is unbound on purpose
|
|
116
139
|
|
|
117
|
-
//
|
|
118
|
-
//
|
|
119
|
-
//
|
|
120
|
-
const dropped = [];
|
|
121
|
-
|
|
122
|
-
|
|
140
|
+
// One pipeline per app, so the quick ones need not wait on Quorum's 1.7 MB
|
|
141
|
+
// catalog. Only a card failure drops an app; an owner or catalog that fails
|
|
142
|
+
// leaves it listed with less to say about it.
|
|
143
|
+
const apps = [], dropped = [];
|
|
144
|
+
let pending = bound.length;
|
|
145
|
+
const settle = () => { pending--; apps.sort((a, b) => a.name.localeCompare(b.name)); onUpdate({ apps: [...apps], pending }); };
|
|
146
|
+
onUpdate({ apps: [], pending });
|
|
147
|
+
await Promise.all(bound.map(async ([appId, { uri, block }]) => {
|
|
148
|
+
let card;
|
|
123
149
|
try {
|
|
124
150
|
if (!/^https:\/\//.test(uri)) throw new Error("card is not https");
|
|
125
|
-
|
|
151
|
+
card = await json(uri);
|
|
126
152
|
const why = verifyCard(card, appId, chain);
|
|
127
153
|
if (why) throw new Error(why);
|
|
128
|
-
|
|
129
|
-
} catch (e) { dropped.push({ card: uri, why: e?.message ?? String(e) }); return null; }
|
|
130
|
-
}))).filter(Boolean);
|
|
154
|
+
} catch (e) { dropped.push({ card: uri, why: e?.message ?? String(e) }); return settle(); }
|
|
131
155
|
|
|
132
|
-
const apps = await Promise.all(verified.map(async ({ appId, uri, block, card }) => {
|
|
133
156
|
const p = card.capabilities.extensions.find((e) => e.uri === APP_EXTENSION).params;
|
|
134
|
-
const owner
|
|
135
|
-
|
|
136
|
-
|
|
157
|
+
const [[owner, agentId, rep], { corpora, unreachable }] = await Promise.all([
|
|
158
|
+
client.readContract({ address: chain.appRegistry, abi: APPS, functionName: "getAppOwner", args: [appId] }).catch(() => null)
|
|
159
|
+
.then(async (owner) => {
|
|
160
|
+
const agentId = owner && pairAgent(registered, owner, block);
|
|
161
|
+
return [owner, agentId, agentId == null ? null : await reputation(client, agentId, chain)];
|
|
162
|
+
}),
|
|
163
|
+
survey((p.views ?? []).filter((v) => /^https:\/\//.test(v))),
|
|
164
|
+
]);
|
|
137
165
|
const name = typeof card.name === "string" && card.name.trim() ? card.name.trim() : `App ${appId.slice(0, 10)}`;
|
|
138
|
-
|
|
166
|
+
apps.push({
|
|
139
167
|
app: slug(name), name, description: typeof card.description === "string" ? card.description : "",
|
|
140
168
|
site: https(card.url), card: uri, appId, owner,
|
|
141
169
|
boundAt: Number(block), fromBlock: Number(p.fromBlock),
|
|
142
170
|
tools: (card.skills ?? []).map((s) => s.id),
|
|
143
171
|
agent: agentId == null ? null : { id: Number(agentId), scan: chain.scan(agentId) },
|
|
144
|
-
reputation:
|
|
172
|
+
reputation: rep,
|
|
145
173
|
// null, not 0, when no catalog answered: "0 records" would claim it was counted.
|
|
146
174
|
rows: corpora.length ? corpora.reduce((n, c) => n + (c.rows || 0), 0) : null,
|
|
147
175
|
unreachable: unreachable.length,
|
|
148
176
|
domains: corpora.map((c) => ({ name: c.domain, description: c.description, rows: c.rows, coverage: c.coverage, model: c.model })),
|
|
149
|
-
};
|
|
177
|
+
});
|
|
178
|
+
settle();
|
|
150
179
|
}));
|
|
151
|
-
apps.sort((a, b) => a.name.localeCompare(b.name));
|
|
152
180
|
return { apps, dropped };
|
|
153
181
|
}
|
|
154
182
|
|
|
@@ -200,7 +228,7 @@ export function rankApps(apps, query, { qv = null, model = null } = {}) {
|
|
|
200
228
|
* domains, so an app reports how many, and which one matched. */
|
|
201
229
|
export const brief = ({ domains, ...a }) => ({ ...a, domains: domains.length });
|
|
202
230
|
|
|
203
|
-
// ── self-check: `node
|
|
231
|
+
// ── self-check: `node src/discover/explore.js` ──────────────────────────────────
|
|
204
232
|
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/explore.js")) {
|
|
205
233
|
const j = (x) => JSON.stringify(x, (_, v) => (typeof v === "bigint" ? `${v}n` : v));
|
|
206
234
|
const eq = (a, b, m) => { if (j(a) !== j(b)) throw new Error(`${m}: ${j(a)} != ${j(b)}`); };
|
|
@@ -246,7 +246,7 @@ export function readout(total, cohort, { minSupport = MIN_SUPPORT } = {}) {
|
|
|
246
246
|
};
|
|
247
247
|
}
|
|
248
248
|
|
|
249
|
-
// ── self-check: `node
|
|
249
|
+
// ── self-check: `node src/market/cohort.js` ──────────────────────────────────
|
|
250
250
|
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/cohort.js")) {
|
|
251
251
|
const vec = (a, b) => { const v = new Float32Array(STAT_DIM); v[0] = a; v[1] = b; return v; };
|
|
252
252
|
const log = (n, seed) => Array.from({ length: n }, (_, i) => ({
|
|
@@ -45,7 +45,7 @@
|
|
|
45
45
|
// says what was measured and what the publisher owns near it. Everything past
|
|
46
46
|
// that is a model of a market that has run five simulated rounds.
|
|
47
47
|
|
|
48
|
-
import { vocabulary, affinities } from "../
|
|
48
|
+
import { vocabulary, affinities } from "../taste/taste-doc.js";
|
|
49
49
|
|
|
50
50
|
/** Matched reactions per reader below which a region is not drawing attention.
|
|
51
51
|
* 0.5 = the median reader reacted to something here at least every other
|
|
@@ -382,7 +382,7 @@ export function brief(rep, { title = "demand" } = {}) {
|
|
|
382
382
|
return L.join("\n");
|
|
383
383
|
}
|
|
384
384
|
|
|
385
|
-
// ── self-check: `node
|
|
385
|
+
// ── self-check: `node src/market/demand.js` ────────────────────────────────────
|
|
386
386
|
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/demand.js")) {
|
|
387
387
|
const { keypair, statistics, contribute, aggregate, readout } = await import("./cohort.js");
|
|
388
388
|
|
|
@@ -42,7 +42,7 @@
|
|
|
42
42
|
// Nothing is lost, because nobody buys row 47 of a stranger's reactions. A taste
|
|
43
43
|
// corpus is bought WHOLE, and the question a buyer actually asks — "does this
|
|
44
44
|
// reader have dense signal where I am training?" — is answered by the centroids
|
|
45
|
-
// alone. That is already how `
|
|
45
|
+
// alone. That is already how `src/discover/directory.js` ranks a corpus nobody has
|
|
46
46
|
// downloaded. Per-row vectors served no buyer and leaked the entire product.
|
|
47
47
|
//
|
|
48
48
|
// The centroids are themselves averages, so `k` is capped to keep at least
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
// IS that reaction's vector under another name, which is the same leak wearing a
|
|
51
51
|
// hat, and a reader with nine reactions is exactly who most needs this to hold.
|
|
52
52
|
|
|
53
|
-
import { packVec } from "../
|
|
53
|
+
import { packVec } from "../core/embed.js";
|
|
54
54
|
import { lineage } from "./terms.js";
|
|
55
55
|
|
|
56
56
|
// 32, not 8: on Kingsfoil the top 3 of 23 views held 92% of the true top 10 with 32
|
|
@@ -168,7 +168,7 @@ export function reactionCorpus(events = [], {
|
|
|
168
168
|
|
|
169
169
|
// The split is applied HERE, to the rows themselves, not merely declared in
|
|
170
170
|
// the manifest. Declaring it and shipping the fields anyway is the exact bug
|
|
171
|
-
// `
|
|
171
|
+
// `src/publish/lint.js` calls "paywall names fields the free shard ships anyway",
|
|
172
172
|
// and this file had it: it returned every field on every row and left the
|
|
173
173
|
// withholding to a bake that does not run on this path.
|
|
174
174
|
const wire = [], locked = [];
|
|
@@ -236,7 +236,7 @@ export function reactionCorpus(events = [], {
|
|
|
236
236
|
return { manifest, rows: wire, locked, lineage: up, byCorpus: Object.fromEntries(byCorpus) };
|
|
237
237
|
}
|
|
238
238
|
|
|
239
|
-
// ── self-check: `node
|
|
239
|
+
// ── self-check: `node src/market/reactions.js` ────────────────────────────────
|
|
240
240
|
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/reactions.js")) {
|
|
241
241
|
const { splitPayment } = await import("./terms.js");
|
|
242
242
|
const A = (n) => `0x${String(n).repeat(40).slice(0, 40)}`;
|
|
@@ -16,7 +16,7 @@ import { bytesToHex } from "viem";
|
|
|
16
16
|
import {
|
|
17
17
|
CHUNK_SIZE, NONCE_LEN, aadFor, chunkKey, deleteResource,
|
|
18
18
|
getWorkerPubkey, pack, putChunk, unpack,
|
|
19
|
-
} from "
|
|
19
|
+
} from "../publish/envelope.js";
|
|
20
20
|
|
|
21
21
|
export { CHUNK_SIZE, aadFor, chunkKey, deleteResource, getWorkerPubkey };
|
|
22
22
|
|
|
@@ -64,7 +64,7 @@ export async function encryptAndUpload({ file, resourceId, workerUrl, uploadToke
|
|
|
64
64
|
}
|
|
65
65
|
}
|
|
66
66
|
|
|
67
|
-
// ── self-check: `node
|
|
67
|
+
// ── self-check: `node src/market/settle.js` — no network, no worker ─────────────
|
|
68
68
|
//
|
|
69
69
|
// It decrypts with bare AES-GCM rather than importing a buyer, because what is
|
|
70
70
|
// under test here is the ENCRYPTOR: chunk addressing, the AAD index binding, the
|
|
@@ -136,7 +136,7 @@ export function splitPayment(amount, { terms, publisher, lineage: up = [] } = {}
|
|
|
136
136
|
* price anyone reads. */
|
|
137
137
|
export const label = (base, currency = "USDC") => `${(Number(base) / 1e6).toFixed(2)} ${currency}`;
|
|
138
138
|
|
|
139
|
-
// ── self-check: `node
|
|
139
|
+
// ── self-check: `node src/market/terms.js` ────────────────────────────────────
|
|
140
140
|
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/terms.js")) {
|
|
141
141
|
const A = "0x" + "a".repeat(40); // app owner
|
|
142
142
|
const P = "0x" + "b".repeat(40); // publisher
|
|
@@ -47,7 +47,7 @@ import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
|
|
|
47
47
|
import { basename, join, relative, resolve, sep } from "node:path";
|
|
48
48
|
import { createPublicClient, createWalletClient, defineChain, http } from "viem";
|
|
49
49
|
import { privateKeyToAccount } from "viem/accounts";
|
|
50
|
-
import { encryptAndUpload } from "
|
|
50
|
+
import { encryptAndUpload } from "../market/settle.js";
|
|
51
51
|
import { resourceIdFor } from "./envelope.js";
|
|
52
52
|
import { newUid, readManifest, writeManifest } from "./manifest.js";
|
|
53
53
|
|
|
@@ -299,7 +299,7 @@ async function main() {
|
|
|
299
299
|
for (const [path, ptr] of Object.entries(sealed.published)) console.log(` ${path} ${ptr.resourceId}`);
|
|
300
300
|
}
|
|
301
301
|
|
|
302
|
-
// ── self-check: `node publish/cli.js --selfcheck` (offline) ──────────────────
|
|
302
|
+
// ── self-check: `node src/publish/cli.js --selfcheck` (offline) ──────────────────
|
|
303
303
|
|
|
304
304
|
if (flags.selfcheck) {
|
|
305
305
|
const assert = (cond, msg) => { if (!cond) throw new Error(msg); };
|
|
@@ -150,7 +150,7 @@ export async function deleteResource({ workerUrl, uploadToken, resourceId, chunk
|
|
|
150
150
|
return Math.max(1, chunks);
|
|
151
151
|
}
|
|
152
152
|
|
|
153
|
-
// ── self-check: `node publish/envelope.js` ───────────────────────────────────
|
|
153
|
+
// ── self-check: `node src/publish/envelope.js` ───────────────────────────────────
|
|
154
154
|
//
|
|
155
155
|
// These vectors are PINNED to the values sond3r's own envelope.js produced before
|
|
156
156
|
// this file was extracted from it, with ns = "sond3r". They are not a style test:
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
// them. So: build a base view (main's recipe) and a candidate (the change) from the
|
|
13
13
|
// same data, on the same machine, and run both through here. The exit code is the gate.
|
|
14
14
|
//
|
|
15
|
-
// Every check runs the reader's own code (
|
|
15
|
+
// Every check runs the reader's own code (src/agent/tools.js `search`, the same model),
|
|
16
16
|
// so a score is what an agent calling fangorn-mcp would get, not what a test harness
|
|
17
17
|
// with its own ranker would.
|
|
18
18
|
//
|