@fangorn-network/westmarch 0.0.10-dev → 0.0.12-dev

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/README.md +32 -23
  2. package/dist/ts/eval.js +1 -1
  3. package/docs/app-to-agent.md +5 -5
  4. package/package.json +39 -36
  5. package/plugin/scripts/fx.mjs +7 -7
  6. package/site/agent.js +2 -2
  7. package/site/dist/assets/embed.worker-CZXpLhAu.js +1 -0
  8. package/site/dist/assets/index-oyUqokZh.js +8 -0
  9. package/site/dist/assets/{ort-wasm-simd-threaded.asyncify-DMmc6YqF.wasm → ort-wasm-simd-threaded.asyncify-CxOG5pUO.wasm} +0 -0
  10. package/site/dist/assets/preload-helper-BaNbYf_w.js +1 -0
  11. package/site/dist/assets/transformers.web-CM54uIeJ.js +40 -0
  12. package/site/dist/assets/transformers.web-D8W1u_I2.js +49 -0
  13. package/site/dist/index.html +2 -2
  14. package/site/feed.js +2 -2
  15. package/site/store.js +3 -3
  16. package/{publish → src/agent}/agent-card.js +3 -3
  17. package/{consume → src/agent}/mcp-bin.js +1 -1
  18. package/{consume → src/agent}/mcp.js +3 -3
  19. package/{consume → src/agent}/tools.js +3 -3
  20. package/{consume → src/agent}/ui.js +1 -1
  21. package/{consume → src/agent}/view-tools.js +4 -4
  22. package/{consume → src/agent}/x402.js +1 -1
  23. package/{consume → src/core}/embed.js +33 -8
  24. package/{consume → src/core}/ort-preload.js +1 -1
  25. package/{consume → src/core}/rank.js +1 -1
  26. package/{consume → src/core}/roles.js +1 -1
  27. package/{consume → src/core}/shard.js +1 -1
  28. package/{consume → src/discover}/apps.js +2 -2
  29. package/{consume → src/discover}/corpora.js +2 -2
  30. package/{consume → src/discover}/directory.js +3 -3
  31. package/{consume → src/discover}/explore.js +56 -28
  32. package/{publish → src/market}/cohort.js +1 -1
  33. package/{publish → src/market}/demand.js +2 -2
  34. package/{publish → src/market}/reactions.js +4 -4
  35. package/{publish → src/market}/settle.js +2 -2
  36. package/{publish → src/market}/terms.js +1 -1
  37. package/{publish → src/publish}/cli.js +2 -2
  38. package/{publish → src/publish}/envelope.js +1 -1
  39. package/{publish → src/publish}/eval.ts +1 -1
  40. package/{consume → src/publish}/lint.js +5 -4
  41. package/{publish → src/publish}/manifest.js +1 -1
  42. package/{publish → src/publish}/ship.js +8 -8
  43. package/{publish → src/publish}/view.js +27 -14
  44. package/{consume → src/taste}/steam.js +1 -1
  45. package/{consume → src/taste}/taste-doc.js +2 -2
  46. package/{consume → src/taste}/taste.js +35 -56
  47. package/{publish → src/taste}/vault.js +1 -1
  48. package/site/dist/assets/embed.worker-Dn3fKctb.js +0 -1
  49. package/site/dist/assets/index-w2wfUwhm.js +0 -8
  50. package/site/dist/assets/preload-helper-uBIymjUX.js +0 -1
  51. package/site/dist/assets/transformers.web-BgUm7F_-.js +0 -46
  52. package/site/dist/assets/transformers.web-u9uSoh5E.js +0 -37
  53. /package/{consume → src/core}/embed.worker.js +0 -0
  54. /package/{publish → src/publish}/enrich.js +0 -0
  55. /package/{publish → src/publish}/graph.js +0 -0
  56. /package/{publish → src/publish}/view-cli.js +0 -0
  57. /package/{publish → src/publish}/westmarch.d.ts +0 -0
@@ -1,8 +1,8 @@
1
1
  <!doctype html>
2
2
  <html lang="en">
3
- <script type="module" crossorigin src="./assets/index-w2wfUwhm.js"></script>
3
+ <script type="module" crossorigin src="./assets/index-oyUqokZh.js"></script>
4
4
  <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-Dd_uD5pT.js">
5
- <link rel="modulepreload" crossorigin href="./assets/preload-helper-uBIymjUX.js">
5
+ <link rel="modulepreload" crossorigin href="./assets/preload-helper-BaNbYf_w.js">
6
6
  <link rel="modulepreload" crossorigin href="./assets/roles-C9V_uDr_.js">
7
7
 
8
8
  <meta charset="utf-8">
package/site/feed.js CHANGED
@@ -2,7 +2,7 @@
2
2
  // occasion (a meeting, an issue, an episode), from the roles the app declared. Nothing here
3
3
  // knows what a row is: an occasion is the rows sharing a subtitle and a facet value, and
4
4
  // its date is their temporal field.
5
- import { pick, subtitleOf, titleOf, values } from "../consume/roles.js";
5
+ import { pick, subtitleOf, titleOf, values } from "../src/core/roles.js";
6
6
 
7
7
  /** The field a reader narrows by first: the app's declared facet, else the first tag with
8
8
  * a handful of values (a town, not every body in every town). */
@@ -45,7 +45,7 @@ export function detail(r, roles) {
45
45
 
46
46
  // ── self-check: `node site/feed.js` ──
47
47
  if (typeof process !== "undefined" && process.argv[1]?.endsWith("/feed.js")) {
48
- const { rolesFrom } = await import("../consume/roles.js");
48
+ const { rolesFrom } = await import("../src/core/roles.js");
49
49
  const roles = rolesFrom([{ role_map: { title: "heading", subtitle: "meeting", temporal: "date", tags: ["city", "body"], text: ["text"] },
50
50
  presentation: { facets: ["city"] } }]);
51
51
  const row = (city, date, heading, text = heading) => ({ city, date, heading, meeting: `Board · ${date}`, body: "Board", text });
package/site/store.js CHANGED
@@ -1,12 +1,12 @@
1
1
  // What a person does on the page, kept on their own device: what they saved, what they
2
2
  // passed on, and what they searched this session. Nothing leaves the browser unless they
3
- // export it. The saved items are also the likes a taste is built from (consume/taste.js),
3
+ // export it. The saved items are also the likes a taste is built from (src/taste/taste.js),
4
4
  // so "For you" is theirs, and so is the file they can hand an agent.
5
5
  //
6
6
  // Saved items are snapshots keyed by the publisher's own id (the identity role), not the
7
7
  // row's CID: a record re-published with a new CID must still be the item they saved.
8
- import { exportTaste, taste } from "../consume/taste.js";
9
- import { packVec, unpackVec } from "../consume/embed.js";
8
+ import { exportTaste, taste } from "../src/taste/taste.js";
9
+ import { packVec, unpackVec } from "../src/core/embed.js";
10
10
 
11
11
  const read = (s, k, dflt) => { try { return JSON.parse(s?.getItem(k) ?? "null") ?? dflt; } catch { return dflt; } };
12
12
  const write = (s, k, v) => { try { s?.setItem(k, JSON.stringify(v)); } catch { /* private mode or full: the page still works */ } };
@@ -106,13 +106,13 @@ export function fangornExtension({ app, fromBlock, namespaces = [], views = [],
106
106
  };
107
107
  }
108
108
 
109
- // ── self-check: `node publish/agent-card.js` ────────────────────────────────
109
+ // ── self-check: `node src/agent/agent-card.js` ────────────────────────────────
110
110
  if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/agent-card.js")) {
111
111
  const eq = (a, b, m) => { if (JSON.stringify(a) !== JSON.stringify(b)) throw new Error(`${m}: ${JSON.stringify(a)} != ${JSON.stringify(b)}`); };
112
112
  const throws = (f, re, m) => { try { f(); } catch (e) { if (re.test(e.message)) return; throw e; } throw new Error(`must throw: ${m}`); };
113
113
 
114
- const { APP_EXTENSION } = await import("../consume/apps.js");
115
- eq(APP_EXTENSION, FANGORN_APP_EXTENSION, "consume/apps.js's copy of the extension uri");
114
+ const { APP_EXTENSION } = await import("../discover/apps.js");
115
+ eq(APP_EXTENSION, FANGORN_APP_EXTENSION, "src/discover/apps.js's copy of the extension uri");
116
116
 
117
117
  let fired = 0;
118
118
  const tools = await captureTools(() => {
@@ -2,5 +2,5 @@
2
2
  // Static imports run in order, so the library is loaded before the server
3
3
  // starts. Not `await import("./mcp.js")`: bun 1.1 then wraps each module in a
4
4
  // function, and their top-level awaits stop parsing.
5
- import "./ort-preload.js";
5
+ import "../core/ort-preload.js";
6
6
  import "./mcp.js";
@@ -29,7 +29,7 @@ import { createHash } from "node:crypto";
29
29
  import net from "node:net";
30
30
  import { homedir } from "node:os";
31
31
  import { createInterface } from "node:readline";
32
- import { APP_EXTENSION, listApps, toApp } from "./apps.js";
32
+ import { APP_EXTENSION, listApps, toApp } from "../discover/apps.js";
33
33
 
34
34
  // The current AppRegistry's first event on Arbitrum Sepolia. A reader's
35
35
  // default, not deployment config: pass --from-block for another deployment.
@@ -522,12 +522,12 @@ const deps = {
522
522
  // ponytail: cached for a minute, like findApps. Two getLogs, a card and catalog per app.
523
523
  exploreApps: async (query) => {
524
524
  if (!explored || Date.now() - exploredAt > 60_000) {
525
- const [ex, { createPublicClient, http }, { arbitrumSepolia }] = await Promise.all([import("./explore.js"), import("viem"), import("viem/chains")]);
525
+ const [ex, { createPublicClient, http }, { arbitrumSepolia }] = await Promise.all([import("../discover/explore.js"), import("viem"), import("viem/chains")]);
526
526
  explored = { ex, dir: await ex.readDirectory(createPublicClient({ chain: arbitrumSepolia, transport: http() })) };
527
527
  exploredAt = Date.now();
528
528
  }
529
529
  const { ex, dir } = explored;
530
- const { EMBED_MODEL } = await import("./embed.js");
530
+ const { EMBED_MODEL } = await import("../core/embed.js");
531
531
  const qv = query?.trim() ? await vt.queryVector(query) : null;
532
532
  const apps = ex.rankApps(dir.apps, query, { qv, model: EMBED_MODEL }).map(ex.brief);
533
533
  return { apps, ranked_by: apps[0]?.ranked_by ?? null };
@@ -11,8 +11,8 @@
11
11
  //
12
12
  // Pure functions, so `node tools.js` checks them with no browser and no network.
13
13
 
14
- import { bestPassage, cosine, lexScore, norm, zFloor } from "./rank.js";
15
- import { briefOf, collections, linkOf, subtitleOf, titleOf, typeOf, values } from "./roles.js";
14
+ import { bestPassage, cosine, lexScore, norm, zFloor } from "../core/rank.js";
15
+ import { briefOf, collections, linkOf, subtitleOf, titleOf, typeOf, values } from "../core/roles.js";
16
16
 
17
17
  /** Fields that are plumbing rather than content — never worth faceting or
18
18
  * showing as "what this corpus holds". */
@@ -232,7 +232,7 @@ export function getRow(rows, id, roles) {
232
232
 
233
233
  // ── self-check: `node tools.js` — no browser, no network ────────────────────
234
234
  if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/tools.js")) {
235
- const { rolesFrom } = await import("./roles.js");
235
+ const { rolesFrom } = await import("../core/roles.js");
236
236
  const row = (id, fields, vector) => ({
237
237
  id, owner: "0xa", ...fields,
238
238
  text: [fields.name, fields.desc].filter(Boolean).join(" "),
@@ -504,7 +504,7 @@ export function stage({ picks = [], heading = "", note = "" } = {}) {
504
504
  <script>window.__ITEMS__ = ${json};${STAGE_JS}<\/script>`;
505
505
  }
506
506
 
507
- // ── self-check: `node consume/ui.js` ────────────────────────────────────────
507
+ // ── self-check: `node src/agent/ui.js` ────────────────────────────────────────
508
508
  if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/ui.js")) {
509
509
  const roles = { measures: ["rating", "year", "runtime", "votes"] };
510
510
  const html = rankedList({
@@ -16,12 +16,12 @@
16
16
  //
17
17
  // `get`/`similar`/`count` answer over what is loaded, and say so.
18
18
 
19
- import { configure, loadShard, trimView } from "./shard.js";
20
- import { rolesFrom, textOf } from "./roles.js";
19
+ import { configure, loadShard, trimView } from "../core/shard.js";
20
+ import { rolesFrom, textOf } from "../core/roles.js";
21
21
  import { browse, facet, getRow, neighbors, search } from "./tools.js";
22
- import { lexScore, rankDomains } from "./rank.js";
22
+ import { lexScore, rankDomains } from "../core/rank.js";
23
23
  import { existsSync } from "node:fs";
24
- import { EMBED_MODEL, embedQueryDirect, warmDirect } from "./embed.js";
24
+ import { EMBED_MODEL, embedQueryDirect, warmDirect } from "../core/embed.js";
25
25
 
26
26
  const ROUTE_TOP = 3;
27
27
 
@@ -62,7 +62,7 @@ export async function payAndFetch(url, { privateKey, account, maxPrice, fetchImp
62
62
  return { status: paid.status, body, price: req.maxAmountRequired, receipt: receiptHeader ? JSON.parse(unb64(receiptHeader)) : null };
63
63
  }
64
64
 
65
- // ── self-check: `node consume/x402.js` — a stand-in seller that checks the signature ──
65
+ // ── self-check: `node src/agent/x402.js` — a stand-in seller that checks the signature ──
66
66
  if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/x402.js")) {
67
67
  const { privateKeyToAccount, generatePrivateKey } = await import("viem/accounts");
68
68
  const { verifyTypedData } = await import("viem");
@@ -13,6 +13,11 @@
13
13
 
14
14
  const MODEL = "nomic-ai/nomic-embed-text-v1.5";
15
15
  const DIM = 256;
16
+ // The document side's precision. q8 everywhere by default (the browser's queries are q8).
17
+ // WESTMARCH_EMBED_DTYPE=fp16 is a different encoder (~0.96 cosine to q8): a view records it
18
+ // and refuses to mix the two (view.js), so switching an app is one --rebake.
19
+ const DTYPE = globalThis.process?.env?.WESTMARCH_EMBED_DTYPE ?? "q8";
20
+ export const EMBED_DTYPE = DTYPE;
16
21
 
17
22
  // Stamped onto every vector a publisher commits. Vectors from different models
18
23
  // (or dims) are not comparable — cosine between them is noise, and silently so.
@@ -25,10 +30,11 @@ let _extractor = null;
25
30
  const extractor = () => (_extractor ??= (async () => {
26
31
  const { pipeline } = await import("@huggingface/transformers");
27
32
  // WESTMARCH_EMBED_DEVICE=cuda puts the publisher's document side on a GPU (Node only; needs
28
- // onnxruntime-node's CUDA binaries). Still q8 at batch 1, so still the same encoder: its
29
- // vectors agree with CPU q8 to ~0.99, closer than the browser's WASM q8 queries (~0.98).
33
+ // onnxruntime-node's CUDA binaries and the matching CUDA runtime, cuBLAS, cuRAND and cuDNN on
34
+ // LD_LIBRARY_PATH). q8's integer ops do not run on CUDA, so q8 there is slower than on CPU:
35
+ // pair it with WESTMARCH_EMBED_DTYPE=fp16, which is ~4x faster than CPU q8, batched.
30
36
  const device = globalThis.process?.env?.WESTMARCH_EMBED_DEVICE;
31
- return pipeline("feature-extraction", MODEL, { dtype: "q8", ...(device && { device }) });
37
+ return pipeline("feature-extraction", MODEL, { dtype: DTYPE, ...(device && { device }) });
32
38
  })());
33
39
 
34
40
  /** Standardize over the full vector, slice to `dim`, L2-normalize the slice. */
@@ -59,11 +65,30 @@ export async function embedQueryDirect(text) {
59
65
  return matryoshka(output.data);
60
66
  }
61
67
 
62
- /** One document, in-process. The batch loop stays with the caller — see embedDocuments. */
63
- export async function embedDocumentDirect(text) {
64
- const ex = await extractor();
65
- const output = await ex(`search_document: ${text}`, { pooling: "mean", normalize: false });
66
- return matryoshka(output.data);
68
+ /** One document, in-process. Calls made together are embedded as one batch (a GPU's
69
+ * speed); q8 stays at batch 1, since batching moves its activation quantization. */
70
+ export function embedDocumentDirect(text) {
71
+ return new Promise((resolve, reject) => {
72
+ pending.push({ text, resolve, reject });
73
+ if (pending.length === 1) setTimeout(flushDocuments, 0);
74
+ });
75
+ }
76
+ const BATCH = DTYPE === "q8" ? 1 : 32;
77
+ let pending = [];
78
+ async function flushDocuments() {
79
+ const all = pending;
80
+ pending = [];
81
+ try {
82
+ const ex = await extractor();
83
+ for (let i = 0; i < all.length; i += BATCH) {
84
+ const part = all.slice(i, i + BATCH);
85
+ const output = await ex(part.map((p) => `search_document: ${p.text}`), { pooling: "mean", normalize: false });
86
+ const d = output.dims.at(-1);
87
+ part.forEach((p, j) => p.resolve(matryoshka(output.data.slice(j * d, (j + 1) * d))));
88
+ }
89
+ } catch (e) {
90
+ for (const p of all) p.reject(e);
91
+ }
67
92
  }
68
93
 
69
94
  /** Load the model in-process, and settle when it is actually ready. */
@@ -9,7 +9,7 @@ import { dlopen } from "bun:ffi";
9
9
  import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
10
10
  import { homedir } from "node:os";
11
11
  // build-bin.js puts the target's library here before it compiles.
12
- import lib from "../dist/ort/libonnxruntime.so.1" with { type: "file" };
12
+ import lib from "../../dist/ort/libonnxruntime.so.1" with { type: "file" };
13
13
 
14
14
  try {
15
15
  const dir = `${homedir()}/.cache/fangorn-mcp/lib/${lib.split("/").pop()}`; // bun names it by content hash
@@ -158,7 +158,7 @@ export function rankDomains(domains = [], q) {
158
158
  .sort((a, b) => (b.affinity ?? -Infinity) - (a.affinity ?? -Infinity));
159
159
  }
160
160
 
161
- // ── self-check: `node consume/rank.js` ──────────────────────────────────────
161
+ // ── self-check: `node src/core/rank.js` ──────────────────────────────────────
162
162
  if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/rank.js")) {
163
163
  // Word boundaries: the bug this replaced ranked "intoxicated" for "cat".
164
164
  if (lexScore({ text: "intoxicated education" }, "cat") !== 0) throw new Error("substring matched — word boundaries lost");
@@ -366,7 +366,7 @@ export function actionsOf(row, roles) {
366
366
  }).filter((a) => a.locked.length || !a.missing.length);
367
367
  }
368
368
 
369
- // ── self-check: `node consume/roles.js` ─────────────────────────────────────
369
+ // ── self-check: `node src/core/roles.js` ─────────────────────────────────────
370
370
  if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/roles.js")) {
371
371
  // The real places manifest's shape.
372
372
  const places = {
@@ -437,7 +437,7 @@ export async function suggestDomains(q, { url } = {}) {
437
437
  return rankDomains(domains, q);
438
438
  }
439
439
 
440
- // ── self-check: `node consume/shard.js` — no network, no view ────────────────
440
+ // ── self-check: `node src/core/shard.js` — no network, no view ────────────────
441
441
  if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/shard.js")) {
442
442
  const { gzipSync } = await import("node:zlib");
443
443
  const warn = console.warn, info = console.info;
@@ -30,7 +30,7 @@ function safeView(url) {
30
30
  }
31
31
 
32
32
  /** The SDK's FANGORN_APP_EXTENSION. Copied so this file needs no SDK;
33
- * publish/agent-card.js's self-check fails if the two drift. */
33
+ * src/agent/agent-card.js's self-check fails if the two drift. */
34
34
  export const APP_EXTENSION = "https://fangorn.network/a2a/app/v1";
35
35
 
36
36
  /** A verified card → an app entry. `views` is westmarch's param; the SDK does not read it. */
@@ -70,7 +70,7 @@ export async function listApps(fangorn, { fromBlock, toBlock } = {}) {
70
70
  return { apps, rejected };
71
71
  }
72
72
 
73
- // ── self-check: `node consume/apps.js` ──────────────────────────────────────
73
+ // ── self-check: `node src/discover/apps.js` ──────────────────────────────────────
74
74
  if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/apps.js")) {
75
75
  const j = (x) => JSON.stringify(x, (_, v) => (typeof v === "bigint" ? `${v}n` : v));
76
76
  const eq = (a, b, m) => { if (j(a) !== j(b)) throw new Error(`${m}: ${j(a)} != ${j(b)}`); };
@@ -15,7 +15,7 @@
15
15
  // each and a person opens a handful; when a session holds enough of them to
16
16
  // matter, drop the least recently focused — the map is already keyed for it.
17
17
 
18
- import { trimView } from "./shard.js";
18
+ import { trimView } from "../core/shard.js";
19
19
 
20
20
  /**
21
21
  * A set of open corpora, and the vocabulary for naming one.
@@ -188,7 +188,7 @@ export function follow(picks = [], corpora = [], { at = null } = {}) {
188
188
  return out;
189
189
  }
190
190
 
191
- // ── self-check: `node consume/corpora.js` ───────────────────────────────────
191
+ // ── self-check: `node src/discover/corpora.js` ───────────────────────────────────
192
192
  if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/corpora.js")) {
193
193
  const s = session();
194
194
  const add = (view, name, n) => {
@@ -22,8 +22,8 @@
22
22
  // still well under a second. Past that, the directory needs its own coverage
23
23
  // index, which is the same trick one level up.
24
24
 
25
- import { trimView } from "./shard.js";
26
- import { rankDomains, zFloor } from "./rank.js";
25
+ import { trimView } from "../core/shard.js";
26
+ import { rankDomains, zFloor } from "../core/rank.js";
27
27
  import { listApps } from "./apps.js";
28
28
 
29
29
  /** Catalog URL for a view base, tolerating either form the registry prints. */
@@ -167,7 +167,7 @@ export async function sourcesFromChain(fangorn, opts) {
167
167
  return { sources: apps.flatMap((a) => a.views.map((view) => ({ view, name: a.name, appId: a.appId }))), rejected };
168
168
  }
169
169
 
170
- // ── self-check: `node consume/directory.js` ─────────────────────────────────
170
+ // ── self-check: `node src/discover/directory.js` ─────────────────────────────────
171
171
  if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/directory.js")) {
172
172
  const warn = console.warn; console.warn = () => {};
173
173
  const cov = (...vectors) => ({ dim: 2, sampled: 100, vectors, counts: vectors.map(() => 50) });
@@ -8,26 +8,30 @@
8
8
  // ERC-8004 identity `Registered` the agent minted for the app
9
9
  // ERC-8004 reputation feedback count and average per agent
10
10
  //
11
- // Both registries answer the whole range in one `eth_getLogs` on Arbitrum
12
- // Sepolia (39 and ~1.5k logs today), so the page and the MCP tool read the same
13
- // chain the same way, and a new app shows up on the next read.
11
+ // Both registries are scanned from their first event in 10M-block windows, in
12
+ // parallel, so the page and the MCP tool read the same chain the same way, and
13
+ // a new app shows up on the next read.
14
14
  //
15
15
  // `client` is a viem PublicClient (getLogs, readContract): passed in, so this
16
16
  // runs in a tab and in node, and the self-check needs no chain.
17
17
  //
18
- // ponytail: one getLogs per registry from block 0. An RPC that caps log ranges
19
- // will refuse it; window the scan then (the SDK's getLogsInWindows).
18
+ // ponytail: windows grow with the chain (1 + 8 at block 316M, ~0.2s). Index
19
+ // the logs somewhere if that ever gets slow.
20
20
 
21
21
  import { parseAbi, parseAbiItem } from "viem";
22
22
  import { APP_EXTENSION } from "./apps.js";
23
23
  import { survey, comparable } from "./directory.js";
24
- import { rankDomains, zFloor, lexScore } from "./rank.js";
24
+ import { rankDomains, zFloor, lexScore } from "../core/rank.js";
25
25
 
26
26
  export const CHAIN = {
27
27
  chainId: 421614,
28
28
  appRegistry: "0x11d228c4774af3d9cae3b4b6874a12576a1a83ec",
29
29
  identity: "0x8004A818BFB912233c491871b3d84c89A494BD9e",
30
30
  reputation: "0x8004B663056A597Dffe9eCcC1965A193B7388713",
31
+ // Each registry's first event (2026-10-05): nothing to read before it.
32
+ // Scanning from 0 instead was 32 windows per registry, ~5s of a 7.7s load.
33
+ appRegistryFrom: 311_639_205n,
34
+ identityFrom: 241_557_787n,
31
35
  scan: (agentId) => `https://8004scan.io/agents/arbitrum-sepolia/${agentId}`,
32
36
  };
33
37
 
@@ -101,54 +105,78 @@ async function reputation(client, agentId, chain) {
101
105
  }
102
106
  }
103
107
 
108
+ // The public Arbitrum Sepolia RPC refuses an address+event getLogs spanning more
109
+ // than 10M blocks (from 2026-10; one call from 0 worked before).
110
+ const LOG_WINDOW = 10_000_000n;
111
+
112
+ async function getLogsSince(client, start, filter) {
113
+ const head = await client.getBlockNumber();
114
+ const windows = [];
115
+ for (let from = start; from <= head; from += LOG_WINDOW) {
116
+ const to = from + LOG_WINDOW - 1n < head ? from + LOG_WINDOW - 1n : head;
117
+ windows.push(client.getLogs({ ...filter, fromBlock: from, toBlock: to }));
118
+ }
119
+ return (await Promise.all(windows)).flat(); // in window order: oldest first still holds
120
+ }
121
+
104
122
  /**
105
123
  * Every verified app, with its 8004 agent, reputation, and catalog.
106
124
  * Unverified apps are dropped, not listed; `dropped` says why, for debugging.
125
+ *
126
+ * `onUpdate({ apps, pending })`, if given, is called once the bindings are read
127
+ * and again as each app lands or drops, so a page can show apps as they arrive.
128
+ * `pending` counts the bound apps not yet settled.
107
129
  */
108
- export async function readDirectory(client, { chain = CHAIN } = {}) {
130
+ export async function readDirectory(client, { chain = CHAIN, onUpdate = () => {} } = {}) {
109
131
  const [bindLogs, regLogs] = await Promise.all([
110
- client.getLogs({ address: chain.appRegistry, event: APP_AGENT_CHANGED, fromBlock: 0n, toBlock: "latest" }),
111
- client.getLogs({ address: chain.identity, event: REGISTERED, fromBlock: 0n, toBlock: "latest" }),
132
+ getLogsSince(client, chain.appRegistryFrom ?? 0n, { address: chain.appRegistry, event: APP_AGENT_CHANGED }),
133
+ getLogsSince(client, chain.identityFrom ?? 0n, { address: chain.identity, event: REGISTERED }),
112
134
  ]);
113
135
  const registered = regLogs.map((l) => ({ agentId: l.args.agentId, owner: l.args.owner, block: l.blockNumber }));
114
136
  const latest = new Map(); // logs come oldest first: the last write wins
115
137
  for (const l of bindLogs) latest.set(l.args.app_id, { uri: l.args.agent_uri, block: l.blockNumber });
138
+ const bound = [...latest].filter(([, { uri }]) => uri); // an empty uri is unbound on purpose
116
139
 
117
- // Two passes: verify every card (small), then read catalogs (Quorum's is
118
- // 1.7 MB, Kingsfoil has 23). Only a card failure drops an app; an owner or
119
- // catalog that fails leaves it listed with less to say about it.
120
- const dropped = [];
121
- const verified = (await Promise.all([...latest].map(async ([appId, { uri, block }]) => {
122
- if (!uri) return null; // unbound on purpose
140
+ // One pipeline per app, so the quick ones need not wait on Quorum's 1.7 MB
141
+ // catalog. Only a card failure drops an app; an owner or catalog that fails
142
+ // leaves it listed with less to say about it.
143
+ const apps = [], dropped = [];
144
+ let pending = bound.length;
145
+ const settle = () => { pending--; apps.sort((a, b) => a.name.localeCompare(b.name)); onUpdate({ apps: [...apps], pending }); };
146
+ onUpdate({ apps: [], pending });
147
+ await Promise.all(bound.map(async ([appId, { uri, block }]) => {
148
+ let card;
123
149
  try {
124
150
  if (!/^https:\/\//.test(uri)) throw new Error("card is not https");
125
- const card = await json(uri);
151
+ card = await json(uri);
126
152
  const why = verifyCard(card, appId, chain);
127
153
  if (why) throw new Error(why);
128
- return { appId, uri, block, card };
129
- } catch (e) { dropped.push({ card: uri, why: e?.message ?? String(e) }); return null; }
130
- }))).filter(Boolean);
154
+ } catch (e) { dropped.push({ card: uri, why: e?.message ?? String(e) }); return settle(); }
131
155
 
132
- const apps = await Promise.all(verified.map(async ({ appId, uri, block, card }) => {
133
156
  const p = card.capabilities.extensions.find((e) => e.uri === APP_EXTENSION).params;
134
- const owner = await client.readContract({ address: chain.appRegistry, abi: APPS, functionName: "getAppOwner", args: [appId] }).catch(() => null);
135
- const agentId = owner && pairAgent(registered, owner, block);
136
- const { corpora, unreachable } = await survey((p.views ?? []).filter((v) => /^https:\/\//.test(v)));
157
+ const [[owner, agentId, rep], { corpora, unreachable }] = await Promise.all([
158
+ client.readContract({ address: chain.appRegistry, abi: APPS, functionName: "getAppOwner", args: [appId] }).catch(() => null)
159
+ .then(async (owner) => {
160
+ const agentId = owner && pairAgent(registered, owner, block);
161
+ return [owner, agentId, agentId == null ? null : await reputation(client, agentId, chain)];
162
+ }),
163
+ survey((p.views ?? []).filter((v) => /^https:\/\//.test(v))),
164
+ ]);
137
165
  const name = typeof card.name === "string" && card.name.trim() ? card.name.trim() : `App ${appId.slice(0, 10)}`;
138
- return {
166
+ apps.push({
139
167
  app: slug(name), name, description: typeof card.description === "string" ? card.description : "",
140
168
  site: https(card.url), card: uri, appId, owner,
141
169
  boundAt: Number(block), fromBlock: Number(p.fromBlock),
142
170
  tools: (card.skills ?? []).map((s) => s.id),
143
171
  agent: agentId == null ? null : { id: Number(agentId), scan: chain.scan(agentId) },
144
- reputation: agentId == null ? null : await reputation(client, agentId, chain),
172
+ reputation: rep,
145
173
  // null, not 0, when no catalog answered: "0 records" would claim it was counted.
146
174
  rows: corpora.length ? corpora.reduce((n, c) => n + (c.rows || 0), 0) : null,
147
175
  unreachable: unreachable.length,
148
176
  domains: corpora.map((c) => ({ name: c.domain, description: c.description, rows: c.rows, coverage: c.coverage, model: c.model })),
149
- };
177
+ });
178
+ settle();
150
179
  }));
151
- apps.sort((a, b) => a.name.localeCompare(b.name));
152
180
  return { apps, dropped };
153
181
  }
154
182
 
@@ -200,7 +228,7 @@ export function rankApps(apps, query, { qv = null, model = null } = {}) {
200
228
  * domains, so an app reports how many, and which one matched. */
201
229
  export const brief = ({ domains, ...a }) => ({ ...a, domains: domains.length });
202
230
 
203
- // ── self-check: `node consume/explore.js` ──────────────────────────────────
231
+ // ── self-check: `node src/discover/explore.js` ──────────────────────────────────
204
232
  if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/explore.js")) {
205
233
  const j = (x) => JSON.stringify(x, (_, v) => (typeof v === "bigint" ? `${v}n` : v));
206
234
  const eq = (a, b, m) => { if (j(a) !== j(b)) throw new Error(`${m}: ${j(a)} != ${j(b)}`); };
@@ -246,7 +246,7 @@ export function readout(total, cohort, { minSupport = MIN_SUPPORT } = {}) {
246
246
  };
247
247
  }
248
248
 
249
- // ── self-check: `node publish/cohort.js` ──────────────────────────────────
249
+ // ── self-check: `node src/market/cohort.js` ──────────────────────────────────
250
250
  if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/cohort.js")) {
251
251
  const vec = (a, b) => { const v = new Float32Array(STAT_DIM); v[0] = a; v[1] = b; return v; };
252
252
  const log = (n, seed) => Array.from({ length: n }, (_, i) => ({
@@ -45,7 +45,7 @@
45
45
  // says what was measured and what the publisher owns near it. Everything past
46
46
  // that is a model of a market that has run five simulated rounds.
47
47
 
48
- import { vocabulary, affinities } from "../consume/taste-doc.js";
48
+ import { vocabulary, affinities } from "../taste/taste-doc.js";
49
49
 
50
50
  /** Matched reactions per reader below which a region is not drawing attention.
51
51
  * 0.5 = the median reader reacted to something here at least every other
@@ -382,7 +382,7 @@ export function brief(rep, { title = "demand" } = {}) {
382
382
  return L.join("\n");
383
383
  }
384
384
 
385
- // ── self-check: `node publish/demand.js` ────────────────────────────────────
385
+ // ── self-check: `node src/market/demand.js` ────────────────────────────────────
386
386
  if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/demand.js")) {
387
387
  const { keypair, statistics, contribute, aggregate, readout } = await import("./cohort.js");
388
388
 
@@ -42,7 +42,7 @@
42
42
  // Nothing is lost, because nobody buys row 47 of a stranger's reactions. A taste
43
43
  // corpus is bought WHOLE, and the question a buyer actually asks — "does this
44
44
  // reader have dense signal where I am training?" — is answered by the centroids
45
- // alone. That is already how `consume/directory.js` ranks a corpus nobody has
45
+ // alone. That is already how `src/discover/directory.js` ranks a corpus nobody has
46
46
  // downloaded. Per-row vectors served no buyer and leaked the entire product.
47
47
  //
48
48
  // The centroids are themselves averages, so `k` is capped to keep at least
@@ -50,7 +50,7 @@
50
50
  // IS that reaction's vector under another name, which is the same leak wearing a
51
51
  // hat, and a reader with nine reactions is exactly who most needs this to hold.
52
52
 
53
- import { packVec } from "../consume/embed.js";
53
+ import { packVec } from "../core/embed.js";
54
54
  import { lineage } from "./terms.js";
55
55
 
56
56
  // 32, not 8: on Kingsfoil the top 3 of 23 views held 92% of the true top 10 with 32
@@ -168,7 +168,7 @@ export function reactionCorpus(events = [], {
168
168
 
169
169
  // The split is applied HERE, to the rows themselves, not merely declared in
170
170
  // the manifest. Declaring it and shipping the fields anyway is the exact bug
171
- // `consume/lint.js` calls "paywall names fields the free shard ships anyway",
171
+ // `src/publish/lint.js` calls "paywall names fields the free shard ships anyway",
172
172
  // and this file had it: it returned every field on every row and left the
173
173
  // withholding to a bake that does not run on this path.
174
174
  const wire = [], locked = [];
@@ -236,7 +236,7 @@ export function reactionCorpus(events = [], {
236
236
  return { manifest, rows: wire, locked, lineage: up, byCorpus: Object.fromEntries(byCorpus) };
237
237
  }
238
238
 
239
- // ── self-check: `node publish/reactions.js` ────────────────────────────────
239
+ // ── self-check: `node src/market/reactions.js` ────────────────────────────────
240
240
  if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/reactions.js")) {
241
241
  const { splitPayment } = await import("./terms.js");
242
242
  const A = (n) => `0x${String(n).repeat(40).slice(0, 40)}`;
@@ -16,7 +16,7 @@ import { bytesToHex } from "viem";
16
16
  import {
17
17
  CHUNK_SIZE, NONCE_LEN, aadFor, chunkKey, deleteResource,
18
18
  getWorkerPubkey, pack, putChunk, unpack,
19
- } from "./envelope.js";
19
+ } from "../publish/envelope.js";
20
20
 
21
21
  export { CHUNK_SIZE, aadFor, chunkKey, deleteResource, getWorkerPubkey };
22
22
 
@@ -64,7 +64,7 @@ export async function encryptAndUpload({ file, resourceId, workerUrl, uploadToke
64
64
  }
65
65
  }
66
66
 
67
- // ── self-check: `node publish/settle.js` — no network, no worker ─────────────
67
+ // ── self-check: `node src/market/settle.js` — no network, no worker ─────────────
68
68
  //
69
69
  // It decrypts with bare AES-GCM rather than importing a buyer, because what is
70
70
  // under test here is the ENCRYPTOR: chunk addressing, the AAD index binding, the
@@ -136,7 +136,7 @@ export function splitPayment(amount, { terms, publisher, lineage: up = [] } = {}
136
136
  * price anyone reads. */
137
137
  export const label = (base, currency = "USDC") => `${(Number(base) / 1e6).toFixed(2)} ${currency}`;
138
138
 
139
- // ── self-check: `node publish/terms.js` ────────────────────────────────────
139
+ // ── self-check: `node src/market/terms.js` ────────────────────────────────────
140
140
  if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/terms.js")) {
141
141
  const A = "0x" + "a".repeat(40); // app owner
142
142
  const P = "0x" + "b".repeat(40); // publisher
@@ -47,7 +47,7 @@ import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
47
47
  import { basename, join, relative, resolve, sep } from "node:path";
48
48
  import { createPublicClient, createWalletClient, defineChain, http } from "viem";
49
49
  import { privateKeyToAccount } from "viem/accounts";
50
- import { encryptAndUpload } from "./settle.js";
50
+ import { encryptAndUpload } from "../market/settle.js";
51
51
  import { resourceIdFor } from "./envelope.js";
52
52
  import { newUid, readManifest, writeManifest } from "./manifest.js";
53
53
 
@@ -299,7 +299,7 @@ async function main() {
299
299
  for (const [path, ptr] of Object.entries(sealed.published)) console.log(` ${path} ${ptr.resourceId}`);
300
300
  }
301
301
 
302
- // ── self-check: `node publish/cli.js --selfcheck` (offline) ──────────────────
302
+ // ── self-check: `node src/publish/cli.js --selfcheck` (offline) ──────────────────
303
303
 
304
304
  if (flags.selfcheck) {
305
305
  const assert = (cond, msg) => { if (!cond) throw new Error(msg); };
@@ -150,7 +150,7 @@ export async function deleteResource({ workerUrl, uploadToken, resourceId, chunk
150
150
  return Math.max(1, chunks);
151
151
  }
152
152
 
153
- // ── self-check: `node publish/envelope.js` ───────────────────────────────────
153
+ // ── self-check: `node src/publish/envelope.js` ───────────────────────────────────
154
154
  //
155
155
  // These vectors are PINNED to the values sond3r's own envelope.js produced before
156
156
  // this file was extracted from it, with ns = "sond3r". They are not a style test:
@@ -12,7 +12,7 @@
12
12
  // them. So: build a base view (main's recipe) and a candidate (the change) from the
13
13
  // same data, on the same machine, and run both through here. The exit code is the gate.
14
14
  //
15
- // Every check runs the reader's own code (consume/tools.js `search`, the same model),
15
+ // Every check runs the reader's own code (src/agent/tools.js `search`, the same model),
16
16
  // so a score is what an agent calling fangorn-mcp would get, not what a test harness
17
17
  // with its own ranker would.
18
18
  //