@fangorn-network/westmarch 0.0.11-dev → 0.0.12-dev

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fangorn-network/westmarch",
-  "version": "0.0.11-dev",
+  "version": "0.0.12-dev",
   "description": "Publisher and consumer surfaces for a Fangorn app: shape a graph, publish it, read it back as a searchable, agent-drivable catalog",
   "license": "MIT",
   "type": "module",

package/src/core/embed.js

@@ -13,6 +13,11 @@
 
 const MODEL = "nomic-ai/nomic-embed-text-v1.5";
 const DIM = 256;
+// The document side's precision. q8 everywhere by default (the browser's queries are q8).
+// WESTMARCH_EMBED_DTYPE=fp16 is a different encoder (~0.96 cosine to q8): a view records it
+// and refuses to mix the two (view.js), so switching an app is one --rebake.
+const DTYPE = globalThis.process?.env?.WESTMARCH_EMBED_DTYPE ?? "q8";
+export const EMBED_DTYPE = DTYPE;
 
 // Stamped onto every vector a publisher commits. Vectors from different models
 // (or dims) are not comparable — cosine between them is noise, and silently so.
@@ -25,10 +30,11 @@ let _extractor = null;
 const extractor = () => (_extractor ??= (async () => {
     const { pipeline } = await import("@huggingface/transformers");
     // WESTMARCH_EMBED_DEVICE=cuda puts the publisher's document side on a GPU (Node only; needs
-    // onnxruntime-node's CUDA binaries). Still q8 at batch 1, so still the same encoder: its
-    // vectors agree with CPU q8 to ~0.99, closer than the browser's WASM q8 queries (~0.98).
+    // onnxruntime-node's CUDA binaries and the matching CUDA runtime, cuBLAS, cuRAND and cuDNN on
+    // LD_LIBRARY_PATH). q8's integer ops do not run on CUDA, so q8 there is slower than on CPU:
+    // pair it with WESTMARCH_EMBED_DTYPE=fp16, which is ~4x faster than CPU q8, batched.
     const device = globalThis.process?.env?.WESTMARCH_EMBED_DEVICE;
-    return pipeline("feature-extraction", MODEL, { dtype: "q8", ...(device && { device }) });
+    return pipeline("feature-extraction", MODEL, { dtype: DTYPE, ...(device && { device }) });
 })());
 
 /** Standardize over the full vector, slice to `dim`, L2-normalize the slice. */
@@ -59,11 +65,30 @@ export async function embedQueryDirect(text) {
     return matryoshka(output.data);
 }
 
-/** One document, in-process. The batch loop stays with the caller — see embedDocuments. */
-export async function embedDocumentDirect(text) {
-    const ex = await extractor();
-    const output = await ex(`search_document: ${text}`, { pooling: "mean", normalize: false });
-    return matryoshka(output.data);
+/** One document, in-process. Calls made together are embedded as one batch (a GPU's
+ *  speed); q8 stays at batch 1, since batching moves its activation quantization. */
+export function embedDocumentDirect(text) {
+    return new Promise((resolve, reject) => {
+        pending.push({ text, resolve, reject });
+        if (pending.length === 1) setTimeout(flushDocuments, 0);
+    });
+}
+const BATCH = DTYPE === "q8" ? 1 : 32;
+let pending = [];
+async function flushDocuments() {
+    const all = pending;
+    pending = [];
+    try {
+        const ex = await extractor();
+        for (let i = 0; i < all.length; i += BATCH) {
+            const part = all.slice(i, i + BATCH);
+            const output = await ex(part.map((p) => `search_document: ${p.text}`), { pooling: "mean", normalize: false });
+            const d = output.dims.at(-1);
+            part.forEach((p, j) => p.resolve(matryoshka(output.data.slice(j * d, (j + 1) * d))));
+        }
+    } catch (e) {
+        for (const p of all) p.reject(e);
+    }
 }
 
 /** Load the model in-process, and settle when it is actually ready. */

package/src/discover/explore.js

@@ -122,8 +122,12 @@ async function getLogsSince(client, start, filter) {
 /**
  * Every verified app, with its 8004 agent, reputation, and catalog.
  * Unverified apps are dropped, not listed; `dropped` says why, for debugging.
+ *
+ * `onUpdate({ apps, pending })`, if given, is called once the bindings are read
+ * and again as each app lands or drops, so a page can show apps as they arrive.
+ * `pending` counts the bound apps not yet settled.
  */
-export async function readDirectory(client, { chain = CHAIN } = {}) {
+export async function readDirectory(client, { chain = CHAIN, onUpdate = () => {} } = {}) {
     const [bindLogs, regLogs] = await Promise.all([
         getLogsSince(client, chain.appRegistryFrom ?? 0n, { address: chain.appRegistry, event: APP_AGENT_CHANGED }),
         getLogsSince(client, chain.identityFrom ?? 0n, { address: chain.identity, event: REGISTERED }),
@@ -131,23 +135,24 @@ export async function readDirectory(client, { chain = CHAIN } = {}) {
     const registered = regLogs.map((l) => ({ agentId: l.args.agentId, owner: l.args.owner, block: l.blockNumber }));
     const latest = new Map();   // logs come oldest first: the last write wins
     for (const l of bindLogs) latest.set(l.args.app_id, { uri: l.args.agent_uri, block: l.blockNumber });
+    const bound = [...latest].filter(([, { uri }]) => uri);   // an empty uri is unbound on purpose
 
-    // Two passes: verify every card (small), then read catalogs (Quorum's is
-    // 1.7 MB, Kingsfoil has 23). Only a card failure drops an app; an owner or
-    // catalog that fails leaves it listed with less to say about it.
-    const dropped = [];
-    const verified = (await Promise.all([...latest].map(async ([appId, { uri, block }]) => {
-        if (!uri) return null;   // unbound on purpose
+    // One pipeline per app, so the quick ones need not wait on Quorum's 1.7 MB
+    // catalog. Only a card failure drops an app; an owner or catalog that fails
+    // leaves it listed with less to say about it.
+    const apps = [], dropped = [];
+    let pending = bound.length;
+    const settle = () => { pending--; apps.sort((a, b) => a.name.localeCompare(b.name)); onUpdate({ apps: [...apps], pending }); };
+    onUpdate({ apps: [], pending });
+    await Promise.all(bound.map(async ([appId, { uri, block }]) => {
+        let card;
         try {
             if (!/^https:\/\//.test(uri)) throw new Error("card is not https");
-            const card = await json(uri);
+            card = await json(uri);
             const why = verifyCard(card, appId, chain);
             if (why) throw new Error(why);
-            return { appId, uri, block, card };
-        } catch (e) { dropped.push({ card: uri, why: e?.message ?? String(e) }); return null; }
-    }))).filter(Boolean);
+        } catch (e) { dropped.push({ card: uri, why: e?.message ?? String(e) }); return settle(); }
 
-    const apps = await Promise.all(verified.map(async ({ appId, uri, block, card }) => {
         const p = card.capabilities.extensions.find((e) => e.uri === APP_EXTENSION).params;
         const [[owner, agentId, rep], { corpora, unreachable }] = await Promise.all([
             client.readContract({ address: chain.appRegistry, abi: APPS, functionName: "getAppOwner", args: [appId] }).catch(() => null)
@@ -158,7 +163,7 @@ export async function readDirectory(client, { chain = CHAIN } = {}) {
             survey((p.views ?? []).filter((v) => /^https:\/\//.test(v))),
         ]);
         const name = typeof card.name === "string" && card.name.trim() ? card.name.trim() : `App ${appId.slice(0, 10)}`;
-        return {
+        apps.push({
             app: slug(name), name, description: typeof card.description === "string" ? card.description : "",
             site: https(card.url), card: uri, appId, owner,
             boundAt: Number(block), fromBlock: Number(p.fromBlock),
@@ -169,9 +174,9 @@ export async function readDirectory(client, { chain = CHAIN } = {}) {
             rows: corpora.length ? corpora.reduce((n, c) => n + (c.rows || 0), 0) : null,
             unreachable: unreachable.length,
             domains: corpora.map((c) => ({ name: c.domain, description: c.description, rows: c.rows, coverage: c.coverage, model: c.model })),
-        };
+        });
+        settle();
     }));
-    apps.sort((a, b) => a.name.localeCompare(b.name));
     return { apps, dropped };
 }
 

package/src/publish/ship.js

@@ -161,7 +161,7 @@ function assertSameDeployment(cli) {
 
 // ── the steps ─────────────────────────────────────────────────────────────────
 
-export async function ship(configPath, { crawl = true, deploy = true, dry = false, replace = false, only = null, register = false } = {}) {
+export async function ship(configPath, { crawl = true, deploy = true, dry = false, replace = false, only = null, register = false, rebake = false } = {}) {
     const cfg = loadConfig(configPath);
     if (!register && (!Object.keys(cfg.types).length || !cfg.sources.length))
         throw new Error(`${configPath}: no types or sources yet, so nothing to build; --register sets up the app alone`);
@@ -246,7 +246,7 @@ export async function ship(configPath, { crawl = true, deploy = true, dry = fals
     const namespaces = [...new Set(cfg.sources.map((s) => s.namespace))];
     if (register && !dry) view.writeCatalog(join(site, "view"));   // empty, or what an earlier build left
     const report = dry || register ? {} : await view.publishView({ fangorn, app: cfg.app, namespaces, out: join(site, "view"),
-        fromBlock: BigInt(state.fromBlock), embed: view.cachedEmbed(join(dir, "vectors.ndjson")), log });
+        fromBlock: BigInt(state.fromBlock), embed: view.cachedEmbed(join(dir, "vectors.ndjson")), rebake, log });
 
     // 6. the page, the card, the headers
     if (!existsSync(join(STOCK, "index.html"))) throw new Error(`no stock page at ${STOCK}; run \`npx vite build\` in westmarch/site`);
@@ -480,7 +480,7 @@ if (process.argv[1]?.endsWith("ship.js") || process.argv[1]?.endsWith("westmarch
     }
     const [path, ...flags] = process.argv.slice(2);
     if (!path || flags.includes("--help")) {
-        console.error("usage: westmarch-ship app.json [--no-crawl] [--no-deploy] [--dry-run] [--replace] [--only ns1,ns2]\n" +
+        console.error("usage: westmarch-ship app.json [--no-crawl] [--no-deploy] [--dry-run] [--replace] [--rebake] [--only ns1,ns2]\n" +
                       "       westmarch-ship app.json --register [--dry-run]   (claim, join, empty site, card bound: no data)\n" +
                       "       westmarch-ship app.json --local <out> [--crawl] [--only ns1,ns2]   (no chain, no deploy)");
         process.exit(path ? 0 : 2);
@@ -492,7 +492,7 @@ if (process.argv[1]?.endsWith("ship.js") || process.argv[1]?.endsWith("westmarch
         process.exit(0);
     }
     await ship(path, { crawl: !flags.includes("--no-crawl"), deploy: !flags.includes("--no-deploy"), dry: flags.includes("--dry-run"),
-                       replace: flags.includes("--replace"), register: flags.includes("--register"),
+                       replace: flags.includes("--replace"), register: flags.includes("--register"), rebake: flags.includes("--rebake"),
                        only })
         .catch((e) => { console.error(`[ship] ✗ ${e.message}`); process.exit(1); });
     process.exit(0);   // a CUDA embedder keeps the process alive (and can abort in its teardown)

package/src/publish/view.js

@@ -32,12 +32,12 @@
 import { gunzipSync, gzipSync } from "node:zlib";
 import { createHash } from "node:crypto";
 import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
-import { EMBED_DIM, EMBED_MODEL, embedDocumentDirect, packVec, unpackVec } from "../core/embed.js";
+import { EMBED_DIM, EMBED_DTYPE, EMBED_MODEL, embedDocumentDirect, packVec, unpackVec } from "../core/embed.js";
 import { coverage as fitCoverage } from "../market/reactions.js";
 
 export const SCHEMA_NAMESPACE = "fangorn.schema";
 export const TYPE_TAG = "fangorn.type.v1";
-export const EMBEDDER = { runtime: "transformers.js", model: EMBED_MODEL, dtype: "q8" };
+export const EMBEDDER = { runtime: "transformers.js", model: EMBED_MODEL, dtype: EMBED_DTYPE };
 const SINGULAR = ["identity", "title", "subtitle", "temporal", "spatial", "media"];
 const MULTI = ["tags", "measures", "relations", "text"];
 const ZERO = "0x0000000000000000000000000000000000000000";
@@ -263,11 +263,13 @@ async function publishDomain({ fangorn, schema, publisher, ns, domain, dir, reba
 
     let n = 0;
     const lines = [];
-    for (const id of fresh) {
-        const fields = records.get(id);
-        const v = await embed(composeText(fields, rolesOf.get(fields.entityType)));
-        lines.push(JSON.stringify({ track_id: id, fields, v: packVec(v) }));
-        if (++n % 100 === 0) log(`${domain}: embedded ${n}/${fresh.length}`);
+    // 32 at a time: the embedder batches calls made together (embed.js), the GPU's speed.
+    for (let i = 0; i < fresh.length; i += 32) {
+        const ids = fresh.slice(i, i + 32);
+        const vs = await Promise.all(ids.map((id) => embed(composeText(records.get(id), rolesOf.get(records.get(id).entityType)))));
+        ids.forEach((id, j) => lines.push(JSON.stringify({ track_id: id, fields: records.get(id), v: packVec(vs[j]) })));
+        if (Math.floor((n + ids.length) / 1000) > Math.floor(n / 1000)) log(`${domain}: embedded ${n + ids.length}/${fresh.length}`);
+        n += ids.length;
     }
     if (!fresh.length && !gone.length && had.manifest && !rebake) {
         applySpec(dir, had.manifest, schema, had.rows, dead);   // a schema change alone still lands