@fangorn-network/westmarch 0.0.5-dev → 0.0.7-dev

package/plugin/scripts/fx.mjs

@@ -0,0 +1,268 @@
+#!/usr/bin/env node
+// The Fangorn app index, headless. One command per verb, JSON on stdout.
+//
+// WHY THERE IS NO MCP SERVER HERE
+// -------------------------------
+// An index that POINTS AT apps rather than hosting them needs no runtime of its
+// own. Every verb below is a function `@fangorn/westmarch` already exports, and
+// the agent's shell is the transport. Nobody has to stand up a server to be in
+// the index — a publisher bakes a view, and that is the whole obligation.
+//
+// WHY THIS IS A PLUGIN AND NOT A PAGE
+// -----------------------------------
+// One thing here is not stateless: the taste kernel. It lives in
+// ~/.fangorn/taste.json — on the filesystem, between the human and the agent —
+// because it has to outlive any one app, any one tab and any one ORIGIN.
+// localStorage cannot do that: a taste learned in fangorn.tv is invisible to the
+// next app by construction, which is the whole reason apps keep rebuilding it.
+// A file the agent can read is the shortest thing that carries it across.
+//
+// ponytail: a plaintext file, no wallet, no sync. `publish/vault.js` is the
+// upgrade path — same object, sealed to a signature, kept in the graph — and it
+// is worth doing the day the kernel needs to travel between machines, not now.
+// ponytail: every command re-fetches its view. Fine at 917 rows, ~seconds at
+// 42k; add a disk cache keyed by view URL when a big corpus makes it hurt.
+
+import { homedir } from "node:os";
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { configure, domainManifests, loadShard, trimView } from "../../consume/shard.js";
+import { findCorpora, sourcesFromChain } from "../../consume/directory.js";
+import { linkOf, rolesFrom, textOf, titleOf } from "../../consume/roles.js";
+import { exportTaste, recommend, taste } from "../../consume/taste.js";
+import { EMBED_MODEL, embedQuery, packVec, unpackVec } from "../../consume/embed.js";
+import { browse, describe, facet, getRow, neighbors, search } from "../../consume/tools.js";
+
+const HOME = process.env.FANGORN_HOME ?? join(homedir(), ".fangorn");
+const TASTE = join(HOME, "taste.json");
+
+// ── the kernel on disk ───────────────────────────────────────────────────────
+const store = () => { try { return JSON.parse(readFileSync(TASTE, "utf8")); } catch { return { model: EMBED_MODEL, likes: [], dislikes: [] }; } };
+const save = (s) => { mkdirSync(HOME, { recursive: true }); writeFileSync(TASTE, JSON.stringify(s, null, 1)); };
+/** Picks are stored PACKED (base64 int8, the shard encoding) and unpacked here.
+ *  Storing the picks rather than only the derived vector is what keeps the
+ *  kernel correctable: you can read what it was built from and drop one. */
+const picks = (l = []) => l.map((p) => ({ ...p, vector: unpackVec(p.v) })).filter((p) => p.vector);
+const kernel = (s = store()) => taste(picks(s.likes), picks(s.dislikes));
+
+// ── args ─────────────────────────────────────────────────────────────────────
+const argv = process.argv.slice(2);
+const [cmd, ...rest] = argv;
+const pos = rest.filter((a) => !a.startsWith("--"));
+const flag = (n, d) => { const a = rest.find((x) => x.startsWith(`--${n}=`)); return a === undefined ? d : a.slice(n.length + 3); };
+const num = (n, d) => { const v = flag(n); return v === undefined ? d : Number(v); };
+const pairs = (s) => s ? Object.fromEntries(String(s).split(",").map((p) => p.split("=").map((x) => x.trim()))) : undefined;
+const list = (s) => String(s ?? "").split(",").map((x) => x.trim()).filter(Boolean);
+const out = (o) => console.log(JSON.stringify(o, null, 1));
+const die = (m) => { console.error(m); process.exit(1); };
+
+// ── a view, loaded ───────────────────────────────────────────────────────────
+let roles = rolesFrom([], []);
+async function load(view) {
+    if (!view) die("need a view URL");
+    const base = trimView(view);
+    configure({
+        resolveView: () => base,
+        // Roles must be ready BEFORE the first row is parsed: rowText runs during
+        // parsing, so deriving them afterwards leaves every row with empty text
+        // and lexical search silently matching nothing.
+        onManifests: (ms) => { roles = rolesFrom(ms); },
+        rowText: (f) => textOf(f, roles),
+    });
+    const rows = await loadShard(base);
+    roles = rolesFrom(domainManifests(), rows);
+    let model = null;
+    try { model = (await (await fetch(`${base}/cdn/catalog`)).json()).embedding_model ?? null; } catch { /* unranked, not fatal */ }
+    return { rows, roles, base, model };
+}
+
+/** A query vector, or null with the reason. Never throws: every verb here
+ *  degrades to lexical and SAYS so rather than returning confident nonsense. */
+const qvOf = async (q) => { try { return { qv: await embedQuery(q), why: null }; } catch (e) { return { qv: null, why: e?.message ?? String(e) }; } };
+
+/** The corpus and the kernel must share an embedding space or cosine between
+ *  them is noise that still looks like a score. */
+const mismatch = (model, s) => model && s.model && model !== s.model
+    ? `taste was built with ${s.model}, this corpus is ${model} — scores would be noise` : null;
+
+const sources = async () => {
+    const f = flag("sources"); if (f) return list(f);
+    if (process.env.FANGORN_SOURCES) return list(process.env.FANGORN_SOURCES);
+    try { return JSON.parse(readFileSync(join(HOME, "sources.json"), "utf8")); } catch { /* none pinned */ }
+    return chainSources();
+};
+
+// Every app bound on chain, the scan fangorn-mcp's list-apps runs. From the
+// current AppRegistry's first event on Arbitrum Sepolia; --from-block overrides.
+async function chainSources() {
+    // The SDK scans 1,000 blocks per log request by default, which on the public
+    // RPC takes minutes; fangorn-mcp's install line sets the same window.
+    process.env.FANGORN_LOG_WINDOW ??= "100000";
+    const [{ Fangorn, FangornConfig }, { generatePrivateKey }] = await Promise.all([
+        import("@fangorn-network/sdk"), import("viem/accounts")]);
+    const fangorn = Fangorn.create({ privateKey: generatePrivateKey(), config: FangornConfig });   // reads only
+    const { sources } = await sourcesFromChain(fangorn, { fromBlock: BigInt(flag("from-block", "311637349")) });
+    return sources;
+}
+
+// ── verbs ────────────────────────────────────────────────────────────────────
+switch (cmd) {
+
+// WHICH app has this. Downloads no corpus — coverage centroids only, ~4KB each.
+case "find": {
+    const src = await sources();
+    if (!src.length) die("no apps found on chain; pass --sources=a,b or write ~/.fangorn/sources.json");
+    out(await findCorpora(pos.join(" "), { sources: src, embed: embedQuery, model: EMBED_MODEL, limit: num("limit", 10) }));
+    break;
+}
+
+case "describe": {
+    const { rows, roles: r, model } = await load(pos[0]);
+    out({
+        ...describe(rows), model,
+        shape: r.declared ? "declared" : "sniffed",
+        title: r.title, subtitle: r.subtitle, tags: r.tags, text: r.text,
+        entityTypes: r.entityTypes, gates: r.gates,
+        // Where the reader goes next. The index's job ends here.
+        launch: r.launch, externalUrl: r.externalUrl, actions: r.actions,
+    });
+    break;
+}
+
+case "browse": {
+    const { rows, roles: r } = await load(pos[0]);
+    out(browse(rows, r, { type: flag("type"), where: pairs(flag("where")), sort: flag("sort"),
+                          desc: flag("asc") === undefined, limit: num("limit", 20), offset: num("offset", 0), fields: list(flag("fields")) }));
+    break;
+}
+
+case "search": {
+    const { rows, roles: r } = await load(pos[0]);
+    const q = pos.slice(1).join(" ");
+    const { qv, why } = await qvOf(q);
+    out({ query: q, mode: qv ? "semantic where rows have vectors" : "lexical", ...(why ? { why } : {}),
+          hits: search(rows, q, r, { qv, limit: num("limit", 10), where: pairs(flag("where")), fields: list(flag("fields")) }) });
+    break;
+}
+
+case "facet": {
+    const { rows } = await load(pos[0]);
+    out(facet(rows, pos[1], { limit: num("limit", 20), where: pairs(flag("where")) }));
+    break;
+}
+
+case "row": {
+    const { rows, roles: r } = await load(pos[0]);
+    const g = getRow(rows, pos[1], r);
+    out(g ? { ...g, url: linkOf(rows.find((x) => x.id === g.id) ?? {}, r) } : null);
+    break;
+}
+
+case "similar": {
+    const { rows, roles: r } = await load(pos[0]);
+    out(neighbors(rows, pos[1], r, { limit: num("limit", 10), fields: list(flag("fields")) }));
+    break;
+}
+
+// ── the kernel ───────────────────────────────────────────────────────────────
+case "like": case "dislike": {
+    const { rows, roles: r, base, model } = await load(pos[0]);
+    const s = store();
+    const into = cmd === "like" ? s.likes : s.dislikes;
+    const added = [];
+    for (const id of pos.slice(1)) {
+        const row = rows.find((x) => x.id === id);
+        if (!row) { console.error(`no row ${id}`); continue; }
+        if (!row.vector) { console.error(`${id} has no vector — nothing to learn from`); continue; }
+        // Newest last: taste() weights by recency, so order in the file is signal.
+        const at = into.findIndex((p) => p.id === id);
+        if (at >= 0) into.splice(at, 1);
+        into.push({ id, title: titleOf(row, r), view: base, at: new Date().toISOString().slice(0, 10), v: packVec(Array.from(row.vector)) });
+        added.push(id);
+    }
+    s.model ??= model ?? EMBED_MODEL;
+    save(s);
+    const t = kernel(s);
+    out({ added, likes: s.likes.length, dislikes: s.dislikes.length, taste: t && exportTaste(t), file: TASTE });
+    break;
+}
+
+case "forget": {
+    const s = store();
+    const drop = new Set(pos);
+    const before = s.likes.length + s.dislikes.length;
+    s.likes = s.likes.filter((p) => !drop.has(p.id));
+    s.dislikes = s.dislikes.filter((p) => !drop.has(p.id));
+    save(s);
+    out({ removed: before - s.likes.length - s.dislikes.length, likes: s.likes.length, dislikes: s.dislikes.length });
+    break;
+}
+
+case "taste": {
+    const s = store();
+    const t = kernel(s);
+    out(t ? { model: s.model, n: t.n, from: t.from, rejected: t.rejected, drift: t.drift,
+              // Where each pick was learned. A kernel built in one publisher's
+              // corpus is meant to be pointed at another's — this is the record
+              // of whether it actually has been.
+              views: [...new Set([...s.likes, ...s.dislikes].map((p) => p.view))],
+              export: exportTaste(t), file: TASTE }
+            : { n: 0, file: TASTE, note: "nothing liked yet — `fx like <view> <id>`" });
+    break;
+}
+
+// The headline: rank a corpus by a taste learned somewhere else entirely.
+case "recommend": {
+    const { rows, roles: r, model } = await load(pos[0]);
+    const s = store();
+    const t = kernel(s);
+    if (!t) die("no taste yet — `fx like <view> <id>` first");
+    const warn = mismatch(model, s);
+    if (warn) die(warn);
+    const seen = new Set([...s.likes, ...s.dislikes].map((p) => p.id));
+    out({ from: t.from, against: pos[0],
+          hits: recommend(rows, t, { limit: num("limit", 10), exclude: seen })
+              .map(({ row, score }) => ({ id: row.id, title: titleOf(row, r), score, url: linkOf(row, r) })) });
+    break;
+}
+
+// ── the handoff. The index points; the app does the work. ────────────────────
+case "launch": {
+    const { rows, roles: r } = await load(pos[0]);
+    const id = flag("row");
+    if (id) {
+        const row = rows.find((x) => x.id === id);
+        if (!row) die(`no row ${id}`);
+        out({ url: linkOf(row, r), title: titleOf(row, r) });
+        break;
+    }
+    if (!r.launch) die("this corpus declares no presentation.launch — nowhere to hand off to");
+    const t = kernel();
+    const fill = { taste: t ? JSON.stringify(exportTaste(t)) : "", q: pos.slice(1).join(" ") };
+    const url = String(r.launch).replace(/\{(\w+)\}/g, (_, k) => encodeURIComponent(fill[k] ?? ""));
+    const u = new URL(url);
+    if (u.protocol !== "https:" && u.protocol !== "http:") die(`refusing a ${u.protocol} launch URL`);
+    // The kernel travels in the URL because that is the only channel a page on
+    // someone else's origin can receive it on. It is the app's copy from here.
+    out({ url: u.toString(), carries: t ? { n: t.n, from: t.from } : null });
+    break;
+}
+
+default:
+    console.log(`fx <verb>
+
+  find <query> [--sources=a,b|--from-block=N]       which app has this. No corpus downloaded.
+  describe <view>                                  rows, field coverage, roles, where it hands off
+  browse <view> [--type= --where=k=v --sort= --asc --limit= --offset=]
+  search <view> <query> [--where=k=v --limit=]     semantic where rows have vectors, else lexical
+  facet <view> <field> [--where=k=v --limit=]      count a field's values
+  row <view> <id>                                  one record, whole
+  similar <view> <id> [--limit=]                   nearest by vector
+
+  like|dislike <view> <id...>                      teach the kernel. Writes ${TASTE}
+  forget <id...>                                   drop picks from it
+  taste                                            what it is, and what it was built from
+  recommend <view> [--limit=]                      rank ANY corpus by it
+
+  launch <view> [query] | launch <view> --row=<id> the handoff URL, carrying the kernel`);
+}

package/publish/ship.js

@@ -3,6 +3,7 @@
 //
 //   westmarch-ship app.json [--no-crawl] [--no-deploy] [--dry-run]
 //   westmarch-ship app.json --local <out> [--crawl]     a view from staged records, no chain (see buildLocal)
+//   westmarch-ship app.json --register                  the app alone: claim, join, empty site, card bound
 //
 // Every step checks the chain (or Cloudflare) first and only acts on a difference, so
 // re-running is how you update: new data, a changed schema, a new source, a moved site.
@@ -16,6 +17,11 @@
 //   7. deploy to Cloudflare Pages
 //   8. register the ERC-8004 agent and bind the card, if the chain points elsewhere
 //
+// --register is the fast path, run before there is any data: 1, 2, 6 with an empty view, 7,
+// and 8 binding the card without minting. The mint copies the card's skills into its IPFS
+// registration file and is never redone, so it waits for the first full ship, when the card
+// lists what the app really does.
+//
 // Chain writes go through the `fangorn` CLI (its configured wallet signs, and
 // `fangorn wallet` shows which). Reads go through the SDK. State lives in `.ship/` next
 // to app.json: the site, the view (its own state, see view.js) and a small state.json.
@@ -84,12 +90,12 @@ export function expandSources(sources, base) {
 export function loadConfig(path) {
     const c = JSON.parse(readFileSync(path, "utf8"));
     c.sources = expandSources(c.sources ?? [], dirname(resolve(path)));
-    for (const k of ["app", "name", "description", "types", "sources", "site"])
+    for (const k of ["app", "name", "description", "site"])   // types and sources can come after --register
         if (!c[k]) throw new Error(`${path}: missing "${k}"`);
     if (!c.site.project) throw new Error(`${path}: site.project names the Cloudflare Pages project`);
     for (const s of c.sources) if (!s.namespace || !Array.isArray(s.command))
         throw new Error(`${path}: each source needs a namespace and a command (an argv array)`);
-    return { relations: [], tags: [], ...c };
+    return { relations: [], tags: [], types: {}, ...c };
 }
 
 /** The schema commit for `types` + `relations`: one fangorn.type.v1 vertex per type,
@@ -155,8 +161,10 @@ function assertSameDeployment(cli) {
 
 // ── the steps ─────────────────────────────────────────────────────────────────
 
-export async function ship(configPath, { crawl = true, deploy = true, dry = false, replace = false, only = null } = {}) {
+export async function ship(configPath, { crawl = true, deploy = true, dry = false, replace = false, only = null, register = false } = {}) {
     const cfg = loadConfig(configPath);
+    if (!register && (!Object.keys(cfg.types).length || !cfg.sources.length))
+        throw new Error(`${configPath}: no types or sources yet, so nothing to build; --register sets up the app alone`);
     CLI = cliArgv(cfg);
     assertSameDeployment(CLI);
     const base = dirname(resolve(configPath));
@@ -201,22 +209,24 @@ export async function ship(configPath, { crawl = true, deploy = true, dry = fals
 
     // 3. schema
     let onChain = null;
-    try { onChain = view.parseSchema((await fangorn.readNamespace(owner, view.SCHEMA_NAMESPACE)).contents); } catch { /* none yet */ }
-    if (!onChain || canonical(onChain.types, onChain.relations) !== canonical(cfg.types, cfg.relations)) {
-        const sdir = join(dir, "schema");
-        mkdirSync(sdir, { recursive: true });
-        writeFileSync(join(sdir, "graph.json"), JSON.stringify(schemaGraph(cfg.types, cfg.relations), null, 1));
-        log(`schema ${onChain ? "changed" : "is new"}: committing ${Object.keys(cfg.types).join(", ")}`);
-        fangornCli(cfg.app, ["repo", "init", view.SCHEMA_NAMESPACE], { cwd: sdir, dry });
-        fangornCli(cfg.app, ["commit", "graph.json", "-m", "schema", "--replace"], { cwd: sdir, dry });
-        fangornCli(cfg.app, ["push"], { cwd: sdir, dry });
-    } else log("schema unchanged");
+    if (!register) {
+        try { onChain = view.parseSchema((await fangorn.readNamespace(owner, view.SCHEMA_NAMESPACE)).contents); } catch { /* none yet */ }
+        if (!onChain || canonical(onChain.types, onChain.relations) !== canonical(cfg.types, cfg.relations)) {
+            const sdir = join(dir, "schema");
+            mkdirSync(sdir, { recursive: true });
+            writeFileSync(join(sdir, "graph.json"), JSON.stringify(schemaGraph(cfg.types, cfg.relations), null, 1));
+            log(`schema ${onChain ? "changed" : "is new"}: committing ${Object.keys(cfg.types).join(", ")}`);
+            fangornCli(cfg.app, ["repo", "init", view.SCHEMA_NAMESPACE], { cwd: sdir, dry });
+            fangornCli(cfg.app, ["commit", "graph.json", "-m", "schema", "--replace"], { cwd: sdir, dry });
+            fangornCli(cfg.app, ["push"], { cwd: sdir, dry });
+        } else log("schema unchanged");
+    }
 
     // 4. sources: crawl + publish straight into the app. One source failing (a site that
     // blocks us, a feed that is down) keeps its last published data and does not stop the
     // others; the ship still exits non-zero at the end, naming it.
     const failed = [];
-    if (crawl) for (const s of cfg.sources.filter((s) => !only || only.includes(s.namespace))) {
+    if (crawl && !register) for (const s of cfg.sources.filter((s) => !only || only.includes(s.namespace))) {
         const cwd = s.cwd ? resolve(base, s.cwd.replace(/^~/, homedir())) : base;
         const paidArgs = cfg.paid && s.paid_dir ? ["--paid-dir", resolve(cwd, s.paid_dir)] : [];
         try {
@@ -234,8 +244,9 @@ export async function ship(configPath, { crawl = true, deploy = true, dry = fals
     const site = join(dir, "site");
     if (!existsSync(join(site, "view/cdn/catalog")) && state.url && !dry) await mirrorView(`${state.url}/view`, join(site, "view"));
     const namespaces = [...new Set(cfg.sources.map((s) => s.namespace))];
-    const report = dry ? {} : await view.publishView({ fangorn, app: cfg.app, namespaces, out: join(site, "view"),
-        fromBlock: BigInt(state.fromBlock), log });
+    if (register && !dry) view.writeCatalog(join(site, "view"));   // empty, or what an earlier build left
+    const report = dry || register ? {} : await view.publishView({ fangorn, app: cfg.app, namespaces, out: join(site, "view"),
+        fromBlock: BigInt(state.fromBlock), embed: view.cachedEmbed(join(dir, "vectors.ndjson")), log });
 
     // 6. the page, the card, the headers
     if (!existsSync(join(STOCK, "index.html"))) throw new Error(`no stock page at ${STOCK}; run \`npx vite build\` in westmarch/site`);
@@ -297,8 +308,12 @@ export async function ship(configPath, { crawl = true, deploy = true, dry = fals
         "/.well-known/agent-card.json\n  Access-Control-Allow-Origin: *\n"
         + "/view/*\n  Access-Control-Allow-Origin: *\n"
         + `/assets/*\n${immutable}`
-        // An hour, not immutable: a shard is deleted when a record is retracted (see view.js).
-        + domains.map((d) => `/view/cdn/domains/${d}/shards/*\n  Cache-Control: public, max-age=3600\n`).join(""));
+        // A shard's name is its content hash, so a browser may keep it for good: a change or a
+        // retraction writes new names and the manifest stops pointing at the old ones. `private`
+        // keeps it out of shared caches, so a retracted record's bytes stop being served the
+        // moment the deploy drops the file (see view.js). An hour made readers on a phone fetch
+        // the whole catalog again after every hour away.
+        + domains.map((d) => `/view/cdn/domains/${d}/shards/*\n  Cache-Control: private, max-age=31536000, immutable\n`).join(""));
 
     // 7. deploy
     if (deploy) {
@@ -311,13 +326,29 @@ export async function ship(configPath, { crawl = true, deploy = true, dry = fals
     // 8. ERC-8004 + the binding
     const cardUrl = `${url}/.well-known/agent-card.json`;
     const bound = await registry.appAgentUri(appId).catch(() => "");
-    if (deploy && bound !== cardUrl) {
-        log(`binding ${cardUrl} (on chain: ${bound || "nothing"})`);
+    const mint = () => {
         const out = dry ? "" : fangornCli(cfg.app, ["app", "agent", cardUrl]);
         state.agentId = (out.match(/Agent ID:\s+(\S+)/) ?? [])[1] ?? state.agentId;
+        delete state.agentPending;
         save();
+    };
+    if (deploy && bound !== cardUrl && register) {
+        log(`binding ${cardUrl} (on chain: ${bound || "nothing"}); the agent is minted on the first full ship`);
+        fangornCli(cfg.app, ["app", "agent", cardUrl, "--skip-register"], { dry });
+        state.agentPending = true;
+        save();
+    } else if (deploy && bound !== cardUrl) {
+        log(`binding ${cardUrl} (on chain: ${bound || "nothing"})`);
+        mint();
+    } else if (deploy && state.agentPending && !register) {
+        log(`minting the ERC-8004 agent from the card as built`);
+        mint();
     } else if (deploy) log("card already bound");
 
+    if (register) {
+        log(`registered: ${url}  ·  card ${cardUrl}  ·  ${cfg.app} is yours; build it with westmarch-ship ${configPath} --local <out>`);
+        return { url, cardUrl, report };
+    }
     const added = Object.values(report).reduce((n, r) => n + r.added, 0);
     log(`done: ${url}  ·  card ${cardUrl}  ·  agent ${state.agentId ?? "(unchanged)"}  ·  +${added} records embedded`);
     if (failed.length) throw new Error(`deployed, but these sources failed and kept their old records: ${failed.join(", ")}`);
@@ -450,6 +481,7 @@ if (process.argv[1]?.endsWith("ship.js") || process.argv[1]?.endsWith("westmarch
     const [path, ...flags] = process.argv.slice(2);
     if (!path || flags.includes("--help")) {
         console.error("usage: westmarch-ship app.json [--no-crawl] [--no-deploy] [--dry-run] [--replace] [--only ns1,ns2]\n" +
+                      "       westmarch-ship app.json --register [--dry-run]   (claim, join, empty site, card bound: no data)\n" +
                       "       westmarch-ship app.json --local <out> [--crawl] [--only ns1,ns2]   (no chain, no deploy)");
         process.exit(path ? 0 : 2);
     }
@@ -460,7 +492,8 @@ if (process.argv[1]?.endsWith("ship.js") || process.argv[1]?.endsWith("westmarch
         process.exit(0);
     }
     await ship(path, { crawl: !flags.includes("--no-crawl"), deploy: !flags.includes("--no-deploy"), dry: flags.includes("--dry-run"),
-                       replace: flags.includes("--replace"),
+                       replace: flags.includes("--replace"), register: flags.includes("--register"),
                        only })
         .catch((e) => { console.error(`[ship] ✗ ${e.message}`); process.exit(1); });
+    process.exit(0);   // a CUDA embedder keeps the process alive (and can abort in its teardown)
 }

package/publish/view.js

@@ -201,7 +201,7 @@ export function liveFields(out) {
  * Returns per-domain counts of what changed.
  */
 export async function publishView({ fangorn, app, namespaces, out, fromBlock = 0n, rebake = false,
-                                    embed = embedDocumentDirect, log = console.log }) {
+                                    embed = embedDocumentDirect, log = console.log, shardRows = 20000 }) {
     if (!namespaces?.length) throw new Error("publishView: name the data namespace(s) to publish");
     fangorn.setAppId(app);
     const appId = fangorn.getAppId();
@@ -222,7 +222,7 @@ export async function publishView({ fangorn, app, namespaces, out, fromBlock = 0
         for (const { owner: publisher } of timelines) {
             const domain = domainFor(appId, publisher, ns);
             report[domain] = await publishDomain({
-                fangorn, schema, publisher, ns, domain, dir: `${out}/cdn/domains/${domain}`, rebake, embed, log,
+                fangorn, schema, publisher, ns, domain, dir: `${out}/cdn/domains/${domain}`, rebake, embed, log, shardRows,
             });
         }
     }
@@ -230,7 +230,7 @@ export async function publishView({ fangorn, app, namespaces, out, fromBlock = 0
     return report;
 }
 
-async function publishDomain({ fangorn, schema, publisher, ns, domain, dir, rebake, embed, log }) {
+async function publishDomain({ fangorn, schema, publisher, ns, domain, dir, rebake, embed, log, shardRows }) {
     const { contents } = await fangorn.readNamespace(publisher, ns);
     const records = new Map((contents.vertices ?? []).filter((v) => v.payload && typeof v.payload === "object")
         .map((v) => [v.cid, { ...v.payload, entityType: v.schemaId }]));
@@ -238,7 +238,9 @@ async function publishDomain({ fangorn, schema, publisher, ns, domain, dir, reba
     if (rebake) rmSync(dir, { recursive: true, force: true });   // old shards would stay reachable
     const had = loadDomain(dir);
     const made = had.manifest?.embedder ?? (had.manifest ? { runtime: "fastembed", dtype: "fp32" } : null);
-    if (made && JSON.stringify(made) !== JSON.stringify(EMBEDDER))
+    // Same model and dtype is the same encoder, whatever ran it: q8 on CPU, CUDA and WASM agree to
+    // ~0.98-0.99 cosine, fp32 against q8 only to ~0.95.
+    if (made && (made.model ?? EMBED_MODEL) + made.dtype !== EMBEDDER.model + EMBEDDER.dtype)
         throw new Error(`${domain} was embedded by ${made.runtime} ${made.dtype}, not ${EMBEDDER.runtime} ${EMBEDDER.dtype}; ` +
             "vectors from two encoders are not comparable. Rebake it with --rebake, or keep publishing it with the tool that made it.");
 
@@ -275,12 +277,17 @@ async function publishDomain({ fangorn, schema, publisher, ns, domain, dir, reba
     mkdirSync(`${dir}/shards`, { recursive: true });
     const manifest = had.manifest ?? { name: domain, description: "", count: 0, dim: EMBED_DIM, model: EMBED_MODEL,
         distance: "Cosine", embedder: EMBEDDER, filter: { owner: [publisher], namespace: [ns] }, shards: [], tombstones: [] };
-    const writeShard = (ls) => {
-        const gz = gzipSync(`${ls.join("\n")}\n`, { level: 9 });
-        const sha256 = createHash("sha256").update(gz).digest("hex");
-        const file = `shard-${String(manifest.shards.length).padStart(4, "0")}-${sha256.slice(0, 12)}.ndjson.gz`;
-        writeFileSync(`${dir}/shards/${file}`, gz);
-        manifest.shards.push({ file, count: ls.length, bytes: gz.length, sha256 });
+    // Pages serves files up to 25 MiB; 52k Steam games gzipped to 28 MiB in one shard.
+    // ponytail: split by row count (~550 B a row gzipped); split by bytes if rows get much fatter.
+    const writeShard = (all) => {
+        for (let i = 0; i < all.length; i += shardRows) {
+            const ls = all.slice(i, i + shardRows);
+            const gz = gzipSync(`${ls.join("\n")}\n`, { level: 9 });
+            const sha256 = createHash("sha256").update(gz).digest("hex");
+            const file = `shard-${String(manifest.shards.length).padStart(4, "0")}-${sha256.slice(0, 12)}.ndjson.gz`;
+            writeFileSync(`${dir}/shards/${file}`, gz);
+            manifest.shards.push({ file, count: ls.length, bytes: gz.length, sha256 });
+        }
     };
     if (lines.length) {
         writeShard(lines);
@@ -332,7 +339,7 @@ function applySpec(dir, manifest, schema, rows, dead, guessed = new Map()) {
     }
 }
 
-function writeCatalog(out) {
+export function writeCatalog(out) {
     const root = `${out}/cdn/domains`;
     mkdirSync(`${out}/cdn`, { recursive: true });
     const domains = existsSync(root) ? readdirSync(root).sort().map((d) => readJson(`${root}/${d}/manifest`, null)).filter(Boolean) : [];
@@ -389,6 +396,11 @@ if (typeof process !== "undefined" && import.meta.url === `file://${process.argv
     const run = () => publishView({ fangorn, app: APP, namespaces: ["shop"], out, embed, log: () => {} });
 
     const d = domainFor(APP, PUB, "shop");
+    const split = mkdtempSync(`${tmpdir()}/view-`);
+    await publishView({ fangorn, app: APP, namespaces: ["shop"], out: split, embed, log: () => {}, shardRows: 2 });
+    m0: { const m = readJson(`${split}/cdn/domains/${d}/manifest`);
+          eq([m.count, m.shards.map((s) => s.count)], [3, [2, 1]], "rows past shardRows go to the next shard (Pages caps a file at 25 MiB)"); }
+    embedded.length = 0;
     eq((await run())[d], { added: 3, removed: 0 }, "first run embeds everything");
     if (!embedded.includes("Title: Oak table. Tags: . a solid oak dining table")) throw new Error(`declared roles compose the text: ${embedded}`);
     let m = readJson(`${out}/cdn/domains/${d}/manifest`);
@@ -424,6 +436,8 @@ if (typeof process !== "undefined" && import.meta.url === `file://${process.argv
     const { rolesFrom } = await import("../consume/roles.js");
     eq(getRow(got, "b2", rolesFrom([m]))?.id, "c2b", "get by the publisher's own id, not the CID");
 
+    m.embedder = { ...EMBEDDER, runtime: "transformers.js cuda" }; writeJson(`${out}/cdn/domains/${d}/manifest`, m);
+    await run();   // throws if another runtime of the same q8 model is refused
     m.embedder = { runtime: "fastembed", dtype: "fp32" }; writeJson(`${out}/cdn/domains/${d}/manifest`, m);
     await throws(run, /not comparable/, "appending to another encoder's view");
     eq((await publishView({ fangorn, app: APP, namespaces: ["shop"], out, embed, rebake: true, log: () => {} }))[d],

package/site/agent.js

@@ -1,6 +1,7 @@
 // The stock page's tools for agents in a browser: the same verbs as fangorn-mcp's data
 // tools, over the rows this page loaded. `ctx` holds them; ship's card captures these.
 import { describe, facet, getRow, search } from "../consume/tools.js";
+import { KNOBS, discover } from "../consume/taste.js";
 
 const ok = (o) => ({ content: [{ type: "text", text: JSON.stringify(o) }] });
 const obj = (properties, required = []) => ({ type: "object", properties, required });
@@ -41,4 +42,26 @@ export function registerAgent(ctx) {
         inputSchema: obj({}),
         execute: async () => ok(ctx.session?.() ?? { saved: [], searches: [], taste: null }),
     });
+    // Like / dislike and "For you": the same calls the page's 👍 👎 and knobs make (ctx.rate,
+    // ctx.votes, set by main.js), so what an agent records shows on the page.
+    mc.registerTool({
+        name: "rate",
+        description: "Record that the person liked or disliked a record (or clear it), by the id search returns. Builds the taste discover ranks with.",
+        inputSchema: obj({ id: { type: "string" }, verdict: { type: "string", enum: ["like", "dislike", "clear"] } }, ["id", "verdict"]),
+        execute: async ({ id, verdict }) => ok(ctx.rate?.(id, verdict) ?? { error: "this page keeps no taste" }),
+    });
+    mc.registerTool({
+        name: "discover",
+        description: "Records for a taste: a kernel over what was liked (newest last) and disliked, not a text search. Pass likes and dislikes (ids from search), or omit them to use the taste on this page. Knobs are 0–1.",
+        inputSchema: obj({ likes: { type: "array", items: { type: "string" } }, dislikes: { type: "array", items: { type: "string" } },
+            ...Object.fromEntries(Object.entries(KNOBS).map(([k, [d, what]]) => [k, { type: "number", minimum: 0, maximum: 1, default: d, description: what }])),
+            seed: { type: "integer", description: "Same seed, same shuffle" }, limit: { type: "integer", minimum: 1, maximum: 50, default: 10 } }),
+        execute: async ({ likes, dislikes, limit = 10, ...knobs }) => {
+            const byId = (ids) => ids.map((id) => ctx.rows.find((r) => r.id === getRow(ctx.rows, id, ctx.roles)?.id)).filter((r) => r?.vector);
+            const v = likes ? { likes: byId(likes), dislikes: byId(dislikes ?? []) } : ctx.votes?.() ?? { likes: [], dislikes: [] };
+            if (!v.likes.length) return ok({ error: "like at least one record first (ids from search, or rate)" });
+            const out = discover(ctx.rows, v.likes, v.dislikes, { ...knobs, limit: Math.min(50, limit) });
+            return ok({ ...out, picks: out.picks.map(({ row, score }) => ({ ...getRow(ctx.rows, row.id, ctx.roles), score })) });
+        },
+    });
 }

package/site/dist/assets/embed.worker-Dn3fKctb.js

@@ -0,0 +1 @@
+let e=null;const t=()=>e??=(async()=>{let{pipeline:e}=await import(`./transformers.web-BgUm7F_-.js`),t={}.WESTMARCH_EMBED_DEVICE;return e(`feature-extraction`,`nomic-ai/nomic-embed-text-v1.5`,{dtype:`q8`,...t&&{device:t}})})();function n(e,t=256){let n=e.length,r=0;for(let t=0;t<n;t++)r+=e[t];r/=n;let i=0;for(let t=0;t<n;t++)i+=(e[t]-r)**2;let a=Math.sqrt(i/n+1e-5),o=Array(Math.min(t,n));for(let t=0;t<o.length;t++)o[t]=(e[t]-r)/a;let s=0;for(let e of o)s+=e*e;if(s=Math.sqrt(s),s)for(let e=0;e<o.length;e++)o[e]/=s;return o}async function r(e){return n((await(await t())(`search_query: ${e}`,{pooling:`mean`,normalize:!1})).data)}async function i(e){return n((await(await t())(`search_document: ${e}`,{pooling:`mean`,normalize:!1})).data)}const a=()=>t().then(()=>{});function o(e){let t=new Uint8Array(e.length);for(let n=0;n<e.length;n++){let r=Math.round(Math.max(-1,Math.min(1,e[n]))*127);t[n]=r<0?r+256:r}let n=``;for(let e of t)n+=String.fromCharCode(e);return btoa(n)}function s(e){try{let t=atob(e),n=new Float32Array(t.length);for(let e=0;e<t.length;e++){let r=t.charCodeAt(e);n[e]=(r>127?r-256:r)/127}return n.length?n:null}catch{return null}}function c(e){let t=new Uint8Array(Math.ceil(e.length/8));for(let n=0;n<e.length;n++)e[n]>=0&&(t[n>>3]|=128>>(n&7));let n=``;for(let e of t)n+=String.fromCharCode(e);return btoa(n)}function l(e){try{let t=atob(e.padEnd(Math.ceil(e.length/4)*4,`=`)),n=new Float32Array(t.length*8);for(let e=0;e<n.length;e++)n[e]=t.charCodeAt(e>>3)>>7-(e&7)&1?1:-1;return n.length?n:null}catch{return null}}if(typeof process<`u`&&import.meta.url===`file://${process.argv[1]}`&&process.argv[1].endsWith(`/embed.js`)){let e=n(Array.from({length:768},(e,t)=>Math.sin(t))),t=0;for(let n of e)t+=n*n;if(Math.abs(Math.sqrt(t)-1)>1e-6)throw Error(`matryoshka must L2-normalize`);if(e.length!==256)throw Error(`matryoshka must truncate to 256`);let r=s(o(e));if(r.length!==e.length)throw Error(`pack/unpack changed the dimension`);let i=0,a=0;for(let t=0;t<e.length;t++)i+=e[t]*r[t],a+=r[t]*r[t];let u=i/Math.sqrt(a);if(u<.999)throw Error(`int8 round-trip cosine ${u} — too lossy to rank with`);let d=s(o([-1,-.5,0,.5,1]));if(d[0]>-.99||d[1]>-.49||d[1]<-.51)throw Error(`negative components corrupted: ${[...d]}`);if(s(`!!!not base64!!!`)!==null)throw Error(`malformed vector must return null, not throw`);let f=c(e);if(f.length>48)throw Error(`sign bits must stay link-sized, got ${f.length} chars`);let p=l(f);if(p.length!==e.length)throw Error(`sign round-trip changed the dimension`);for(let t=0;t<e.length;t++)if(Math.sign(p[t])!==(e[t]<0?-1:1))throw Error(`sign flipped at ${t}`);if(c(p)!==f)throw Error(`sign packing must be idempotent`);if(l(`!!!not base64!!!`)!==null)throw Error(`malformed bits must return null, not throw`);console.log(`embed.js self-check ok — matryoshka norm, int8 round-trip, negatives, malformed input, sign bits`)}const u={query:r,document:i};self.onmessage=async({data:{id:e,kind:t,text:n}})=>{try{if(t===`warm`)return await a(),self.postMessage({id:e,vec:null});self.postMessage({id:e,vec:await u[t](n)})}catch(t){self.postMessage({id:e,error:t?.message??String(t)})}};