@fangorn-network/westmarch 2026.10.27 → 2026.10.29

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fangorn-network/westmarch",
-  "version": "2026.10.27",
+  "version": "2026.10.29",
   "description": "all-in-one builder for fangorn apps",
   "license": "MIT",
   "type": "module",
@@ -45,7 +45,7 @@
     "@fangorn-network/markov": "^0.1.0"
   },
   "peerDependencies": {
-    "@fangorn-network/sdk": "^2026.10.26-dev",
+    "@fangorn-network/sdk": "^2026.10.28",
     "@huggingface/transformers": "^4.2.0",
     "@noble/curves": "^1.9.1",
     "viem": "^2.55.1"
@@ -56,7 +56,7 @@
     }
   },
   "devDependencies": {
-    "@fangorn-network/sdk": "^2026.10.26-dev",
+    "@fangorn-network/sdk": "^2026.10.28",
     "@huggingface/transformers": "^4.2.0",
     "@noble/curves": "^1.9.1",
     "@privy-io/react-auth": "^3.47.0",

package/site/worker.js

@@ -35,7 +35,7 @@ import { createPublicClient, http } from "viem";
 import { generatePrivateKey } from "viem/accounts";
 import { EMBED_MODEL, DOCUMENT, QUERY, matryoshka } from "../src/core/embed.js";
 import { configure, resetShard } from "../src/core/shard.js";
-import { publishView, r2Store } from "../src/publish/view.js";
+import { ORIGIN, publishView, r2Store } from "../src/publish/view.js";
 import { viewTools } from "../src/agent/view-tools.js";
 
 let CONFIG = { paid: null };
@@ -57,8 +57,18 @@ const encode = (env, prompt) => async (texts) =>
 // ── the indexer ───────────────────────────────────────────────────────────────
 
 const STATE = "state.json", LOCK = "index.lock";
-// A run past this is presumed dead and its lock taken over. Cron runs get 15 minutes.
-const LOCK_MS = 14 * 60_000;
+// A live run beats its lock every BEAT_MS; one silent past LOCK_MS is presumed dead (killed for CPU
+// or wall time, which skips `finally`) and its lock taken over. Lock: "<last beat ms> <run id>".
+// ponytail: get-then-put, so two runs starting in the same instant can both take it; R2's onlyIf etag when that shows.
+const LOCK_MS = 2 * 60_000, BEAT_MS = 30_000;
+export async function takeLock(bucket, now = Date.now()) {
+    const held = await (await bucket.get(LOCK))?.text();
+    if (held && now - Number(held.split(" ")[0]) < LOCK_MS) return null;
+    const id = crypto.randomUUID();
+    const beat = () => bucket.put(LOCK, `${Date.now()} ${id}`);
+    await beat();
+    return { id, beat, release: async () => { if ((await (await bucket.get(LOCK))?.text())?.endsWith(id)) await bucket.delete(LOCK); } };
+}
 
 // CONFIG.network: the SDK's network name ("baseSepolia"); absent, its default.
 const network = () => SupportedNetworks[CONFIG.network] ?? FangornConfig;
@@ -96,9 +106,9 @@ export async function presented({ fangorn, chain }, state) {
 }
 
 export async function index(env, { log = console.log } = {}) {
-    const lock = await env.VIEW.get(LOCK);
-    if (lock && Date.now() - Number(await lock.text()) < LOCK_MS) return log("index: a run is in progress; skipped");
-    await env.VIEW.put(LOCK, String(Date.now()));
+    const lock = await takeLock(env.VIEW);
+    if (!lock) return log("index: a run is in progress; skipped");
+    const beating = setInterval(() => lock.beat().catch(() => {}), BEAT_MS);
     try {
         const state = (await (await env.VIEW.get(STATE))?.json()) ?? { block: null, repos: {} };
         const r = reader(env);
@@ -113,7 +123,8 @@ export async function index(env, { log = console.log } = {}) {
         log(`index: through block ${state.block}, ${Object.keys(report).length} domain(s) changed`);
         return report;
     } finally {
-        await env.VIEW.delete(LOCK);
+        clearInterval(beating);
+        await lock.release();
     }
 }
 
@@ -455,7 +466,6 @@ async function paidCall(origin, tool, payment, run) {
     return { ...result, _meta: { ...result._meta, "x402/payment-response": settled } };
 }
 
-export const ORIGIN = "https://origin.westmarch.invalid";
 const BY_ORIGIN = new Set(["/.well-known/agent-card.json", "/openapi.json", "/.well-known/x402"]);
 
 const worker = {
@@ -517,6 +527,15 @@ if (typeof process !== "undefined" && import.meta.url === `file://${process.argv
                      delimitedPrefixes: [...new Set(ks.filter((k) => rest(k).includes("/")).map((k) => `${prefix + rest(k).split("/")[0]}/`))] };
         },
     };
+    const a = await takeLock(VIEW);
+    eq([!!a, await takeLock(VIEW)], [true, null], "a held lock skips the next run");
+    objs.set(LOCK, { bytes: new TextEncoder().encode(`${Date.now() - LOCK_MS - 1} dead`), at: new Date() });
+    const b = await takeLock(VIEW);
+    await a.release();
+    eq([!!b, objs.has(LOCK)], [true, true], "a silent lock is taken over, and its dead owner can't release the new one");
+    await b.release();
+    eq(objs.has(LOCK), false, "the owner releases its own lock");
+
     // Words → a 768-d vector, so a query shares dimensions with the rows that share its words.
     const calls = [];
     const AI = { run: async (model, { text }) => { calls.push({ model, text });

package/src/agent/agent-card.js

@@ -17,6 +17,7 @@
 // point back at it, so writing someone else's appId here gets you nothing.
 
 import { FANGORN_APP_EXTENSION, BaseSepoliaConfig, appId as hashAppId } from "@fangorn-network/sdk";
+import { PLUGIN_MARKETPLACE, PLUGIN_NAME } from "../discover/apps.js";
 
 /** Run `register` against a recording `document.modelContext`; return what it
  *  registered. `register` is whatever the page calls at load. */
@@ -79,7 +80,7 @@ export function agentCard({ name, description, url, version = "0.0.0", tools = [
 
 /** The params `discoverApp` reads, validated the same way it validates them.
  *  A malformed card should fail the build, not the first reader. */
-export function fangornExtension({ app, fromBlock, repos = [], views = [], mcp, paid, access, skill, config = BaseSepoliaConfig }) {
+export function fangornExtension({ app, fromBlock, repos = [], views = [], mcp, paid, access, skill, plugin, config = BaseSepoliaConfig }) {
     if (!app) throw new Error("fangorn.app is required");
     const id = HEX32.test(app) ? app : hashAppId(app);
     const from = String(fromBlock ?? "");
@@ -102,6 +103,8 @@ export function fangornExtension({ app, fromBlock, repos = [], views = [], mcp,
     // `days`) or one x402 payment of `call` per tool call; describe and the catalog are free.
     if (skill && !(/^https?:\/\//.test(skill.url ?? "") && /^[0-9a-f]{64}$/.test(skill.sha256 ?? "")))
         throw new Error(`fangorn.skill needs an http(s) url and the sha256 of the file, in hex: ${JSON.stringify(skill)}`);
+    if (plugin && !(PLUGIN_MARKETPLACE.test(plugin.marketplace ?? "") && PLUGIN_NAME.test(plugin.name ?? "")))
+        throw new Error(`fangorn.plugin needs a marketplace (GitHub owner/repo or https URL) and a plugin name: ${JSON.stringify(plugin)}`);
     if (access && !(/^https?:\/\//.test(access.pass_url ?? "") && /^\d+$/.test(String(access.pass)) && /^\d+$/.test(String(access.call))))
         throw new Error(`fangorn.access needs pass_url, and pass and call in base units: ${JSON.stringify(access)}`);
     return {
@@ -117,6 +120,7 @@ export function fangornExtension({ app, fromBlock, repos = [], views = [], mcp,
                                  symbol: paid.symbol ?? "USDC", decimals: paid.decimals ?? 6, description: paid.description ?? "" } } : {}),
             ...(access ? { access } : {}),
             ...(skill ? { skill: { url: skill.url, sha256: skill.sha256 } } : {}),
+            ...(plugin ? { plugin: { marketplace: plugin.marketplace, name: plugin.name } } : {}),
         },
     };
 }
@@ -159,6 +163,8 @@ if (typeof process !== "undefined" && import.meta.url === `file://${process.argv
     eq(fangornExtension({ app: "a", fromBlock: 0, skill: { url: "https://x.test/skill/a/SKILL.md", sha256: sha } }).params.skill, { url: "https://x.test/skill/a/SKILL.md", sha256: sha }, "a skill rides in the extension");
     throws(() => fangornExtension({ app: "a", fromBlock: 0, skill: { url: "https://x.test/s.md" } }), /skill/, "a skill without its hash");
     throws(() => fangornExtension({ app: "a", fromBlock: 0, skill: { url: "file:///etc/passwd", sha256: sha } }), /skill/, "a non-http skill");
+    eq(fangornExtension({ app: "a", fromBlock: 0, plugin: { marketplace: "o/r", name: "a" } }).params.plugin, { marketplace: "o/r", name: "a" }, "a plugin rides in the extension");
+    throws(() => fangornExtension({ app: "a", fromBlock: 0, plugin: { marketplace: "o/r; rm -rf ~", name: "a" } }), /plugin/, "a plugin that would inject into a shell");
 
     console.log("agent-card.js self-check ok — tools captured without executing, extension matches the SDK's, bad params fail the build");
 }

package/src/agent/mcp.js

@@ -24,7 +24,7 @@ import net from "node:net";
 import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { homedir } from "node:os";
 import { createInterface } from "node:readline";
-import { APP_EXTENSION, listApps, skillOf, toApp } from "../discover/apps.js";
+import { APP_EXTENSION, listApps, pluginOf, skillOf, toApp } from "../discover/apps.js";
 
 const log = (...a) => console.error("[fangorn-mcp]", ...a);   // stdout is the protocol
 // FANGORN_MCP_TRACE=1: step timings on stderr, for finding where an answer waits.
@@ -216,7 +216,7 @@ export function handlers(deps) {
         while (open.has(s)) s += "-";
         const ext = card.capabilities?.extensions?.find((e) => e.uri === APP_EXTENSION);
         const access = ext?.params?.access ?? null;
-        const o = { slug: s, name: card.name, card: cardUrl, appId, skill: skillOf(ext), ...await deps.remoteTools(mcp, { access }) };
+        const o = { slug: s, name: card.name, card: cardUrl, appId, skill: skillOf(ext), plugin: pluginOf(ext), ...await deps.remoteTools(mcp, { access }) };
         trace(`open ${app}: ready`);
         open.set(o.slug, o);
         deps.notify?.("notifications/tools/list_changed");
@@ -253,7 +253,10 @@ export function handlers(deps) {
                           call_as: `${o.slug}__<tool>, or call-app-tool`, tools: o.tools,
                           // An app's own skill: standing instructions for the reader's agent, so never loaded
                           // silently. The fangorn-index skill offers it to the user and pins the hash on a yes.
-                          ...(o.skill ? { skill: { ...o.skill, adopt: "The app ships an agent skill. Offer it to the user (what it would make you do, from its description); adopt it only on their yes, checking sha256, and ask again if the hash changes." } } : {}) });
+                          ...(o.skill ? { skill: { ...o.skill, adopt: "The app ships an agent skill. Offer it to the user (what it would make you do, from its description); adopt it only on their yes, checking sha256, and ask again if the hash changes." } } : {}),
+                          // Its Claude Code plugin: hooks run code on every matching event, so the same rule, louder.
+                          ...(o.plugin ? { plugin: { ...o.plugin, install: `claude plugin install ${o.plugin.name} --marketplace ${o.plugin.marketplace}`,
+                              adopt: "The app ships a Claude Code plugin, whose hooks run code on this machine without being called. Offer it to the user once, saying what its hooks trigger on; install only on their yes. It takes effect next session." } } : {}) });
         },
         "call-app-tool": async ({ app, tool, arguments: args }) => call(app, tool, args),
     };
@@ -303,7 +306,7 @@ if (process.argv[2] === "--selfcheck") {
                                         { name: "Bare", desc: "", card: "https://b.test/card", appId: "0xb" }], rejected: [{}] }),
         exploreApps: async (q) => ({ ranked_by: q ? "meaning" : null, apps: [{ app: "kings-foil", name: "Kings Foil", relevant: true }, { app: "b", name: "B", relevant: false }] }),
         resolveCard: async (url) => (url.includes("b.test") ? { card: { name: "Bare" }, mcp: null } : { card: { name: "Kings Foil", capabilities: { extensions: [{ uri: APP_EXTENSION,
-            params: { skill: { url: "https://k.test/skill/kings-foil/SKILL.md", sha256: "b".repeat(64) } } }] } }, mcp: "https://k.test/mcp" }),
+            params: { skill: { url: "https://k.test/skill/kings-foil/SKILL.md", sha256: "b".repeat(64) }, plugin: { marketplace: "k/kf", name: "kings-foil" } } }] } }, mcp: "https://k.test/mcp" }),
         remoteTools: async () => (opened++, remote),
     };
     deps.notify = serve(handlers(deps), { input, output });
@@ -327,6 +330,7 @@ if (process.argv[2] === "--selfcheck") {
     call(6, "open-app", { app: "https://k.test/card" }); await wait();
     assert(body(5).tools[0].name === "search" && opened === 1, "open once, by name or by card URL");
     assert(body(5).skill?.url === "https://k.test/skill/kings-foil/SKILL.md" && /user/.test(body(5).skill.adopt), "open-app offers the app's skill, for the user to adopt");
+    assert(body(5).plugin?.marketplace === "k/kf" && /user/.test(body(5).plugin.adopt), "open-app offers the app's plugin, for the user to install");
     assert(replies.some((r) => r.method === "notifications/tools/list_changed"), "opening notifies");
     rpc(7, "tools/list"); call(8, "kings-foil__search", { query: "x" }); call(9, "call-app-tool", { app: "kings-foil", tool: "search" });
     call(10, "call-app-tool", { app: "kings-foil", tool: "boom" }); call(11, "open-app", { app: "nope" }); call(13, "open-app", { app: "Bare" }); rpc(12, "nope"); await wait();

package/src/core/embed.js

@@ -31,13 +31,17 @@ export const EMBED_DIM = DIM;
 /** A short name for the space a taste or a reaction lives in. */
 export const SPACE = `embeddinggemma-${DIM}`;
 
-let _model = null;
+let _model = null, _progress = null;
+/** Hear the model download: transformers.js's own events ({status, file, loaded, total}), on
+ *  the thread that loads it (the page's own worker, when it calls the *Direct functions). */
+export const onModelProgress = (f) => { _progress = f; };
 const model = () => (_model ??= (async () => {
     const { AutoModel, AutoTokenizer } = await import("@huggingface/transformers");
     // WESTMARCH_EMBED_DEVICE=cuda puts a Node publisher's document side on a GPU.
     const device = globalThis.process?.env?.WESTMARCH_EMBED_DEVICE;
-    const [tokenizer, m] = await Promise.all([AutoTokenizer.from_pretrained(ONNX),
-        AutoModel.from_pretrained(ONNX, { dtype: DTYPE, ...(device && { device }) })]);
+    const progress_callback = (p) => _progress?.(p);
+    const [tokenizer, m] = await Promise.all([AutoTokenizer.from_pretrained(ONNX, { progress_callback }),
+        AutoModel.from_pretrained(ONNX, { dtype: DTYPE, progress_callback, ...(device && { device }) })]);
     // `sentence_embedding`, not mean pooling: the model's dense head runs after the pool.
     return async (texts) => {
         const { sentence_embedding: e } = await m(await tokenizer(texts, { padding: true, truncation: true }));

package/src/discover/apps.js

@@ -39,6 +39,14 @@ export const skillOf = (ext) => {
     return k && safeView(k.url) && /^[0-9a-f]{64}$/.test(k.sha256 ?? "") ? { url: k.url, sha256: k.sha256 } : null;
 };
 
+// The app's Claude Code plugin, for what a skill can't do (hooks). Both fields reach a shell command
+// in the reader's agent, so they are held to a GitHub owner/repo (or https URL) and a plain name.
+export const PLUGIN_MARKETPLACE = /^([\w.-]+\/[\w.-]+|https:\/\/[\w.\/-]+)$/, PLUGIN_NAME = /^[a-z0-9][a-z0-9-]*$/;
+export const pluginOf = (ext) => {
+    const p = ext?.params?.plugin;
+    return p && PLUGIN_MARKETPLACE.test(p.marketplace ?? "") && PLUGIN_NAME.test(p.name ?? "") ? { marketplace: p.marketplace, name: p.name } : null;
+};
+
 /** A verified card → an app entry. `views` is westmarch's param; the SDK does not read it. */
 export function toApp({ card, appId, fromBlock, repos }, cardUrl) {
     const ext = card.capabilities.extensions.find((e) => e.uri === APP_EXTENSION);
@@ -46,7 +54,7 @@ export function toApp({ card, appId, fromBlock, repos }, cardUrl) {
     const p = ext?.params?.paid;
     const paid = p && /^https?:\/\//.test(p.url ?? "") && p.url.includes("{id}") && /^\d+$/.test(String(p.price)) ? p : null;
     return {
-        appId, card: cardUrl, fromBlock, repos, views, paid, skill: skillOf(ext),
+        appId, card: cardUrl, fromBlock, repos, views, paid, skill: skillOf(ext), plugin: pluginOf(ext),
         mcp: safeView(ext?.params?.mcp),
         name: card.name ?? appId,
         desc: card.description ?? "",

package/src/publish/ship.js

@@ -83,6 +83,7 @@ const log = (s) => console.error(`[ship] ${s}`);
 //                                 //   (pages/intro.html, a fragment, replaces the description on the front page)
 //             "nav": [{ "href": "coverage.html", "label": "Coverage" }],   // optional: header links to them
 //             "examples": ["…"],  // optional: questions to start from, on the front page (and a private app's gate)
+//             "plugin": {"marketplace": "owner/repo", "name": "…"},   // optional: its Claude Code plugin (hooks), offered by open-app
 //             "skill": "skill/SKILL.md",   // optional: the app's agent skill, served at skill/<app>/SKILL.md and named
 //                                          //   in the card with its sha256; fangorn-index offers it to whoever opens the app
 //             "agent": "app/agent.js" },   // optional: the module whose registerAgent the card lists, when
@@ -165,10 +166,13 @@ export function presentedRepos(cfg) {
 
 /** The Worker's wrangler config: the stock worker, the site as its assets, its bucket, the AI
  *  binding, the cron and CONFIG. `worker` is site/worker.js unless a test points elsewhere. */
-export function wranglerConfig({ name, site, account, cron = "*/5 * * * *", config, url, ipfsGateway, worker = join(HERE, "../../site/worker.js") }) {
+export function wranglerConfig({ name, site, account, cron = "*/5 * * * *", cpuMs, config, url, ipfsGateway, worker = join(HERE, "../../site/worker.js") }) {
     return {
         name, main: worker, compatibility_date: "2025-09-15", compatibility_flags: ["nodejs_compat"],
         ...(account ? { account_id: account } : {}),
+        // site.cpu_ms: the indexer's CPU per run. Reading one big commit (50k records) outlasts the
+        // 30 s Workers Paid default; up to 300000. The Free plan refuses the deploy with it set.
+        ...(cpuMs ? { limits: { cpu_ms: cpuMs } } : {}),
         assets: { directory: site, binding: "ASSETS" },
         r2_buckets: [{ binding: "VIEW", bucket_name: `${name}-view` }],
         ai: { binding: "AI" },
@@ -272,7 +276,7 @@ export async function buildForDeploy(configPath, wranglerPath = join(dirname(res
         log(`the R2 bucket ${w.r2_buckets?.[0]?.bucket_name} doesn't name the Worker; two apps sharing one overwrite each other's view`);
 
     process.env.FANGORN_NETWORK = process.env.CHAIN_NAME = cfg.network;
-    const [{ createPublicClient, http }, view, { ORIGIN }] = await Promise.all([import("viem"), import("./view.js"), import("../../site/worker.js")]);
+    const [{ createPublicClient, http }, view] = await Promise.all([import("viem"), import("./view.js")]);
     const FangornConfig = await view.network(cfg.network);
     const fangorn = await view.readOnlyFangorn();
     fangorn.setAppId(cfg.app);
@@ -283,7 +287,7 @@ export async function buildForDeploy(configPath, wranglerPath = join(dirname(res
 
     const base = dirname(resolve(configPath));
     const { site, writeCard } = await buildSite(cfg, base);
-    writeCard(ORIGIN, { fromBlock, repos: [] });
+    writeCard(view.ORIGIN, { fromBlock, repos: [] });
     const gateway = process.env.FANGORN_IPFS_GATEWAY;
     w.assets = { ...w.assets, directory: relative(dirname(resolve(wranglerPath)), site) };
     w.vars = { ...w.vars, CONFIG: { app: cfg.app, name: cfg.name, description: cfg.description, fromBlock, network: cfg.network,
@@ -400,7 +404,7 @@ export async function ship(configPath, { crawl = true, deploy = true, dry = fals
     const workerConfig = (u) => {
         const path = join(dir, "wrangler.json");
         writeFileSync(path, JSON.stringify(wranglerConfig({
-            name: cfg.site.project, site, account: cfg.site.account, cron: cfg.site.cron, url: cfg.site.url, ipfsGateway: view.ipfsGateway(),
+            name: cfg.site.project, site, account: cfg.site.account, cron: cfg.site.cron, cpuMs: cfg.site.cpu_ms, url: cfg.site.url, ipfsGateway: view.ipfsGateway(),
             config: { app: cfg.app, name: cfg.name, description: cfg.description, fromBlock: state.fromBlock, network: cfg.network,
                       schema: { types: cfg.types, relations: cfg.relations }, repos: register ? [] : repos, paid: sale(u).sale, private: access(u).config },
         }), null, 1));
@@ -576,7 +580,8 @@ export async function buildSite(cfg, base) {
                     description: `The keys to sealed records (POST ${u}/paid), per record.` }] : []),
             ],
             fangorn: { app: cfg.app, fromBlock, repos, views: [`${u}/view`], mcp: `${u}/mcp`, config, ...(paid ? { paid } : {}), ...(access ? { access } : {}),
-                       ...(skill ? { skill: { url: `${u}/${skill.path}`, sha256: skill.sha256 } } : {}) },
+                       ...(skill ? { skill: { url: `${u}/${skill.path}`, sha256: skill.sha256 } } : {}),
+                       ...(cfg.site.plugin ? { plugin: cfg.site.plugin } : {}) },
         });
         mkdirSync(join(site, ".well-known"), { recursive: true });
         writeFileSync(join(site, ".well-known/agent-card.json"), JSON.stringify(agentCard, null, 1));
@@ -715,8 +720,11 @@ export async function indexHere(configPath, { port = 8791 } = {}) {
     if (secret) writeFileSync(vars, `PAID_SECRET=${secret}\n`, { mode: 0o600 });
     const url = `http://127.0.0.1:${port}`, start = Math.floor(Date.now() / 1000);
     const wrangler = spawn("npx", ["wrangler", "dev", "--config", wpath, "--port", String(port), "--ip", "127.0.0.1", "--test-scheduled"],
-        { cwd: join(HERE, "../.."), stdio: ["ignore", "inherit", "inherit"],
+        { cwd: join(HERE, "../.."), stdio: ["ignore", "pipe", "inherit"],
           env: { ...process.env, ...(cfg.site.account ? { CLOUDFLARE_ACCOUNT_ID: cfg.site.account } : {}) } });
+    // A run already holds the lock (the deployed cron's): this one skips, and the catalog would never move.
+    let skipped = false;
+    wrangler.stdout.on("data", (b) => { process.stdout.write(b); if (String(b).includes("index: a run is in progress; skipped")) skipped = true; });
     try {
         for (;;) {
             if (wrangler.exitCode !== null) throw new Error("wrangler dev exited");
@@ -728,6 +736,7 @@ export async function indexHere(configPath, { port = 8791 } = {}) {
         let catalog = null;
         while (!(catalog?.generated_at >= start)) {
             if (wrangler.exitCode !== null) throw new Error("wrangler dev exited");
+            if (skipped) throw new Error("another indexer run holds the lock (the deployed cron's, most likely); it frees within 2 minutes of that run dying. Try again then.");
             await new Promise((ok) => setTimeout(ok, 5000));
             catalog = await fetch(`${url}/view/cdn/catalog`).then((r) => (r.ok ? r.json() : null), () => null);
         }

package/src/publish/view.js

@@ -40,6 +40,9 @@ import { dirname } from "node:path";
 import { EMBED_DIM, EMBED_DTYPE, EMBED_MODEL, embedDocumentDirect, packVec, unpackVec } from "../core/embed.js";
 import { coverage as fitCoverage } from "../market/reactions.js";
 
+// The origin the agent card is built with; the Worker swaps in wherever it is served from.
+// Here, not in site/worker.js: workerd refuses a Worker module exporting anything but handlers.
+export const ORIGIN = "https://origin.westmarch.invalid";
 export const EMBEDDER = { runtime: "transformers.js", model: EMBED_MODEL, dtype: EMBED_DTYPE };
 const SINGULAR = ["identity", "title", "subtitle", "temporal", "spatial", "media", "thread"];
 const MULTI = ["tags", "measures", "relations", "text"];
@@ -168,12 +171,21 @@ export async function readOnlyFangorn() {
  *  A build that must not touch the chain — a candidate recipe graded before it ships —
  *  runs the exact code a real one does. */
 export function localFangorn(chain, { appId, owner }) {
-    const root = (k) => `0x${createHash("sha256").update(JSON.stringify(chain[k])).digest("hex")}`;
+    // Every version handed out is kept, so repoDiff can diff any two, as the SDK diffs two roots.
+    const versions = new Map();
+    const root = (k) => { const h = `0x${createHash("sha256").update(JSON.stringify(chain[k])).digest("hex")}`; versions.set(h, structuredClone(chain[k]?.records ?? [])); return h; };
+    const repoDiff = async (from, to) => {
+        if (!versions.has(from) || !versions.has(to)) throw new Error("unknown root");
+        const [a, b] = [versions.get(from), versions.get(to)], cidOf = new Map(a.map((r) => [r.id, r.cid])), ids = new Set(b.map((r) => r.id));
+        return { puts: b.filter((r) => cidOf.get(r.id) !== r.cid), deletes: a.filter((r) => !ids.has(r.id)).map((r) => r.id),
+                 superseded: a.filter((r) => !ids.has(r.id) || b.some((x) => x.id === r.id && x.cid !== r.cid)).map((r) => r.cid) };
+    };
     return {
+        engine: { repoDiff }, reads: 0,
         setAppId() {}, getAppId: () => appId,
         getAppRegistry: () => ({ getAppOwner: async () => owner }),
         appRepos: async () => Object.keys(chain).map((k) => ({ owner: k.split("/")[0], repo: k.split("/").slice(1).join("/"), root: root(k) })),
-        readRepo: async (o, repo) => { const c = chain[`${o}/${repo}`]; return { tip: c ? root(`${o}/${repo}`) : null, contents: { records: c?.records ?? [] } }; },
+        readRepo: async function (o, repo) { this.reads++; const c = chain[`${o}/${repo}`]; return { tip: c ? root(`${o}/${repo}`) : null, contents: { records: c?.records ?? [] } }; },
     };
 }
 
@@ -243,11 +255,12 @@ export function r2Store(bucket, prefix = "view/") {
 const readJson = async (store, p, dflt) => { const b = await store.get(p); return b ? JSON.parse(new TextDecoder().decode(b)) : dflt; };
 const writeJson = (store, p, o) => store.put(p, JSON.stringify(o, null, 1));
 
-/** What a domain already serves: its manifest and every row (for tombstones and coverage). */
-async function loadDomain(store, dir) {
+/** What a domain already serves: its manifest and every row (for tombstones and coverage), or
+ *  with `only`, the rows of those shard files alone. */
+async function loadDomain(store, dir, only = null) {
     const manifest = await readJson(store, `${dir}/manifest`, null);
     const rows = new Map();
-    for (const s of manifest?.shards ?? []) {
+    for (const s of (manifest?.shards ?? []).filter((x) => !only || only.includes(x.file))) {
         const b = await store.get(`${dir}/shards/${s.file}`);
         if (!b) throw new Error(`${dir}: shard ${s.file} is named by its manifest and missing`);
         for (const line of gunzipSync(b).toString("utf8").split("\n")) if (line.trim()) { const r = JSON.parse(line); rows.set(r.track_id, r); }
@@ -339,21 +352,38 @@ async function retry(f, tries = 6) {
     }
 }
 
+const CHECKPOINT = 2048;   // rows embedded between saves; a killed run redoes at most this many
 async function publishDomain({ fangorn, store, schema, schemaHash, publisher, repo, head, domain, dir, rebake, embedBatch, embedder, log, shardRows, open }) {
     // The head the domain was built from, and the schema it was built under: neither moved,
     // nothing to read. This is what keeps a cron run with nothing new to one read per repo.
     const was = rebake ? null : await readJson(store, `${dir}/manifest`, null);
     if (was && was.head === head && was.schema_hash === schemaHash) return null;
 
-    const { contents } = await retry(() => fangorn.readRepo(publisher, repo));
+    // The head moved from one this domain was built from: read only what changed between them (the
+    // SDK's repoDiff opens only the changed paths of the record tree), not every record again. A
+    // schema with relations links each row against all the others, so it still reads the whole repo;
+    // so does an SDK whose diff can't name the replaced versions, and a diff that fails (a force-push
+    // to a head whose blocks are gone).
+    // ponytail: relations read whole; diff their targets too when a big corpus declares one.
+    const diff = was?.head && !schema.relations?.length && fangorn.engine?.repoDiff
+        ? await retry(() => fangorn.engine.repoDiff(was.head, head)).catch((e) => { log(`${domain}: diff from ${was.head} failed (${e.message}); reading it whole`); return null; })
+        : null;
+    const got = diff?.superseded ? diff.puts : (await retry(() => fangorn.readRepo(publisher, repo))).contents.records;
     // `open`: a private app's rows are sealed on chain (x402.js's sealRow); the Worker opens them here.
-    const records = new Map(await Promise.all(contents.records.filter((r) => r.payload && typeof r.payload === "object")
+    const records = new Map(await Promise.all(got.filter((r) => r.payload && typeof r.payload === "object")
         .map(async (r) => [r.cid, { id: r.id, fields: { ...(await open(r.payload)), entityType: r.type } }])));
     linkRelations(records, schema);
     const fieldsOf = (cid) => records.get(cid).fields;
 
     if (rebake) await store.del(dir);   // old shards would stay reachable
-    const had = await loadDomain(store, dir);
+    // Append only: the diff replaced nothing, so no row already served changes, and the 50k of them
+    // are left unread. Only the shards saved since the last head (`ahead`, a killed run's checkpoints)
+    // are read, to skip what they hold. A removal, an edit, a rebake or a coverage refit reads all.
+    // Rows in `ahead` are already counted, and come back among the puts of a diff from the old head.
+    const aheadRows = (was?.shards ?? []).filter((x) => was.ahead?.includes(x.file)).reduce((n, x) => n + x.count, 0);
+    const append = !!was && !!diff?.superseded && !diff.superseded.length && !was.tombstones?.length
+        && (was.count ?? 0) + diff.puts.length - aheadRows < COVERAGE_REFIT * (was.coverage?.fit_at ?? 0);
+    const had = await loadDomain(store, dir, append ? was.ahead ?? [] : null);
     const made = had.manifest?.embedder ?? (had.manifest ? { runtime: "fastembed", dtype: "fp32" } : null);
     // Same model and dtype is the same encoder, whatever ran it: q8 on CPU, CUDA and WASM agree to
     // ~0.98-0.99 cosine, fp32 against q8 noticeably less.
@@ -364,10 +394,11 @@ async function publishDomain({ fangorn, store, schema, schemaHash, publisher, re
     const dead = new Set(had.manifest?.tombstones ?? []);
     const live = (id) => had.rows.has(id) && !dead.has(id);
     const fresh = [...records.keys()].filter((id) => !live(id));
-    const gone = [...had.rows.keys()].filter((id) => live(id) && !records.has(id));
+    // From a diff, `records` holds only what changed: what is gone is what it superseded.
+    const gone = diff?.superseded ? diff.superseded.filter(live) : [...had.rows.keys()].filter((id) => live(id) && !records.has(id));
     // A row already served whose links changed (its matter came later, or moved): new fields,
-    // the same vector, so no embedding, only a rewrite.
-    const relinked = [...had.rows.keys()].filter((id) => live(id) && records.has(id)
+    // the same vector, so no embedding, only a rewrite. Only relations relink, and they read whole.
+    const relinked = diff?.superseded ? [] : [...had.rows.keys()].filter((id) => live(id) && records.has(id)
         && JSON.stringify(had.rows.get(id).fields) !== JSON.stringify(fieldsOf(id)));
     for (const id of relinked) had.rows.get(id).fields = fieldsOf(id);
 
@@ -383,6 +414,34 @@ async function publishDomain({ fangorn, store, schema, schemaHash, publisher, re
         }
     }
 
+    const manifest = had.manifest ?? { name: domain, description: "", count: 0, dim: EMBED_DIM, model: EMBED_MODEL,
+        distance: "Cosine", embedder, filter: { owner: [publisher], repo: [repo] }, shards: [], tombstones: [] };
+    // Pages and R2 serve files of any size, but a reader holds a shard whole; 52k Steam games
+    // gzipped to 28 MiB in one. ponytail: split by row count (~550 B a row gzipped).
+    const writeShard = async (all) => {
+        const files = [];
+        for (let i = 0; i < all.length; i += shardRows) {
+            const ls = all.slice(i, i + shardRows);
+            const gz = gzipSync(`${ls.join("\n")}\n`, { level: 9 });
+            const digest = sha256(gz);
+            const file = `shard-${String(manifest.shards.length).padStart(4, "0")}-${digest.slice(0, 12)}.ndjson.gz`;
+            await store.put(`${dir}/shards/${file}`, gz);
+            manifest.shards.push({ file, count: ls.length, bytes: gz.length, sha256: digest });
+            files.push(file);
+        }
+        return files;
+    };
+    // What is embedded so far, served now under the old head: a run killed later (CPU, wall time:
+    // no exception, no `finally`) loses at most CHECKPOINT rows, and the next run embeds only the rest.
+    const save = async () => {
+        manifest.ahead = [...(manifest.ahead ?? []), ...await writeShard(lines)];
+        const rows = lines.map((l) => JSON.parse(l));
+        for (const r of rows) { had.rows.set(r.track_id, r); dead.delete(r.track_id); }
+        lines.length = 0;
+        manifest.head = had.manifest?.head ?? null;
+        manifest.schema_hash = schemaHash;
+        await applySpec(store, dir, manifest, schema, had.rows, dead, rolesOf, append ? rows : null);
+    };
     let n = 0, stopped = null;
     const lines = [];
     // 32 at a time: what an encoder batches (embed.js, a GPU, the Worker's AI binding).
@@ -400,39 +459,26 @@ async function publishDomain({ fangorn, store, schema, schemaHash, publisher, re
         }
         if (Math.floor((n + ids.length) / 1000) > Math.floor(n / 1000)) log(`${domain}: embedded ${n + ids.length}/${fresh.length}`);
         n += ids.length;
+        if (lines.length >= CHECKPOINT && n < fresh.length) await save();
     }
-    const manifest = had.manifest ?? { name: domain, description: "", count: 0, dim: EMBED_DIM, model: EMBED_MODEL,
-        distance: "Cosine", embedder, filter: { owner: [publisher], repo: [repo] }, shards: [], tombstones: [] };
     // Unfinished: the old head stays, so the next run sees a head to catch up to and embeds only what is missing.
     manifest.head = stopped ? (had.manifest?.head ?? null) : head;
     manifest.schema_hash = schemaHash;
+    if (!stopped) delete manifest.ahead;   // the head is taken: everything is behind it
     if (!fresh.length && !gone.length && !relinked.length && had.manifest && !rebake) {
-        await applySpec(store, dir, manifest, schema, had.rows, dead);   // a schema change alone still lands
+        await applySpec(store, dir, manifest, schema, had.rows, dead, new Map(), append ? [] : null);   // a schema change alone still lands
         return { added: 0, removed: 0 };
     }
 
-    // Pages and R2 serve files of any size, but a reader holds a shard whole; 52k Steam games
-    // gzipped to 28 MiB in one. ponytail: split by row count (~550 B a row gzipped).
-    const writeShard = async (all) => {
-        for (let i = 0; i < all.length; i += shardRows) {
-            const ls = all.slice(i, i + shardRows);
-            const gz = gzipSync(`${ls.join("\n")}\n`, { level: 9 });
-            const digest = sha256(gz);
-            const file = `shard-${String(manifest.shards.length).padStart(4, "0")}-${digest.slice(0, 12)}.ndjson.gz`;
-            await store.put(`${dir}/shards/${file}`, gz);
-            manifest.shards.push({ file, count: ls.length, bytes: gz.length, sha256: digest });
-        }
-    };
     if (stopped) {
-        await writeShard(lines);
-        for (const l of lines) { const r = JSON.parse(l); had.rows.set(r.track_id, r); dead.delete(r.track_id); }
-        await applySpec(store, dir, manifest, schema, had.rows, dead, rolesOf);
+        await save();
         log(`${domain}: stopped at ${n}/${fresh.length} (${stopped.message}); the rest on the next run`);
         return { added: n, removed: 0, partial: true };
     }
+    const last = lines.map((l) => JSON.parse(l));
     if (lines.length) {
         await writeShard(lines);
-        for (const l of lines) { const r = JSON.parse(l); had.rows.set(r.track_id, r); dead.delete(r.track_id); }
+        for (const r of last) { had.rows.set(r.track_id, r); dead.delete(r.track_id); }
     }
     for (const id of gone) dead.add(id);
     // A tombstone hides a row from readers, but its shard still serves the bytes. A record
@@ -441,6 +487,8 @@ async function publishDomain({ fangorn, store, schema, schemaHash, publisher, re
     // files are deleted. No re-embedding: the vectors are already in the rows.
     // ponytail: rewrites the whole domain on every removal; compact past a threshold if domains get big.
     if (dead.size || relinked.length) {
+        // A compaction rewrites `had.rows` as the whole domain, which append mode never loaded.
+        if (append) throw new Error(`${domain}: append mode reached a compaction; nothing written past the new shard`);
         for (const s of manifest.shards) await store.del(`${dir}/shards/${s.file}`);
         manifest.shards = [];
         for (const id of dead) had.rows.delete(id);
@@ -449,16 +497,19 @@ async function publishDomain({ fangorn, store, schema, schemaHash, publisher, re
     }
     manifest.tombstones = [...dead].sort();
     manifest.created_at = Math.floor(Date.now() / 1000);
-    await applySpec(store, dir, manifest, schema, had.rows, dead, rolesOf);
+    await applySpec(store, dir, manifest, schema, had.rows, dead, rolesOf, append ? last : null);
     log(`${domain}: +${fresh.length} embedded, -${gone.length} removed, ${relinked.length} relinked, ${manifest.count} live`);
     return { added: fresh.length, removed: gone.length, ...(relinked.length ? { relinked: relinked.length } : {}) };
 }
 
 /** Counts, types, roles, description, presentation, coverage and the hosts its media role
- *  links to (what a Worker may show inline, see site/worker.js) from what the domain holds. */
-async function applySpec(store, dir, manifest, schema, rows, dead, guessed = new Map()) {
-    const liveRows = [...rows.values()].filter((r) => !dead.has(r.track_id));
-    const types = {};
+ *  links to (what a Worker may show inline, see site/worker.js) from what the domain holds.
+ *  `added`: append mode, where `rows` is not the whole domain: fold these new rows into the
+ *  manifest's totals instead of recounting. Coverage is left as fitted (publishDomain appends only
+ *  while no refit is due). ponytail: hosts only grow on append; a removal recounts them. */
+async function applySpec(store, dir, manifest, schema, rows, dead, guessed = new Map(), added = null) {
+    const liveRows = added ?? [...rows.values()].filter((r) => !dead.has(r.track_id));
+    const types = added ? Object.fromEntries((manifest.entity_types ?? []).map((e) => [e.type, e.count])) : {};
     for (const r of liveRows) types[r.fields?.entityType] = (types[r.fields?.entityType] ?? 0) + 1;
     const tags = Object.entries(types).sort((a, b) => b[1] - a[1]).map(([t]) => t);
     // Compare with what is stored, not with the manifest as it arrived: publishDomain has already
@@ -466,20 +517,20 @@ async function applySpec(store, dir, manifest, schema, rows, dead, guessed = new
     // the old manifest naming deleted shards.
     const before = JSON.stringify(await readJson(store, `${dir}/manifest`, null));
 
-    manifest.count = liveRows.length;
+    manifest.count = added ? (manifest.count ?? 0) + added.length : liveRows.length;
     manifest.entity_types = tags.map((t) => ({ type: t, count: types[t] }));
     const spec = domainSpec(schema, tags);
-    const fields = [...new Set(liveRows.slice(0, 500).flatMap((r) => Object.keys(r.fields ?? {})))].sort();
+    const fields = [...new Set([...(added ? manifest.role_map?.fields ?? [] : []), ...liveRows.slice(0, 500).flatMap((r) => Object.keys(r.fields ?? {}))])].sort();
     manifest.role_map = { ...(spec?.role_map ?? manifest.role_map ?? guessed.values().next().value ?? completeRoles()), fields };
     if (spec?.description) manifest.description = spec.description;
     if (spec && Object.keys(spec.presentation).length) manifest.presentation = spec.presentation;
     const media = [manifest.role_map.media ?? []].flat();
-    manifest.link_hosts = [...new Set(liveRows.flatMap((r) => media.map((m) => r.fields?.[m]))
+    manifest.link_hosts = [...new Set([...(added ? manifest.link_hosts ?? [] : []), ...liveRows.flatMap((r) => media.map((m) => r.fields?.[m]))
         .filter((u) => typeof u === "string" && u.startsWith("https://"))
-        .map((u) => { try { return new URL(u).hostname; } catch { return null; } }).filter(Boolean))].sort();
+        .map((u) => { try { return new URL(u).hostname; } catch { return null; } })].filter(Boolean))].sort();
 
     const cov = manifest.coverage;
-    if (liveRows.length && (!cov || liveRows.length >= COVERAGE_REFIT * (cov.fit_at ?? 0))) {
+    if (!added && liveRows.length && (!cov || liveRows.length >= COVERAGE_REFIT * (cov.fit_at ?? 0))) {
         manifest.coverage = { ...fitCoverage(liveRows.map((r) => unpackVec(r.v))), fit_at: liveRows.length };
     }
     if (JSON.stringify(manifest) !== before) await writeJson(store, `${dir}/manifest`, manifest);
@@ -617,6 +668,49 @@ if (typeof process !== "undefined" && import.meta.url === `file://${process.argv
     eq((await publishView({ fangorn, app: APP, schema, out: big, repos: ["big"], embedBatch: capped, log: () => {} }))[bd],
        { added: 48, removed: 0 }, "the next run embeds only the rest");
     eq(manifestOf(big, bd).count, 80, "and serves all of it");
+    // Killed partway (a Worker over its CPU or wall time: no exception, no return): the last
+    // checkpoint is served, and the next run embeds only what came after it.
+    chain[BIG] = { records: Array.from({ length: CHECKPOINT + 100 }, (_, i) => ({ id: `k${i}`, cid: `kill${i}`, type: "shop.item.v1", payload: { sku: `k${i}`, name: `Kept ${i}` } })) };
+    let left = CHECKPOINT + 64;
+    const dies = (ts) => (left -= ts.length) < 0 ? new Promise(() => {}) : Promise.all(ts.map(embed));
+    publishView({ fangorn, app: APP, schema, out: big, repos: ["big"], embedBatch: dies, log: () => {} });
+    for (let i = 0; i < 200 && manifestOf(big, bd).count < CHECKPOINT; i++) await new Promise((ok) => setTimeout(ok, 10));
+    eq((await publishView({ fangorn, app: APP, schema, out: big, repos: ["big"], embedBatch: capped, log: () => {} }))[bd],
+       { added: 100, removed: 80 }, "a killed run resumes from its last checkpoint");
+    // A built domain whose head moves reads only the diff: 10 new tracks on top of thousands
+    // never read the thousands again, and a removal or an edit drops the version it replaced.
+    const whole = fangorn.reads;
+    chain[BIG].records.push(...Array.from({ length: 10 }, (_, i) => ({ id: `n${i}`, cid: `next${i}`, type: "shop.item.v1", payload: { sku: `n${i}`, name: `New ${i}` } })));
+    chain[BIG].records.splice(0, 1, { ...chain[BIG].records[0], cid: "kill0-v2", payload: { sku: "k0", name: "Kept 0, renamed" } });
+    chain[BIG].records.splice(1, 1);
+    eq([(await publishView({ fangorn, app: APP, schema, out: big, repos: ["big"], embedBatch: capped, log: () => {} }))[bd], fangorn.reads - whole],
+       [{ added: 11, removed: 2 }, 0], "a moved head reads only what changed");
+    eq(manifestOf(big, bd).count, CHECKPOINT + 100 + 10 - 1, "and serves the repo as it now stands");
+    // Only additions: the rows already served are not read at all, only a delta shard is written.
+    const disk = fsStore(big), opened = [];
+    const watched = { ...disk, get: (p) => { opened.push(p); return disk.get(p); } };
+    const n0 = manifestOf(big, bd).count, shards0 = manifestOf(big, bd).shards.length;
+    chain[BIG].records.push(...Array.from({ length: 10 }, (_, i) => ({ id: `m${i}`, cid: `more${i}`, type: "shop.item.v1", payload: { sku: `m${i}`, name: `More ${i}` } })));
+    eq((await publishView({ fangorn, app: APP, schema, store: watched, repos: ["big"], embedBatch: capped, log: () => {} }))[bd],
+       { added: 10, removed: 0 }, "an append embeds only the new records");
+    eq(opened.filter((p) => p.includes("/shards/")), [], "and reads no shard it already served");
+    const after = manifestOf(big, bd);
+    eq([after.count, after.shards.length, after.entity_types[0].count, after.ahead], [n0 + 10, shards0 + 1, n0 + 10, undefined], "counts, types and one delta shard, with the head taken");
+    eq((await liveFields(big)).length, n0 + 10, "and every row is still served");
+    // An append killed after a checkpoint: the next run reads only that checkpoint's shard, and
+    // embeds only what came after it, never the checkpointed rows a second time.
+    const n1 = manifestOf(big, bd).count, fit = manifestOf(big, bd).coverage.fit_at, extra = CHECKPOINT + 2;
+    if (n1 + extra >= COVERAGE_REFIT * fit) throw new Error(`test setup: ${n1} + ${extra} rows would refit coverage (fit at ${fit}), not append`);
+    chain[BIG].records.push(...Array.from({ length: extra }, (_, i) => ({ id: `a${i}`, cid: `app${i}`, type: "shop.item.v1", payload: { sku: `a${i}`, name: `Appended ${i}` } })));
+    left = CHECKPOINT + 1;
+    publishView({ fangorn, app: APP, schema, out: big, repos: ["big"], embedBatch: dies, log: () => {} });
+    for (let i = 0; i < 300 && !manifestOf(big, bd).ahead; i++) await new Promise((ok) => setTimeout(ok, 10));
+    eq([manifestOf(big, bd).count, manifestOf(big, bd).ahead?.length], [n1 + CHECKPOINT, 1], "a killed append serves its checkpoint, named as ahead of the head");
+    opened.length = 0;
+    eq((await publishView({ fangorn, app: APP, schema, store: watched, repos: ["big"], embedBatch: capped, log: () => {} }))[bd],
+       { added: 2, removed: 0 }, "and the next run embeds only what came after it");
+    eq([opened.filter((p) => p.includes("/shards/")).length, manifestOf(big, bd).count, (await liveFields(big)).length],
+       [1, n1 + extra, n1 + extra], "reading only the checkpoint's shard, and serving each row once");
     delete chain[BIG];
     rmSync(big, { recursive: true, force: true });
     eq((await publishView({ fangorn, app: APP, schema, out, embed, rebake: true, log: () => {} }))[d],

package/template/package.json

@@ -6,7 +6,7 @@
     "deploy": "wrangler deploy"
   },
   "dependencies": {
-    "@fangorn-network/westmarch": "2026.10.27",
+    "@fangorn-network/westmarch": "2026.10.29",
     "@huggingface/transformers": "^4.3.1",
     "wrangler": "^4.83.0"
   }