@fangorn-network/westmarch 0.0.20-dev → 0.1.0

package/src/publish/devnet.js

@@ -0,0 +1,146 @@
+// A Fangorn network on this machine: anvil with the contracts built from source behind their
+// proxies, a content-addressed store with an IPFS gateway over it, and (optionally) an encoder
+// that answers as the Worker's AI binding does. What `westmarch-ship --devnet` and
+// scripts/e2e-worker.mjs run an app against, with nothing leaving the machine.
+//
+// Needs forge and anvil, and the contracts' source (FANGORN_CONTRACTS_DIR, default
+// ../contracts next to westmarch). USDC and Semaphore are the contracts' mocks: payments
+// aren't checked and any proof is accepted.
+//
+// The services a deployment runs are here too, each its own Worker under `wrangler dev`, wired
+// to this chain: `keyServer()` and `relay()` from the webworker checkout (FANGORN_WEBWORKER_DIR,
+// default ../webworker), and `worker()` for any other, such as an app's.
+
+import { spawn, spawnSync } from "node:child_process";
+import { createServer } from "node:http";
+import { mkdtempSync, readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { createRequire } from "node:module";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { createTestClient, encodeFunctionData, getAddress, http, parseEther, publicActions, walletActions } from "viem";
+import { generatePrivateKey, privateKeyToAccount } from "viem/accounts";
+import { bytesToHex } from "viem";
+import { DATA_REGISTRY_ABI, SETTLEMENT_REGISTRY_ABI, Fangorn, FangornConfig } from "@fangorn-network/sdk";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const ADMIN_KEY = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80";   // anvil's first account
+/**
+ * The Worker in `dir` under `wrangler dev` with `vars`, on a free port: `{ url, kill }` once `probe`
+ * answers. Its own process group, so `kill` takes workerd down with wrangler. Local bindings only.
+ */
+export async function wranglerDev(dir, vars = {}, { config, probe = "/", name = dir, log = () => {} } = {}) {
+    const free = () => new Promise((ok) => { const s = createServer().listen(0, "127.0.0.1", () => { const p = s.address().port; s.close(() => ok(p)); }); });
+    // Its own inspector port too: two workers started at once both take wrangler's default (9229), and one dies.
+    const [port, inspector] = await Promise.all([free(), free()]);
+    const w = spawn("npx", ["wrangler", "dev", ...(config ? ["--config", config] : []), "--port", String(port), "--inspector-port", String(inspector), "--ip", "127.0.0.1",
+        "--persist-to", mkdtempSync(join(tmpdir(), "devnet-wrangler-")), ...Object.entries(vars).flatMap(([k, v]) => ["--var", `${k}:${v}`])],
+        { cwd: dir, detached: true, stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, NO_D1_WARNING: "true" } });
+    let out = "";
+    w.stdout.on("data", (c) => { out += c; }); w.stderr.on("data", (c) => { out += c; });
+    const kill = () => { try { process.kill(-w.pid); } catch { /* gone */ } };
+    process.on("exit", kill);
+    const url = `http://127.0.0.1:${port}`;
+    for (const t0 = Date.now(); ; await new Promise((r) => setTimeout(r, 500))) {
+        if (w.exitCode !== null) throw new Error(`${name}: wrangler dev exited\n${out.slice(-2000)}`);
+        if (Date.now() - t0 > 120_000) { kill(); throw new Error(`${name}: no answer at ${url}${probe} in 2 minutes\n${out.slice(-2000)}`); }
+        if (await fetch(`${url}${probe}`).then(() => true, () => false)) break;
+    }
+    log(`${name} at ${url}`);
+    return { url, kill };
+}
+
+const listen = (server) => new Promise((ok) => server.listen(0, "127.0.0.1", () => ok(`http://127.0.0.1:${server.address().port}`)));
+
+export async function devnet({ contracts = process.env.FANGORN_CONTRACTS_DIR ?? resolve(HERE, "../../../contracts"),
+                               webworker = process.env.FANGORN_WEBWORKER_DIR ?? resolve(HERE, "../../../webworker"), log = () => {} } = {}) {
+    const kids = [], servers = [];
+    const stop = () => { for (const k of kids) k.kill(); for (const s of servers) s.close(); };
+    process.on("exit", stop);
+
+    // ── chain ─────────────────────────────────────────────────────────────────
+    if (spawnSync("forge", ["build"], { cwd: contracts, stdio: ["ignore", "ignore", "inherit"] }).status !== 0) throw new Error(`forge build failed in ${contracts}`);
+    const anvil = spawn("anvil", ["--port", "0", "--host", "127.0.0.1", "--chain-id", String(FangornConfig.chain.id)], { stdio: ["ignore", "pipe", "inherit"] });
+    kids.push(anvil);
+    const rpcUrl = await new Promise((ok) => { let out = ""; anvil.stdout.on("data", (c) => { out += c; const m = /Listening on (\S+)/.exec(out); if (m) ok(`http://${m[1]}`); }); });
+    const chain = createTestClient({ mode: "anvil", chain: FangornConfig.chain, transport: http(rpcUrl) }).extend(publicActions).extend(walletActions);
+    const admin = privateKeyToAccount(ADMIN_KEY);
+    // `file`: the .sol it is in, when that isn't `n`.sol.
+    const artifact = (n, file = n) => { const j = JSON.parse(readFileSync(`${contracts}/out/${file}.sol/${n}.json`, "utf8")); return { abi: j.abi, bytecode: j.bytecode.object }; };
+    const deploy = async (n, args, file) => getAddress((await chain.waitForTransactionReceipt({ hash: await chain.deployContract({ account: admin, ...artifact(n, file), args }) })).contractAddress);
+    const proxied = async (n, init) => deploy("ERC1967Proxy", [await deploy(n, []), init]);
+    const data = await proxied("DataRegistry", encodeFunctionData({ abi: DATA_REGISTRY_ABI, functionName: "initialize", args: [admin.address] }));
+    const usdc = await deploy("MockUSDC", [], "Mocks");
+    const settlement = await proxied("SettlementRegistry", encodeFunctionData({ abi: SETTLEMENT_REGISTRY_ABI, functionName: "initialize",
+        args: [admin.address, usdc, await deploy("MockSemaphore", [], "Mocks")] }));
+    // As a Worker's CONFIG.deployment takes it: strings, no BigInt.
+    const deployment = { dataRegistryContractAddress: data, settlementRegistryContractAddress: settlement, fromBlock: "0" };
+    const config = { ...FangornConfig, ...deployment, fromBlock: 0n, rpcUrl };
+    log(`anvil ${rpcUrl}, DataRegistry ${data}, SettlementRegistry ${settlement}`);
+
+    // ── storage: a content-addressed store, and the gateway that serves it ────
+    // multiformats, from the SDK's own dependencies.
+    const sdk = createRequire(createRequire(import.meta.url).resolve("@fangorn-network/sdk"));
+    const [{ CID }, { sha256 }, raw] = await Promise.all(["multiformats/cid", "multiformats/hashes/sha2", "multiformats/codecs/raw"].map((m) => import(sdk.resolve(m))));
+    const store = new Map();   // multihash hex → bytes
+    const key = (cid) => Buffer.from(CID.parse(String(cid).replace(/^ipfs:\/\//, "")).multihash.bytes).toString("hex");
+    const unused = async () => { throw new Error("unused"); };
+    const backend = {
+        putBlock: async (b) => { store.set(key(b.cid), b.bytes); },
+        putFile: async (bytes) => { const cid = CID.createV1(raw.code, await sha256.digest(bytes)); store.set(key(cid), bytes); return `ipfs://${cid}`; },
+        getFile: async (uri) => store.get(key(uri)) ?? Promise.reject(new Error(`no ${uri}`)),
+        getRawBlock: async (uri) => store.get(key(uri)) ?? Promise.reject(new Error(`no ${uri}`)),
+        put: unused, putMany: unused, get: unused, delete: async () => {},
+    };
+    const gateway = createServer((req, res) => {
+        try { const b = store.get(key(req.url.replace(/^\/ipfs\//, ""))); if (!b) throw 0; res.end(Buffer.from(b)); }
+        catch { res.statusCode = 404; res.end(); }
+    });
+    servers.push(gateway);
+    const gatewayUrl = await listen(gateway);
+
+    /** A funded client. */
+    const client = async (privateKey = generatePrivateKey()) => {
+        await chain.setBalance({ address: privateKeyToAccount(privateKey).address, value: parseEther("1") });
+        return Fangorn.create({ privateKey, config, storage: { backend } });
+    };
+
+    /** An encoder on localhost that answers as the AI binding does: POST {text} → {data}; GET → the embedder it is. */
+    const encoder = async () => {
+        const [{ embedRaw, EMBED_MODEL, EMBED_DTYPE }, { env }] = await Promise.all([import("../core/embed.js"), import("@huggingface/transformers")]);
+        env.cacheDir = `${process.env.HOME}/.cache/fangorn-mcp/models`;
+        const embedder = JSON.stringify({ runtime: "transformers.js", model: EMBED_MODEL, dtype: EMBED_DTYPE });
+        // One at a time: the Worker sends several batches at once, and the model runs one.
+        let queue = Promise.resolve();
+        const server = createServer((req, res) => {
+            if (req.method === "GET") return res.end(embedder);
+            let body = "";
+            req.on("data", (c) => { body += c; });
+            req.on("end", () => { queue = queue.then(async () => {
+                try { res.end(JSON.stringify({ data: await embedRaw(JSON.parse(body).text) })); }
+                catch (e) { res.statusCode = 500; res.end(String(e.message)); }
+            }); });
+        });
+        servers.push(server);
+        return listen(server);
+    };
+
+    // ── Workers, wired to this chain ─────────────────────────────────────────
+    const worker = (dir, vars, opts) => wranglerDev(dir, vars, { log, ...opts }).then(({ url, kill }) => (kids.push({ kill }), url));
+
+    /** The key server (fangorn-access-worker), with a fresh X25519 secret. */
+    const keyServer = () => worker(join(webworker, "fangorn-access-worker"),
+        { WORKER_X25519_SECRET: bytesToHex(crypto.getRandomValues(new Uint8Array(32))), SETTLEMENT_REGISTRY: settlement, RPC_URL: rpcUrl },
+        { probe: "/pubkey", name: "key server" });
+
+    /** Barliman, paying gas from a funded treasury key; `treasury` is the referrer on every register. */
+    const relay = async (treasury = privateKeyToAccount(generatePrivateKey()).address) => {
+        const key = generatePrivateKey();
+        await chain.setBalance({ address: privateKeyToAccount(key).address, value: parseEther("10") });
+        const url = await worker(join(webworker, "barliman"), { TREASURY_KEY: key, TREASURY: treasury, RPC_URL: rpcUrl,
+            DATA_REGISTRY: data, SETTLEMENT_REGISTRY: settlement }, { probe: "/supported", name: "relay (Barliman)" });
+        return { url, treasury };
+    };
+
+    return { rpcUrl, deployment, config, chain, usdc, backend, gatewayUrl, client, encoder, worker, keyServer, relay, stop, kids };
+}

package/src/publish/graph.js

@@ -183,7 +183,7 @@ export function treeFromGraph(contents) {
 // `process` doesn't exist in the browser, and consumers import nest() from here
 // to build a viewer tree — so this guard has to survive being evaluated by a
 // browser.
-if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/graph.js")) {
+if (import.meta.url === `file://${globalThis.process?.argv?.[1]}` && process.argv[1].endsWith("/graph.js")) {
     // sond3r's pinned names, which is also the case with every option exercised.
     const SOND3R = {
         file: "video",

package/src/publish/init.js

@@ -0,0 +1,74 @@
+#!/usr/bin/env node
+// A new app from the template: its recipe (app.json, wrangler.json, eval/), no data.
+//   npx @fangorn-network/westmarch init <app> [dir] [--no-install]
+//   westmarch build      the deploy button's build step (ship.js buildForDeploy)
+// The app's data is what its members commit; the repo this makes can be public.
+import { cpSync, existsSync, mkdtempSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { spawnSync } from "node:child_process";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { buildForDeploy, nameApp } from "./ship.js";
+
+const ROOT = join(dirname(fileURLToPath(import.meta.url)), "../..");
+const TEMPLATE = join(ROOT, "template");
+// What fangorn.network accepts: a name a shell and a URL need no quoting for.
+const APP_NAME = /^[a-z0-9][a-z0-9._-]{1,62}$/;
+
+export function init(app, dir = app, { install = true, log = console.log } = {}) {
+    if (!APP_NAME.test(app ?? "")) throw new Error(`"${app}": 2-63 characters, lowercase letters, digits, dot, dash or underscore`);
+    dir = resolve(dir);
+    if (existsSync(dir) && readdirSync(dir).length) throw new Error(`${dir} is not empty`);
+    cpSync(TEMPLATE, dir, { recursive: true });
+    renameSync(join(dir, "gitignore"), join(dir, ".gitignore"));   // npm drops a published .gitignore
+
+    const cfg = nameApp(JSON.parse(readFileSync(join(dir, "app.json"), "utf8")), app);
+    writeFileSync(join(dir, "app.json"), JSON.stringify(cfg, null, 1) + "\n");
+    const w = JSON.parse(readFileSync(join(dir, "wrangler.json"), "utf8"));
+    Object.assign(w, { name: cfg.site.project, r2_buckets: [{ ...w.r2_buckets[0], bucket_name: `${cfg.site.project}-view` }] });
+    writeFileSync(join(dir, "wrangler.json"), JSON.stringify(w, null, 1) + "\n");
+    writeFileSync(join(dir, "README.md"), readFileSync(join(dir, "README.md"), "utf8").replaceAll("my-app", app));
+    const pkg = JSON.parse(readFileSync(join(dir, "package.json"), "utf8"));
+    pkg.dependencies["@fangorn-network/westmarch"] = `^${JSON.parse(readFileSync(join(ROOT, "package.json"), "utf8")).version}`;
+    writeFileSync(join(dir, "package.json"), JSON.stringify(pkg, null, 2) + "\n");
+
+    if (install) {
+        const sh = (cmd, ...args) => { if (spawnSync(cmd, args, { cwd: dir, stdio: "inherit" }).status !== 0) throw new Error(`${cmd} ${args[0]} failed`); };
+        sh("npm", "install");   // the lock CI installs from
+        sh("git", "init", "-q", "-b", "main");
+        sh("git", "add", "-A");
+        sh("git", "commit", "-q", "-m", `${app}: from the westmarch template`);
+    }
+    log(`${app} is in ${dir}. Next:
+  1. app.json: the description, and the type's fields (role_map)
+  2. npx westmarch-ship app.json           # creates the repo ${app}, deploys the Worker, registers its agent`);
+    return dir;
+}
+
+if (process.argv[2] === "--selfcheck") {
+    const dir = join(mkdtempSync(join(tmpdir(), "init-")), "app");
+    try {
+        init("my.app_1", dir, { install: false, log: () => {} });
+        const cfg = JSON.parse(readFileSync(join(dir, "app.json"), "utf8"));
+        if (cfg.app !== "my.app_1" || cfg.site.project !== "my-app-1" || !cfg.types["my.app_1.item.v1"]?.presentation.types["my.app_1.item.v1"])
+            throw new Error(`app.json not filled: ${JSON.stringify(cfg)}`);
+        if (!existsSync(join(dir, ".gitignore"))) throw new Error("template files missing");
+        const w = JSON.parse(readFileSync(join(dir, "wrangler.json"), "utf8"));
+        if (w.name !== "my-app-1" || w.r2_buckets[0].bucket_name !== "my-app-1-view") throw new Error(`wrangler.json not filled: ${JSON.stringify(w)}`);
+        if (readFileSync(join(dir, "README.md"), "utf8").includes("my-app")) throw new Error("README still names my-app");
+        let threw = false;
+        try { init("x", join(dir, "y"), { install: false, log: () => {} }); } catch { threw = true; }
+        if (!threw) throw new Error("a one-letter name must be refused");
+        console.log("init: ok");
+    } finally { rmSync(dirname(dir), { recursive: true, force: true }); }
+} else if (process.argv[2] === "init") {
+    const [app, dir] = process.argv.slice(3).filter((a) => !a.startsWith("--"));
+    try { init(app, dir, { install: !process.argv.includes("--no-install") }); }
+    catch (e) { console.error(e.message); process.exit(1); }
+} else if (process.argv[2] === "build") {
+    try { await buildForDeploy(process.argv[3] ?? "app.json"); }
+    catch (e) { console.error(e.message); process.exit(1); }
+} else {
+    console.error("usage: westmarch init <app> [dir] [--no-install] | westmarch build [app.json]");
+    process.exit(2);
+}

package/src/publish/lint.js

@@ -21,11 +21,12 @@
 import { configure, loadShard, resetShard, setDomains, trimView } from "../core/shard.js";
 import { shapeOf } from "../agent/ui.js";
 import { inSample, linkOf, rolesFrom, textOf } from "../core/roles.js";
+import { EMBED_MODEL } from "../core/embed.js";
 
 /** The model every corpus on this network is baked with. Cosine between two
  *  models' vectors is noise that still looks like a score, so a mismatch is not
  *  a warning — it is exclusion from the directory. */
-export const NETWORK_MODEL = "nomic-ai/nomic-embed-text-v1.5";
+export const NETWORK_MODEL = EMBED_MODEL;
 
 const F = (level, what, why, fix) => ({ level, what, why, fix });
 
@@ -346,7 +347,7 @@ export function format(r) {
 }
 
 // ── cli + self-check: `node src/publish/lint.js [viewUrl]` ─────────────────────
-if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/lint.js")) {
+if (import.meta.url === `file://${globalThis.process?.argv?.[1]}` && process.argv[1].endsWith("/lint.js")) {
     const arg = process.argv.slice(2).find((a) => !a.startsWith("--"));
     if (arg) {
         console.log(format(await lint(arg, { rows: !process.argv.includes("--no-rows"), links: process.argv.includes("--links") })));