@fangorn-network/westmarch 0.0.1-dev

package/publish/enrich.js

@@ -0,0 +1,211 @@
+// What a file is ABOUT, read from the file itself.
+//
+// fileText() (shard.js) is the honest floor: a name, its folders, and whatever
+// the publisher typed. For most libraries that's a filename and nothing else,
+// and a vector built from a filename ranks about as well as one — measured on
+// this storefront, "philosophy reading" put a metalcore mp3 above a
+// phenomenology PDF, and "something spacey" ranked milky-way.mp4 dead last.
+//
+// The file usually knows better than its name. A PDF carries its real title
+// ("Jakob_von_Uexkull_Beyond_Bubbles_On_Umwe.pdf" is truncated mid-word; the
+// document says "Beyond Bubbles: On Umwelt and Biophilosophy" — the two concept
+// words the filename lost). An mp3 carries artist, album and genre. Reading
+// them moved top-1 accuracy on an 11-query probe from 7/11 to 9/11, where
+// filename cleanup alone moved it 7/11 → 7/11.
+//
+// This runs at publish time on the publisher's own machine, over a file already
+// being read to encrypt and upload. Extraction is stdlib-only and every step is
+// best-effort: a malformed PDF, an exotic tag, an unreadable file all fall back
+// to the name, because a weaker vector is not a reason to fail a publish.
+//
+// ponytail: format sniffing by mime, no parser dependencies. PDF and ID3 cover
+// the text and music cases. Images and speech need a model, not a parser — see
+// the note at the bottom.
+
+import { readFileSync } from "node:fs";
+import { inflateSync } from "node:zlib";
+
+/** How much extracted content joins the text. The vector is 256-d and averaged
+ *  over the whole document, so a long tail of boilerplate dilutes the title it
+ *  was meant to reinforce. */
+export const CONTENT_CHARS = 600;
+
+/** Filename → words. Extensions, separators and release-note brackets are
+ *  tokens the model has to average over and none of them mean anything:
+ *  "Bilmuri - KINDA HARD [OFFICIAL VISUALIZER].mp3" → "Bilmuri KINDA HARD". */
+export function nameWords(s) {
+    return String(s)
+        .replace(/\.[a-z0-9]{1,5}$/i, "")          // extension
+        .replace(/\[[^\]]*\]|\([^)]*\)/g, " ")     // [OFFICIAL VISUALIZER], (ver 2 by …)
+        .replace(/[_\-.]+/g, " ")
+        .replace(/([a-z])([A-Z])/g, "$1 $2")       // camelCase
+        .replace(/\s+/g, " ").trim();
+}
+
+// The mime never reached the embedded text at all, which is why "movies"
+// matched nothing in a catalog full of video/mp4. Plain words, not mime
+// strings — the query side says "movie", never "video/mp4".
+//
+// Kept short on purpose: a long shared phrase on every video pulls all videos
+// toward each other and away from what makes each one different.
+const KIND = [
+    [/^audio\//, "music track"],
+    [/^video\//, "video, a movie you watch"],
+    [/^image\//, "picture, a photo you look at"],
+    [/pdf$/, "PDF document you read"],
+    [/^text\//, "text document"],
+];
+export const kindWords = (mime = "") => KIND.find(([re]) => re.test(String(mime)))?.[1] ?? "";
+
+/**
+ * PDF → its title and the top of its text.
+ *
+ * Enough of a reader to find words, not a PDF library: pull `/Title` from the
+ * info dictionary, then inflate the first few content streams and keep what's
+ * inside the string literals the text operators draw. Everything that isn't a
+ * flate stream is skipped silently, which is most of a PDF.
+ *
+ * ponytail: first streams, not first page — page order isn't stream order, so
+ * this can catch a cover page instead of an abstract. It still beat the
+ * filename by two ranks. Reach for a real parser when a measured miss says to.
+ */
+export function pdfText(buf, max = CONTENT_CHARS) {
+    const s = buf.toString("latin1");
+    const title = /\/Title\s*\(([^)]{0,200})\)/.exec(s)?.[1] ?? "";
+    let out = "", streams = 0;
+    for (const m of s.matchAll(/stream\r?\n/g)) {
+        if (streams >= 3 || out.length > max) break;
+        const start = m.index + m[0].length;
+        const end = s.indexOf("endstream", start);
+        if (end < 0) break;
+        try {
+            const page = inflateSync(buf.subarray(start, end)).toString("latin1");
+            // Text lives inside ( … ) literals; \) is an escaped paren, not a close.
+            const text = [...page.matchAll(/\((?:\\.|[^\\)])*\)/g)].map((x) => x[0].slice(1, -1)).join("");
+            if (text.length > 80) { out += `${text} `; streams++; }
+        } catch { /* not a flate stream — most of them aren't */ }
+    }
+    return `${title} ${out}`.replace(/\\\d{3}|\s+/g, " ").trim().slice(0, max);
+}
+
+// The four ID3v2 frames that say what a track is. TCON is genre, which is the
+// one a query like "something mellow" can actually reach.
+const ID3_FRAMES = { TIT2: 1, TPE1: 1, TALB: 1, TCON: 1 };
+
+/** mp3 → "title artist album genre" from its ID3v2 header, or "". */
+export function id3Text(buf) {
+    if (buf.length < 10 || buf.toString("latin1", 0, 3) !== "ID3") return "";
+    // Syncsafe integer: 7 bits per byte, so a size can never contain 0xFF and be
+    // mistaken for a frame sync. Masking is not optional.
+    const size = ((buf[6] & 127) << 21) | ((buf[7] & 127) << 14) | ((buf[8] & 127) << 7) | (buf[9] & 127);
+    const end = Math.min(10 + size, buf.length);
+    const out = [];
+    let o = 10;
+    while (o + 10 <= end) {
+        const id = buf.toString("latin1", o, o + 4);
+        if (!/^[A-Z0-9]{4}$/.test(id)) break; // padding, or v2.2 — either way, done
+        const len = buf.readUInt32BE(o + 4);
+        if (len <= 0 || o + 10 + len > end) break;
+        if (ID3_FRAMES[id]) {
+            // Byte 0 of a text frame is its encoding; utf8 decoding of latin1 and
+            // utf16 payloads is wrong in ways that are still more searchable than
+            // nothing, so strip the nulls and move on.
+            const v = buf.toString("utf8", o + 11, o + 10 + len).replace(/\0/g, " ").trim();
+            if (v) out.push(v);
+        }
+        o += 10 + len;
+    }
+    return out.join(" ").slice(0, CONTENT_CHARS);
+}
+
+/** Read whatever the bytes will tell us. Never throws — an unreadable or
+ *  unsupported file simply contributes nothing. */
+export function contentText(abs, mime = "") {
+    try {
+        if (/pdf$/.test(mime)) return pdfText(readFileSync(abs));
+        if (/^audio\/(mpeg|mp3)$/.test(mime)) return id3Text(readFileSync(abs));
+    } catch { /* missing, unreadable, or not what its extension claims */ }
+    return "";
+}
+
+/**
+ * The text a buyer's query has to match, for one file.
+ *
+ * Ordered strongest-signal-first — mean pooling doesn't care about position,
+ * but the truncation at CONTENT_CHARS does, and what gets cut should be the
+ * boilerplate at the end of a cover page, not the title.
+ *
+ * @param abs absolute path, or null to skip content extraction (the shard bake
+ *            reads from the chain and has no files).
+ */
+export function enrichText({ name, path, desc, mime, abs = null }) {
+    const folders = path.includes("/") ? path.slice(0, path.lastIndexOf("/")).split("/") : [];
+    return [
+        nameWords(name),
+        ...folders.map(nameWords),
+        desc,
+        kindWords(mime),
+        abs ? contentText(abs, mime) : "",
+    ].filter(Boolean).join(" — ");
+}
+
+// Images and dialogue-free video are the gap this can't close: there is nothing
+// in a JPEG's bytes that says "scifi western". That needs a captioning model,
+// which is a real dependency and a real runtime, not a parser — and it belongs
+// in the same browser that already runs the embedder at publish time.
+
+// ── self-check: `node server/enrich.js` — no files, no model ──────────────────
+if (import.meta.url === `file://${process.argv[1]}`) {
+    const { deflateSync } = await import("node:zlib");
+    const eq = (got, want, msg) => { if (got !== want) throw new Error(`${msg}: got ${JSON.stringify(got)}`); };
+
+    eq(nameWords("Bilmuri - KINDA HARD [OFFICIAL VISUALIZER].mp3"), "Bilmuri KINDA HARD", "release junk survived");
+    eq(nameWords("Jakob_von_Uexkull_Beyond_Bubbles.pdf"), "Jakob von Uexkull Beyond Bubbles", "underscores survived");
+    eq(nameWords("milky-way.mp4"), "milky way", "hyphens survived");
+    eq(nameWords("myGreatFile.mp4"), "my Great File", "camelCase not split");
+    // A name that is nothing but junk must not become empty and take the file
+    // out of the index entirely.
+    if (!nameWords("track (2).gp5").length) throw new Error("stripping must not empty a name");
+
+    // The bug that started this: mime never reached the text, so "movies" had
+    // nothing to match in a catalog full of video/mp4.
+    if (!kindWords("video/mp4").includes("movie")) throw new Error("video must read as a movie");
+    if (!kindWords("application/pdf").includes("read")) throw new Error("pdf must read as a document");
+    eq(kindWords("application/x-gp5"), "", "unknown mime must add nothing rather than guess");
+    eq(kindWords(), "", "missing mime must not throw");
+
+    // A minimal PDF: info dict + one flate content stream.
+    const body = deflateSync(Buffer.from("BT (Beyond Bubbles: On Umwelt and Biophilosophy, a review of the work of Jakob von Uexkull) Tj ET"));
+    const pdf = Buffer.concat([Buffer.from("%PDF-1.4\n/Title (REVIEW)\n5 0 obj\n<<>>\nstream\n", "latin1"), body, Buffer.from("\nendstream\n", "latin1")]);
+    const got = pdfText(pdf);
+    if (!got.includes("REVIEW")) throw new Error(`/Title lost: ${got}`);
+    if (!got.includes("Umwelt")) throw new Error(`stream text lost: ${got}`);
+    if (pdfText(Buffer.from("not a pdf at all")) !== "") throw new Error("garbage must extract to nothing, not throw");
+    if (pdfText(Buffer.from("%PDF-1.4 stream\nnot deflate data endstream")) !== "") throw new Error("undeflatable stream must be skipped");
+    if (pdfText(pdf, 10).length > 10) throw new Error("max not honoured");
+
+    // ID3v2: syncsafe size, four frames, one ignored.
+    const frame = (id, text) => { const v = Buffer.from(`\0${text}`, "latin1"); const h = Buffer.alloc(10); h.write(id, 0, "latin1"); h.writeUInt32BE(v.length, 4); return Buffer.concat([h, v]); };
+    const frames = Buffer.concat([frame("TIT2", "Kinda Hard"), frame("TPE1", "Bilmuri"), frame("TCON", "Metalcore"), frame("TENC", "ignored")]);
+    const head = Buffer.alloc(10); head.write("ID3", 0, "latin1"); head[3] = 3;
+    // Syncsafe: 200 is > 127, so it must occupy two 7-bit groups.
+    const n = frames.length; head[6] = (n >> 21) & 127; head[7] = (n >> 14) & 127; head[8] = (n >> 7) & 127; head[9] = n & 127;
+    const tag = id3Text(Buffer.concat([head, frames]));
+    eq(tag, "Kinda Hard Bilmuri Metalcore", "id3 frames wrong");
+    if (id3Text(Buffer.from("no tag here")) !== "") throw new Error("untagged mp3 must yield nothing, not throw");
+    if (id3Text(Buffer.alloc(3)) !== "") throw new Error("truncated file must not read past its end");
+    // A frame claiming to run past the tag must stop the walk, not read garbage.
+    const bad = Buffer.concat([head, frame("TIT2", "x")]); bad.writeUInt32BE(9999, 14);
+    if (id3Text(bad) !== "") throw new Error("overlong frame must be rejected");
+
+    // enrichText: order and graceful degradation.
+    const t = enrichText({ name: "milky-way.mp4", path: "videos/milky-way.mp4", mime: "video/mp4" });
+    if (!t.startsWith("milky way")) throw new Error(`name must lead: ${t}`);
+    if (!t.includes("videos") || !t.includes("movie")) throw new Error(`folder and kind missing: ${t}`);
+    // No abs path (the shard bake has no files) must still produce usable text.
+    if (enrichText({ name: "a.mp3", path: "a.mp3", mime: "audio/mpeg" }) !== "a — music track") throw new Error("bare file text wrong");
+    // A file that vanished between prepare and embed must not fail the publish.
+    if (!enrichText({ name: "gone.pdf", path: "x/gone.pdf", mime: "application/pdf", abs: "/nope/gone.pdf" }).includes("gone")) throw new Error("missing file must degrade to the name");
+
+    console.log("enrich.js self-check ok — name hygiene, kind words, pdf title+text, id3 frames, degradation");
+}

package/publish/envelope.js

@@ -0,0 +1,185 @@
+// The on-the-wire envelope format, shared by everything that writes it.
+//
+// `ns` — the app's namespace string — is the first argument of the two id
+// derivations and it is NOT defaulted. It is folded into every resourceId ever
+// minted, so it is permanent: changing an app's ns orphans everything that app
+// already published, and two apps that accidentally share one mint into each
+// other's id space. An app pins it once (sond3r: src/pay/envelope.js) and never
+// again. A default here would be a way to pick it by accident.
+//
+// Three implementations used to need these constants to agree — server/settle.js
+// (relay-side publish), src/pay/encrypt.js (browser-side publish) and src/pay/stream.js /
+// src/pay/buy.js (the buyer, unpacking). They agree because they import from here.
+// A mismatch is not a crash: chunk `i` decrypts to garbage, or a key lands where
+// the worker's /access gate isn't looking, and the file goes dark for everyone
+// who paid for it.
+//
+// Browser-safe on purpose: viem only, no node builtins.
+import { concat, encodePacked, hexToBytes, keccak256, numberToHex, stringToBytes } from "viem";
+
+/** The bytes32 the publisher hands createResource. Stable per manifest uid. */
+export const uidHashFor = (ns, uid) => keccak256(stringToBytes(`${ns}:${uid}`));
+
+/** What that uid becomes on-chain: keccak(publisher ++ uid), derived by the
+ *  registry (`resource_id_of`) and reproduced here. Two publishers with the same
+ *  uid get two different resources, which is what makes an id impossible to
+ *  squat — and it means this derivation must match the contract exactly, or a
+ *  publish uploads ciphertext to a key no on-chain resource points at.
+ *
+ *  It lives here rather than in server/index.js because the BROWSER derives it
+ *  too now (it names the R2 key it uploads ciphertext to) and the server derives
+ *  it when minting the resource. Two copies that disagreed would put the bytes
+ *  and the resource at different ids, and the file would be unbuyable. Pinned
+ *  against the registry's own view in sond3r's server/index.js --selfcheck. */
+export const resourceIdFor = (ns, owner, uid) => keccak256(concat([owner, uidHashFor(ns, uid)]));
+
+export const NONCE_LEN = 12;
+
+/** 64MiB — under Cloudflare's 100MB body cap with room for nonce/tag/headers. */
+export const CHUNK_SIZE = 64 * 1024 * 1024;
+
+/** R2 key for chunk `i`. Chunk 0 is the resourceId itself, so the sealed DEK
+ *  lands exactly where the worker's /access gate looks for it and a small file
+ *  is still a single object. Every buyer-side unpacker must match this. */
+export const chunkKey = (resourceId, i) =>
+    i === 0 ? resourceId : keccak256(encodePacked(["bytes32", "uint32"], [resourceId, i]));
+
+/** AES-GCM additional data: the chunk index, big-endian. MUST match src/pay/buy.js. */
+export const aadFor = (i) => hexToBytes(numberToHex(i, { size: 4 }));
+
+/**
+ * Compress-then-encrypt: what gets sealed is `flag(1) || bytes`, flag 1 = gzip.
+ * ponytail: gzip only when it actually shrinks. mp4/webm/jpeg are already
+ * compressed and gzip *grows* them, so the flag makes the store-raw case the
+ * fallback instead of a 0.03% tax. MUST match unpack() in src/pay/stream.js.
+ */
+export async function pack(plain) {
+    const gz = new Uint8Array(await new Response(
+        new Response(plain).body.pipeThrough(new CompressionStream("gzip")),
+    ).arrayBuffer());
+    const win = gz.length < plain.length;
+    const out = new Uint8Array((win ? gz.length : plain.length) + 1);
+    out[0] = win ? 1 : 0;
+    out.set(win ? gz : plain, 1);
+    return out;
+}
+
+/**
+ * The inverse of pack(). It lives here, beside its inverse, because a packer and
+ * an unpacker in two repos is a format with two definitions — and the failure is
+ * not a crash but a file that uploads fine and is undecryptable forever.
+ */
+export async function unpack(bytes) {
+    if (bytes[0] === 0) return bytes.subarray(1);
+    // Not 0/1 means these bytes aren't packed at all — almost always ciphertext
+    // uploaded before the compression format landed. The publisher re-publishes;
+    // there is nothing the buyer can do.
+    if (bytes[0] !== 1) throw new Error(`unknown chunk encoding ${bytes[0]} — this file needs re-publishing`);
+    return new Uint8Array(await new Response(
+        new Response(bytes.subarray(1)).body.pipeThrough(new DecompressionStream("gzip")),
+    ).arrayBuffer());
+}
+
+/** GET the worker's static X25519 pubkey (what DEKs are sealed to). */
+export async function getWorkerPubkey(workerUrl) {
+    const res = await fetch(`${workerUrl}/pubkey`);
+    if (!res.ok) throw new Error(`/pubkey failed: ${res.status}`);
+    const { pubkey } = await res.json();
+    return hexToBytes(pubkey);
+}
+
+/**
+ * POST one already-encrypted chunk to the access worker's R2.
+ *
+ * `sealedDek` is omitted for objects that are the publisher's own state rather
+ * than something a buyer will pay for — the worker then writes no DEK object, so
+ * /access has nothing to release. See handleUpload in the worker.
+ */
+export async function putChunk({ workerUrl, uploadToken, key, body, sealedDek, immutable = false }) {
+    const res = await fetch(`${workerUrl}/upload/${key}`, {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/octet-stream",
+            ...(sealedDek ? { "X-Sealed-Dek": sealedDek } : {}),
+            // Only for a key that IS its content's digest. A resourceId is derived
+            // from (publisher, uid), so a re-publish rewrites it with new bytes
+            // under a new DEK, and an edge still serving the old ones hands every
+            // buyer ciphertext that decrypts to garbage. A shard named by its own
+            // sha256 cannot have that problem, and is the reason this flag exists.
+            ...(immutable ? { "X-Immutable": "1" } : {}),
+            // The worker's upload gate. On a fresh publisher-owned worker the
+            // first request carrying this claims the bucket to it.
+            ...(uploadToken ? { Authorization: `Bearer ${uploadToken}` } : {}),
+        },
+        body,
+    });
+    if (!res.ok) {
+        const text = await res.text();
+        // A quota refusal is the one upload failure a publisher can act on, so its
+        // message is passed through instead of buried in a URL and a status code.
+        const reason = (() => { try { return JSON.parse(text).error; } catch { return null; } })();
+        throw new Error(res.status === 413 && reason ? reason : `${workerUrl}/upload/${key}: ${res.status} ${text}`);
+    }
+    return res;
+}
+
+/**
+ * Delete every chunk of a published resource, and its sealed DEK, from the
+ * worker. Returns the number of chunks it asked the worker to drop.
+ *
+ * The chunk count comes from the caller's manifest rather than from the worker,
+ * because the worker deliberately won't synthesize keys (see handleDelete in
+ * webworker/fangorn-access-worker) — the key derivation lives here, next to the
+ * upload that used it.
+ *
+ * Failures are RAISED, not swallowed. A delete that half-worked must not be
+ * reported as freed space: the operator is still paying for whatever is left,
+ * and silently crediting the publisher for it is how the storage ledger drifts
+ * away from the R2 bill.
+ */
+export async function deleteResource({ workerUrl, uploadToken, resourceId, chunks = 1 }) {
+    for (let i = 0; i < Math.max(1, chunks); i++) {
+        const res = await fetch(`${workerUrl}/upload/${chunkKey(resourceId, i)}`, {
+            method: "DELETE",
+            headers: uploadToken ? { Authorization: `Bearer ${uploadToken}` } : {},
+        });
+        // 404 is a worker too old to have the route; that's a real failure, since
+        // nothing was deleted and the caller is about to free the allowance.
+        if (!res.ok) throw new Error(`${workerUrl} refused to delete chunk ${i} of ${resourceId}: ${res.status} ${res.statusText}`);
+    }
+    return Math.max(1, chunks);
+}
+
+// ── self-check: `node publish/envelope.js` ───────────────────────────────────
+//
+// These vectors are PINNED to the values sond3r's own envelope.js produced before
+// this file was extracted from it, with ns = "sond3r". They are not a style test:
+// every resourceId ever minted by fangorn.tv is derived by this function, and a
+// change here that shifts one byte makes ciphertext already in R2 unreachable from
+// the resource anyone paid for. If this fails, the extraction broke money.
+if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}`) {
+    const owner = `0x${"11".repeat(20)}`;
+    const uid = "Show/S1/a.mp4";
+
+    const h = uidHashFor("sond3r", uid);
+    if (h !== "0x855b20df257479ad7d5d5be0bcddc8c8554bff19edb33adb818872dd3d9e0d04") {
+        throw new Error(`uidHashFor drifted: ${h}`);
+    }
+    const r = resourceIdFor("sond3r", owner, uid);
+    if (r !== "0x0f919abdf691fae94b36395e67f7ed3301f0faea6027ffe609612656556a28a5") {
+        throw new Error(`resourceIdFor drifted: ${r}`);
+    }
+
+    // The namespace is what separates two apps' id spaces. Same owner, same uid,
+    // different app → different resource, or one app could mint over another's.
+    if (resourceIdFor("wiki", owner, uid) === r) throw new Error("ns does not reach the resourceId");
+    if (uidHashFor("wiki", uid) === h) throw new Error("ns does not reach the uid hash");
+
+    // Chunk 0 IS the resourceId — the worker's /access gate looks for the sealed
+    // DEK there and nowhere else.
+    if (chunkKey(r, 0) !== r) throw new Error("chunk 0 must be the resourceId itself");
+    if (chunkKey(r, 1) === r) throw new Error("chunk 1 collided with chunk 0");
+    if (aadFor(1).length !== 4) throw new Error("aad must be 4 bytes, big-endian");
+
+    console.log("envelope.js self-check ok — pinned sond3r ids, ns separation, chunk keys");
+}