@fangorn-network/westmarch 0.0.1-dev

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,343 @@
1
+ // A consumer's reactions, as a corpus anyone can buy.
2
+ //
3
+ // THE ARGUMENT THIS FILE EXISTS TO MAKE
4
+ // -------------------------------------
5
+ // Surveillance is not the only way to get behavioural data, and this claims it
6
+ // is not even the profitable way. Under surveillance the platform takes the
7
+ // reaction stream for free and the publisher whose catalogue produced it gets
8
+ // nothing — the film that made someone lean forward is an input the film's owner
9
+ // is not paid for. Here the same stream is a corpus with `lineage`, so when a
10
+ // trainer buys one agent's reactions the corpora those reactions were ABOUT are
11
+ // paid out of the sale. Publishers earn from the reaction economy rather than
12
+ // being its raw material. That is a better offer than surveillance makes them,
13
+ // and it is the whole of the wedge.
14
+ //
15
+ // WHY A CONSUMER CAN PUBLISH THIS WITH NO INFRASTRUCTURE
16
+ // ------------------------------------------------------
17
+ // A reaction's vector is the vector of the row reacted to, which the reader
18
+ // already downloaded in order to react to it. So a reaction corpus needs no
19
+ // model, no GPU, no bake, no quickbeam — the vectors are copied, not computed.
20
+ // The only thing that has to be fitted locally is `coverage`, which is eight
21
+ // centroids and is the function below. A consumer becoming a publisher is
22
+ // therefore not a tier you upgrade into; it is this file and a price.
23
+ //
24
+ // WHAT THE FREE INDEX DISCLOSES, AND WHAT IT DOES NOT
25
+ // ---------------------------------------------------
26
+ // Free: the COVERAGE CENTROIDS — where this reader's attention sits, and how
27
+ // densely — plus which corpora they were reading in, and the month.
28
+ // Paid: which item, what it was called, and whether it was a like or a skip.
29
+ //
30
+ // A reaction row ships NO VECTOR, and that is the whole design rather than an
31
+ // omission. The first version shipped one, on the reasoning that a buyer needs
32
+ // to rank before paying. Measured against the live `games` corpus, that handed
33
+ // over everything:
34
+ //
35
+ // 5 of 5 paid rows identified EXACTLY, at cosine 1.0000
36
+ //
37
+ // because a reaction's vector is a byte-for-byte copy of the row it reacted to,
38
+ // and the source corpora are free to download. Anyone could join the two and read
39
+ // the paid column for nothing. No amount of coarsening fixes a copy — the fix is
40
+ // to not publish it.
41
+ //
42
+ // Nothing is lost, because nobody buys row 47 of a stranger's reactions. A taste
43
+ // corpus is bought WHOLE, and the question a buyer actually asks — "does this
44
+ // reader have dense signal where I am training?" — is answered by the centroids
45
+ // alone. That is already how `consume/directory.js` ranks a corpus nobody has
46
+ // downloaded. Per-row vectors served no buyer and leaked the entire product.
47
+ //
48
+ // The centroids are themselves averages, so `k` is capped to keep at least
49
+ // MIN_PER_CENTROID reactions behind each one. A centroid over a single reaction
50
+ // IS that reaction's vector under another name, which is the same leak wearing a
51
+ // hat, and a reader with nine reactions is exactly who most needs this to hold.
52
+
53
+ import { packVec } from "../consume/embed.js";
54
+ import { lineage } from "./terms.js";
55
+
56
+ export const COVERAGE_K = 8;
57
+ export const COVERAGE_DIM = 128;
58
+ /** Fewest reactions that may stand behind one published centroid. */
59
+ export const MIN_PER_CENTROID = 8;
60
+
61
+ const l2 = (v) => { let n = 0; for (let i = 0; i < v.length; i++) n += v[i] * v[i]; return Math.sqrt(n) || 1; };
62
+ const unit = (v) => { const n = l2(v), o = new Float32Array(v.length); for (let i = 0; i < v.length; i++) o[i] = v[i] / n; return o; };
63
+ const dot = (a, b) => { let s = 0, n = Math.min(a.length, b.length); for (let i = 0; i < n; i++) s += a[i] * b[i]; return s; };
64
+
65
+ /**
66
+ * Spherical k-means over the corpus's own vectors — the routing summary that
67
+ * lets a buyer rank this corpus WITHOUT downloading it.
68
+ *
69
+ * Mirrors quickbeam's `cdn._coverage`: fit at full width, truncate to `dim`
70
+ * leading components (Matryoshka), then RENORMALIZE, because a truncated unit
71
+ * vector is not unit and the client compares against a truncated query it
72
+ * renormalizes too. Get that order wrong and routing degrades silently.
73
+ *
74
+ * ponytail: farthest-point init rather than k-means++, so it needs no RNG and
75
+ * two runs over the same rows produce byte-identical centroids — a publisher
76
+ * re-baking must not mint a new manifest that says something different. Swap in
77
+ * k-means++ if cluster quality ever measurably matters; for choosing between
78
+ * corpora, which sit far apart, it does not.
79
+ */
80
+ export function coverage(vectors, { k = COVERAGE_K, dim = COVERAGE_DIM, iters = 10 } = {}) {
81
+ const X = (vectors ?? []).filter((v) => v?.length).map(unit);
82
+ if (!X.length) return null;
83
+ k = Math.max(1, Math.min(k, X.length));
84
+
85
+ // Farthest-point init: start at the first row, then repeatedly take the row
86
+ // least similar to anything chosen so far.
87
+ const C = [X[0]];
88
+ const best = X.map((x) => dot(x, X[0]));
89
+ while (C.length < k) {
90
+ let pick = 0;
91
+ for (let i = 1; i < X.length; i++) if (best[i] < best[pick]) pick = i;
92
+ C.push(X[pick]);
93
+ for (let i = 0; i < X.length; i++) best[i] = Math.max(best[i], dot(X[i], X[pick]));
94
+ }
95
+
96
+ const width = X.reduce((n, v) => Math.max(n, v.length), 0);
97
+ let assign = new Int32Array(X.length);
98
+ for (let it = 0; it < iters; it++) {
99
+ let moved = 0;
100
+ for (let i = 0; i < X.length; i++) {
101
+ let bi = 0, bs = -Infinity;
102
+ for (let c = 0; c < C.length; c++) { const s = dot(X[i], C[c]); if (s > bs) { bs = s; bi = c; } }
103
+ if (assign[i] !== bi) { assign[i] = bi; moved++; }
104
+ }
105
+ const sums = C.map(() => new Float32Array(width));
106
+ const n = new Array(C.length).fill(0);
107
+ for (let i = 0; i < X.length; i++) {
108
+ const s = sums[assign[i]]; n[assign[i]]++;
109
+ for (let j = 0; j < X[i].length; j++) s[j] += X[i][j];
110
+ }
111
+ // An empty cluster keeps its old centroid rather than collapsing to zero.
112
+ for (let c = 0; c < C.length; c++) if (n[c]) C[c] = unit(sums[c]);
113
+ if (!moved) break;
114
+ }
115
+
116
+ const d = Math.min(dim, width);
117
+ const counts = new Array(C.length).fill(0);
118
+ for (const a of assign) counts[a]++;
119
+ return {
120
+ dim: d, sampled: X.length,
121
+ vectors: C.map((c) => [...unit(c.slice(0, d))].map((x) => Math.round(x * 1e4) / 1e4)),
122
+ counts,
123
+ };
124
+ }
125
+
126
+ /** The month a reaction happened in. Coarse ON PURPOSE — an exact timestamp is
127
+ * a join key against any other log, and the training value of "when" is at
128
+ * month resolution anyway. Nothing here needs to be precise enough to re-identify. */
129
+ const month = (at) => new Date(at ?? Date.now()).toISOString().slice(0, 7);
130
+
131
+ /** What the free index shows about a reaction, versus what a sale reveals. */
132
+ export const FREE = ["event", "corpus", "label", "month"];
133
+ export const LOCKED = ["reaction", "item", "title", "text"];
134
+
135
+ /**
136
+ * Reactions → a publishable corpus.
137
+ *
138
+ * `events` are `{ id, corpus, title, vector, reaction, at }` — exactly what a
139
+ * reader already has after ranking a row and acting on it.
140
+ *
141
+ * `sources` maps a corpus name to the address that gets paid when this sells.
142
+ * Upstream is paid out of the PUBLISHER's share (never the app's), split in
143
+ * proportion to how many of these reactions each corpus produced — a corpus that
144
+ * supplied four fifths of the signal earns four fifths of the upstream cut.
145
+ */
146
+ export function reactionCorpus(events = [], {
147
+ publisher, name = "reactions", description, sources = {},
148
+ price = "250000", asset = "USDC", upstreamBps = 2000, sample = 0.01,
149
+ } = {}) {
150
+ const rows = events.filter((e) => e?.vector?.length);
151
+ if (!rows.length) throw new Error("a reaction corpus needs at least one reaction with a vector");
152
+ if (!publisher) throw new Error("a reaction corpus needs a publisher address to pay");
153
+
154
+ const byCorpus = new Map();
155
+ for (const e of rows) byCorpus.set(e.corpus, (byCorpus.get(e.corpus) ?? 0) + 1);
156
+
157
+ // Upstream, proportional to contribution. Every share rounds down and the
158
+ // remainder goes to the largest contributor, so the declared total is exact
159
+ // rather than a basis point short — `terms.lineage` refuses anything over,
160
+ // and a buyer reconciling a receipt against the manifest must find them equal.
161
+ const cited = [...byCorpus].filter(([c]) => sources[c]).sort((a, b) => b[1] - a[1]);
162
+ const citedTotal = cited.reduce((s, [, n]) => s + n, 0);
163
+ const shares = cited.map(([c, n]) => ({ to: sources[c], bps: Math.floor((upstreamBps * n) / citedTotal), note: c }));
164
+ if (shares.length) shares[0].bps += upstreamBps - shares.reduce((s, e) => s + e.bps, 0);
165
+ const up = lineage(shares.filter((e) => e.bps > 0));
166
+
167
+ // The split is applied HERE, to the rows themselves, not merely declared in
168
+ // the manifest. Declaring it and shipping the fields anyway is the exact bug
169
+ // `consume/lint.js` calls "paywall names fields the free shard ships anyway",
170
+ // and this file had it: it returned every field on every row and left the
171
+ // withholding to a bake that does not run on this path.
172
+ const wire = [], locked = [];
173
+ rows.forEach((e, i) => {
174
+ const item = String(e.id ?? i);
175
+ // An ORDINAL, never the item. The id was `${name}:${item}`, which reads as
176
+ // an internal detail until you notice a corpus whose identity role IS its
177
+ // title (`games` is one), and then the free index is publishing the paid
178
+ // column inside its own primary key. Third time this shape appeared: the
179
+ // free thing was derived from the paid thing, and derivation is disclosure.
180
+ //
181
+ // ponytail: ordinals are stable while reactions only ever append, which is
182
+ // what a reaction log does. Deleting one renumbers the rest and breaks a
183
+ // buyer's citations — hash (name, item) if reactions ever become editable.
184
+ const id = `${name}:${i}`;
185
+ wire.push({
186
+ track_id: id,
187
+ owner: publisher,
188
+ fields: {
189
+ entityType: "reaction",
190
+ event: id,
191
+ corpus: String(e.corpus ?? "?"),
192
+ // The free title names the corpus, never the item. A reader
193
+ // scanning the index sees the shape of someone's attention
194
+ // without seeing one thing they paid attention to.
195
+ label: `a reaction in ${e.corpus ?? "?"}`,
196
+ month: month(e.at),
197
+ },
198
+ // No `v`. See the header — it was a copy of the source row's vector.
199
+ });
200
+ locked.push({
201
+ track_id: id,
202
+ fields: {
203
+ reaction: String(e.reaction ?? "like"),
204
+ item,
205
+ title: String(e.title ?? ""),
206
+ text: `${e.reaction ?? "like"} · ${e.title ?? ""}`.trim(),
207
+ v: packVec(e.vector), // the buyer gets full fidelity, having paid
208
+ },
209
+ });
210
+ });
211
+
212
+ const manifest = {
213
+ name,
214
+ description: description ?? `${rows.length} reactions from one reader across ${byCorpus.size} corpora — what held their attention, and what did not.`,
215
+ count: rows.length,
216
+ dim: rows[0].vector.length,
217
+ model: "nomic-ai/nomic-embed-text-v1.5",
218
+ distance: "Cosine",
219
+ role_map: {
220
+ identity: "event", title: "label", subtitle: "corpus", temporal: "month",
221
+ spatial: null, media: null, tags: ["corpus"], measures: [], relations: [],
222
+ text: ["text"],
223
+ },
224
+ entity_types: [{ type: "reaction", count: rows.length }],
225
+ presentation: { types: { reaction: { icon: "👍", accent: "#7a5cc9", singular: "Reaction", plural: "Reactions" } } },
226
+ paywall: {
227
+ free: FREE, locked: LOCKED, price: String(price), asset,
228
+ sample: { rate: sample, count: 0 }, // filled by the caller that writes the shard
229
+ },
230
+ coverage: coverage(rows.map((e) => e.vector),
231
+ { k: Math.max(1, Math.min(COVERAGE_K, Math.floor(rows.length / MIN_PER_CENTROID))) }),
232
+ shards: [],
233
+ };
234
+ return { manifest, rows: wire, locked, lineage: up, byCorpus: Object.fromEntries(byCorpus) };
235
+ }
236
+
237
+ // ── self-check: `node publish/reactions.js` ────────────────────────────────
238
+ if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}`) {
239
+ const { splitPayment } = await import("./terms.js");
240
+ const A = (n) => `0x${String(n).repeat(40).slice(0, 40)}`;
241
+ const vec = (a, b) => { const v = new Float32Array(256); v[0] = a; v[1] = b; return v; };
242
+
243
+ // ── coverage: the summary a buyer ranks without downloading anything ──
244
+ if (coverage([]) !== null) throw new Error("no vectors is no coverage, not an empty one");
245
+ const two = coverage([vec(1, 0), vec(1, 0.01), vec(0, 1), vec(0.01, 1)], { k: 2 });
246
+ if (two.vectors.length !== 2 || two.sampled !== 4) throw new Error("k centroids over n rows");
247
+ if (two.counts.reduce((s, c) => s + c, 0) !== 4) throw new Error("every row is counted exactly once");
248
+ if (two.dim !== 128 || two.vectors[0].length !== 128) throw new Error("centroids truncate to 128d like the bake");
249
+ for (const c of two.vectors) {
250
+ const n = Math.sqrt(c.reduce((s, x) => s + x * x, 0));
251
+ if (Math.abs(n - 1) > 0.01) throw new Error(`a truncated centroid must be RENORMALIZED, got |c| = ${n}`);
252
+ }
253
+ // The two clusters are found, not averaged into one blur.
254
+ const sims = two.vectors.map((c) => Math.abs(c[0]));
255
+ if (Math.max(...sims) < 0.9 || Math.min(...sims) > 0.1) throw new Error(`k-means must separate the two groups: ${sims}`);
256
+ // Deterministic: a re-publish must not mint a manifest that says something new.
257
+ if (JSON.stringify(coverage([vec(1, 0), vec(0, 1), vec(1, 0.2)], { k: 2 }))
258
+ !== JSON.stringify(coverage([vec(1, 0), vec(0, 1), vec(1, 0.2)], { k: 2 }))) throw new Error("two fits over the same rows must agree byte for byte");
259
+ if (coverage([vec(1, 0)], { k: 8 }).vectors.length !== 1) throw new Error("k is capped at the number of rows");
260
+
261
+ // ── the corpus ──
262
+ const EV = [
263
+ { id: "f1", corpus: "archive-films", title: "Nosferatu (1922)", vector: vec(1, 0), reaction: "like", at: "2026-08-14T10:00:00Z" },
264
+ { id: "f2", corpus: "archive-films", title: "Metropolis 1927", vector: vec(0.9, 0.1), reaction: "like", at: "2026-08-15T10:00:00Z" },
265
+ { id: "f3", corpus: "archive-films", title: "some sitcom", vector: vec(0, 1), reaction: "skip", at: "2026-08-16T10:00:00Z" },
266
+ { id: "g1", corpus: "games", title: "Return of the Obra Dinn", vector: vec(0.8, 0.2), reaction: "like", at: "2026-08-17T10:00:00Z" },
267
+ ];
268
+ const SRC = { "archive-films": A(1), games: A(2) };
269
+ const { manifest, rows, locked, lineage: up, byCorpus } = reactionCorpus(EV, { publisher: A(3), name: "taste-of-3", sources: SRC, price: "300000" });
270
+
271
+ if (rows.length !== 4 || manifest.count !== 4) throw new Error("every reaction with a vector becomes a row");
272
+ if (byCorpus["archive-films"] !== 3 || byCorpus.games !== 1) throw new Error("contribution is counted per corpus");
273
+
274
+ // The split the free index is allowed to make — APPLIED, not just declared.
275
+ for (const r of rows) {
276
+ for (const f of FREE) if (!(f in r.fields)) throw new Error(`the free index must carry ${f}`);
277
+ for (const f of LOCKED) if (f in r.fields) throw new Error(`the free index must NOT carry ${f} — declaring a split is not applying one`);
278
+ if (r.fields.month.length !== 7) throw new Error(`time is coarse on purpose, got ${r.fields.month}`);
279
+ if (/\d{2}T\d{2}/.test(r.fields.month)) throw new Error("an exact timestamp is a join key — never publish one");
280
+ }
281
+ if (LOCKED.some((f) => FREE.includes(f))) throw new Error("a field is on exactly one side");
282
+
283
+ // Nothing paid may appear ANYWHERE in a free row, including inside a key. The
284
+ // ids and titles below are the paid column; a substring hit is a leak however
285
+ // it got there.
286
+ for (const r of rows) {
287
+ const blob = JSON.stringify(r);
288
+ for (const e of EV) {
289
+ for (const secret of [e.id, e.title]) {
290
+ if (secret && blob.includes(secret)) throw new Error(`a free row leaks ${JSON.stringify(secret)}: ${blob}`);
291
+ }
292
+ }
293
+ }
294
+ if (rows.some((r) => r.v)) throw new Error("a reaction row must ship NO vector — it is a copy of the source row's, and the source corpora are free");
295
+ if (locked.length !== rows.length) throw new Error("every free row has exactly one paid counterpart");
296
+ if (!locked.every((r) => r.fields.v?.length)) throw new Error("the buyer, having paid, gets the true vector");
297
+ if (new Set(rows.map((r) => r.track_id)).size !== rows.length) throw new Error("ids must be unique or the two halves cannot be joined");
298
+
299
+ // THE JOIN ATTACK, which is what made the vectors indefensible: an attacker
300
+ // holding the (free) source corpus matches a published vector against it and
301
+ // reads the paid column for nothing. Measured at 5/5 exact on live `games`.
302
+ // With no vector published there is nothing to match, so the attack needs the
303
+ // one thing it was trying to avoid: buying the corpus.
304
+ const sourceVectors = EV.map((e) => e.vector);
305
+ for (const r of rows) {
306
+ const anyMatch = sourceVectors.some((sv) => JSON.stringify([...sv]) === JSON.stringify([...(r.vector ?? [])]));
307
+ if (anyMatch) throw new Error("a free row must not be joinable to the source corpus it came from");
308
+ }
309
+ // …and the centroids, which ARE published, never stand for one reaction.
310
+ if (manifest.coverage.vectors.length > Math.floor(EV.length / MIN_PER_CENTROID) && EV.length >= MIN_PER_CENTROID) {
311
+ throw new Error("a centroid over a single reaction is that reaction's vector with a hat on");
312
+ }
313
+ if (reactionCorpus(EV, { publisher: A(3), sources: {} }).manifest.coverage.vectors.length !== 1) {
314
+ throw new Error("4 reactions cannot support 8 centroids — collapse to one rather than publish each");
315
+ }
316
+
317
+ // Upstream is proportional and EXACT — 3 of 4 reactions came from the films.
318
+ const bps = Object.fromEntries(up.map((e) => [e.note, Number(e.bps)]));
319
+ if (bps["archive-films"] !== 1500 || bps.games !== 500) throw new Error(`upstream must follow contribution: ${JSON.stringify(bps)}`);
320
+ if (up.reduce((s, e) => s + Number(e.bps), 0) !== 2000) throw new Error("the declared upstream total must be exact, not a basis point short");
321
+
322
+ // …and it reconciles against a real sale. This is the claim: the corpora that
323
+ // produced the attention are paid when the attention sells.
324
+ const TERMS = { appId: "fangorn.tv", owner: A(9), appBps: 1000 };
325
+ const pay = splitPayment(300000n, { terms: TERMS, publisher: A(3), lineage: up });
326
+ const got = Object.fromEntries(pay.map((p) => [p.to, p.amount]));
327
+ if (got[A(9)] !== 30000n) throw new Error("the app's cut comes off the top and never moves");
328
+ if (got[A(1)] !== 40500n || got[A(2)] !== 13500n) throw new Error(`upstream is paid from the publisher's share: ${JSON.stringify(got, (k, v) => typeof v === "bigint" ? String(v) : v)}`);
329
+ if (got[A(3)] !== 216000n) throw new Error("the reader keeps the rest");
330
+ if (pay.reduce((s, p) => s + p.amount, 0n) !== 300000n) throw new Error("money is neither invented nor destroyed");
331
+
332
+ // A reader who cites a corpus nobody registered simply owes it nothing.
333
+ const orphan = reactionCorpus(EV, { publisher: A(3), sources: { games: A(2) } });
334
+ if (orphan.lineage.length !== 1 || Number(orphan.lineage[0].bps) !== 2000) throw new Error("an unregistered corpus cannot be paid, and its share does not vanish");
335
+
336
+ if (!reactionCorpus(EV, { publisher: A(3), sources: {} }).lineage.length === 0) throw new Error("citing nobody is allowed — the reader keeps it all");
337
+ try { reactionCorpus([], { publisher: A(3) }); throw new Error("x"); } catch (e) { if (!e.message.includes("at least one")) throw new Error("an empty corpus must be refused"); }
338
+ try { reactionCorpus(EV, {}); throw new Error("x"); } catch (e) { if (!e.message.includes("publisher")) throw new Error("a corpus with nobody to pay must be refused"); }
339
+
340
+ console.log("reactions.js self-check ok — reactions become a corpus with no model, coverage is deterministic and renormalized after truncation, "
341
+ + "the free index ranks on centroids alone, publishes no per-row vector to join against and no paid string anywhere including its own keys, "
342
+ + "time is coarse, upstream follows contribution exactly and reconciles against a real sale");
343
+ }
@@ -0,0 +1,165 @@
1
+ // Encrypting a file into the access worker, from Node.
2
+ //
3
+ // This is the publisher's byte path, and it is the half of publishing that no
4
+ // relay can do for you: the DEK is generated here, sealed to the worker's public
5
+ // key here, and the plaintext never leaves the machine that owns it. A relay
6
+ // that offered to do this step would be a relay that had your file.
7
+ //
8
+ // The browser has its own copy of this walk (an app's own encrypt.js, over
9
+ // File.slice instead of fs.read). They must agree byte for byte — they share
10
+ // `pack`, `aadFor` and `chunkKey` from envelope.js precisely so that the only
11
+ // thing that differs between them is how bytes are read off a disk.
12
+ import { open } from "node:fs/promises";
13
+ import { createHash } from "node:crypto";
14
+ import { seal } from "@fangorn-network/sdk";
15
+ import { bytesToHex } from "viem";
16
+ import {
17
+ CHUNK_SIZE, NONCE_LEN, aadFor, chunkKey, deleteResource,
18
+ getWorkerPubkey, pack, putChunk, unpack,
19
+ } from "./envelope.js";
20
+
21
+ export { CHUNK_SIZE, aadFor, chunkKey, deleteResource, getWorkerPubkey };
22
+
23
+ /**
24
+ * Envelope-encrypt the file at `file` chunk by chunk and upload each
25
+ * {ciphertext, sealed DEK} to the access worker's R2. Reads one chunk at a time,
26
+ * so peak memory is ~2x chunkSize no matter how big the file is.
27
+ *
28
+ * @returns { plaintextHash, chunks, size, chunkSize } — the hash to commit
29
+ * on-chain, the chunk count the buyer reassembles, and the plaintext
30
+ * byte geometry a streaming player needs to map a Range request onto a
31
+ * chunk (chunk i starts at i * chunkSize).
32
+ */
33
+ export async function encryptAndUpload({ file, resourceId, workerUrl, uploadToken, chunkSize = CHUNK_SIZE, onProgress }) {
34
+ const dek = crypto.getRandomValues(new Uint8Array(32));
35
+ const aesKey = await crypto.subtle.importKey("raw", dek, "AES-GCM", false, ["encrypt"]);
36
+ const sealedDek = bytesToHex(seal(dek, await getWorkerPubkey(workerUrl), resourceId));
37
+
38
+ const fh = await open(file, "r");
39
+ try {
40
+ const { size } = await fh.stat();
41
+ const chunks = Math.max(1, Math.ceil(size / chunkSize));
42
+ const hash = createHash("sha256");
43
+ const buf = Buffer.allocUnsafe(Math.min(size, chunkSize) || 1);
44
+
45
+ for (let i = 0; i < chunks; i++) {
46
+ const { bytesRead } = await fh.read(buf, 0, buf.length, i * chunkSize);
47
+ const plain = buf.subarray(0, bytesRead);
48
+ hash.update(plain);
49
+
50
+ const nonce = crypto.getRandomValues(new Uint8Array(NONCE_LEN));
51
+ const aesCt = new Uint8Array(
52
+ await crypto.subtle.encrypt({ name: "AES-GCM", iv: nonce, additionalData: aadFor(i) }, aesKey, await pack(plain)),
53
+ );
54
+ const body = new Uint8Array(NONCE_LEN + aesCt.length);
55
+ body.set(nonce, 0);
56
+ body.set(aesCt, NONCE_LEN);
57
+
58
+ await putChunk({ workerUrl, uploadToken, key: chunkKey(resourceId, i), body, sealedDek });
59
+ onProgress?.(i + 1, chunks);
60
+ }
61
+ return { plaintextHash: `0x${hash.digest("hex")}`, chunks, size, chunkSize };
62
+ } finally {
63
+ await fh.close();
64
+ }
65
+ }
66
+
67
+ // ── self-check: `node publish/settle.js` — no network, no worker ─────────────
68
+ //
69
+ // It decrypts with bare AES-GCM rather than importing a buyer, because what is
70
+ // under test here is the ENCRYPTOR: chunk addressing, the AAD index binding, the
71
+ // ragged tail, and that pack/unpack round trip. An app's full publisher→buyer
72
+ // round trip belongs in that app, against its own buyer.
73
+ if (import.meta.url === `file://${process.argv[1]}`) {
74
+ const { writeFileSync, rmSync } = await import("node:fs");
75
+ const { unseal } = await import("@fangorn-network/sdk");
76
+ const { x25519 } = await import("@noble/curves/ed25519");
77
+ const { hexToBytes, keccak256 } = await import("viem");
78
+
79
+ const CHUNK = 1000;
80
+ const tmp = `/tmp/westmarch-settle-${process.pid}.bin`;
81
+ const workerSecret = crypto.getRandomValues(new Uint8Array(32));
82
+ const workerPubkey = x25519.getPublicKey(workerSecret);
83
+ const r2 = new Map();
84
+ let sealedSeen = null;
85
+ const realFetch = globalThis.fetch;
86
+ globalThis.fetch = async (url, init = {}) => {
87
+ const u = String(url);
88
+ if (u.endsWith("/pubkey")) return { ok: true, json: async () => ({ pubkey: bytesToHex(workerPubkey) }) };
89
+ if (u.includes("/upload/")) {
90
+ const key = u.split("/upload/")[1];
91
+ if (init.method === "DELETE") return r2.delete(key), { ok: true };
92
+ sealedSeen = init.headers["X-Sealed-Dek"];
93
+ r2.set(key, new Uint8Array(init.body));
94
+ return { ok: true };
95
+ }
96
+ return { ok: false, status: 404 };
97
+ };
98
+
99
+ const rid = keccak256(new Uint8Array([1]));
100
+ /** Decrypt one stored chunk the way any buyer must. */
101
+ const read = async (i) => {
102
+ const dek = unseal(hexToBytes(sealedSeen), workerSecret, rid);
103
+ const key = await crypto.subtle.importKey("raw", dek, "AES-GCM", false, ["decrypt"]);
104
+ const body = r2.get(chunkKey(rid, i));
105
+ return unpack(new Uint8Array(await crypto.subtle.decrypt(
106
+ { name: "AES-GCM", iv: body.subarray(0, NONCE_LEN), additionalData: aadFor(i) },
107
+ key, body.subarray(NONCE_LEN),
108
+ )));
109
+ };
110
+ const same = (a, b) => a.length === b.length && a.every((x, i) => x === b[i]);
111
+
112
+ try {
113
+ // Random bytes: 2.5 chunks → a ragged tail, and gzip must LOSE, so these
114
+ // exercise pack()'s store-raw branch.
115
+ const bytes = crypto.getRandomValues(new Uint8Array(2500));
116
+ writeFileSync(tmp, bytes);
117
+ const out = await encryptAndUpload({ file: tmp, resourceId: rid, workerUrl: "http://stub", chunkSize: CHUNK });
118
+ if (out.chunks !== 3) throw new Error(`chunk count: got ${out.chunks}, want 3`);
119
+ if (out.plaintextHash !== `0x${createHash("sha256").update(bytes).digest("hex")}`) throw new Error("streamed hash != whole-file sha256");
120
+ if (!r2.has(rid)) throw new Error("chunk 0 must land on the resourceId — that is where /access looks");
121
+ if (r2.size !== 3) throw new Error(`chunk keys collide: ${r2.size} objects for 3 chunks`);
122
+ // nonce(12) + flag(1) + ct(500) + tag(16)
123
+ if (r2.get(chunkKey(rid, 2)).length !== 12 + 1 + 500 + 16) throw new Error("ragged last chunk is the wrong size");
124
+
125
+ const back = new Uint8Array([...await read(0), ...await read(1), ...await read(2)]);
126
+ if (!same(back, bytes)) throw new Error("round trip corrupted the file");
127
+
128
+ // AAD binds each chunk to its index: a chunk decrypted at the wrong index
129
+ // must FAIL, not silently hand back scrambled bytes.
130
+ let swallowed = false;
131
+ try {
132
+ const dek = unseal(hexToBytes(sealedSeen), workerSecret, rid);
133
+ const key = await crypto.subtle.importKey("raw", dek, "AES-GCM", false, ["decrypt"]);
134
+ const body = r2.get(chunkKey(rid, 0));
135
+ await crypto.subtle.decrypt({ name: "AES-GCM", iv: body.subarray(0, NONCE_LEN), additionalData: aadFor(1) }, key, body.subarray(NONCE_LEN));
136
+ swallowed = true;
137
+ } catch { /* expected */ }
138
+ if (swallowed) throw new Error("a chunk decrypted under the wrong index — AAD binding is broken");
139
+
140
+ // Compressible payload: pack()'s other branch. Stored bytes must come in
141
+ // well under the plaintext and still round trip.
142
+ r2.clear();
143
+ const text = new Uint8Array(3000).fill(65);
144
+ writeFileSync(tmp, text);
145
+ const packed = await encryptAndUpload({ file: tmp, resourceId: rid, workerUrl: "http://stub", chunkSize: CHUNK });
146
+ const stored = [...r2.values()].reduce((n, b) => n + b.length, 0);
147
+ if (stored > text.length / 2) throw new Error(`compressible payload barely shrank: ${stored} of ${text.length}`);
148
+ const gzBack = new Uint8Array([...await read(0), ...await read(1), ...await read(2)]);
149
+ if (!same(gzBack, text)) throw new Error("gzip round trip corrupted the file");
150
+
151
+ // Delete frees every chunk, and a worker that refuses RAISES rather than
152
+ // reporting freed space the operator is still billed for.
153
+ if (await deleteResource({ workerUrl: "http://stub", uploadToken: "t", resourceId: rid, chunks: packed.chunks }) !== packed.chunks) throw new Error("deleteResource miscounted");
154
+ if (r2.size !== 0) throw new Error(`delete left ${r2.size} chunk(s) behind`);
155
+ globalThis.fetch = async () => ({ ok: false, status: 401, statusText: "Unauthorized" });
156
+ let quiet = false;
157
+ try { await deleteResource({ workerUrl: "http://stub", resourceId: rid, chunks: 2 }); quiet = true; } catch { /* expected */ }
158
+ if (quiet) throw new Error("a refused delete was swallowed — the ledger would free bytes still in R2");
159
+
160
+ console.log(`settle.js self-check ok — round trip, hash, ragged tail, AAD index binding, gzip ${stored}/${text.length} bytes, delete`);
161
+ } finally {
162
+ globalThis.fetch = realFetch;
163
+ rmSync(tmp, { force: true });
164
+ }
165
+ }