@fangorn-network/westmarch 0.0.1-dev
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +20 -0
- package/README.md +280 -0
- package/consume/apps.js +111 -0
- package/consume/corpora.js +297 -0
- package/consume/directory.js +262 -0
- package/consume/embed.js +251 -0
- package/consume/embed.worker.js +26 -0
- package/consume/lint.js +500 -0
- package/consume/mcp.js +381 -0
- package/consume/rank.js +218 -0
- package/consume/roles.js +556 -0
- package/consume/shard.js +726 -0
- package/consume/steam.js +341 -0
- package/consume/taste-doc.js +342 -0
- package/consume/taste.js +350 -0
- package/consume/tools.js +358 -0
- package/consume/ui.js +724 -0
- package/docs/app-to-agent.md +351 -0
- package/package.json +70 -0
- package/publish/agent-card.js +134 -0
- package/publish/cli.js +336 -0
- package/publish/cohort.js +349 -0
- package/publish/demand.js +537 -0
- package/publish/enrich.js +211 -0
- package/publish/envelope.js +185 -0
- package/publish/graph.js +307 -0
- package/publish/manifest.js +142 -0
- package/publish/reactions.js +343 -0
- package/publish/settle.js +165 -0
- package/publish/terms.js +200 -0
- package/publish/vault.js +162 -0
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
// A consumer's reactions, as a corpus anyone can buy.
|
|
2
|
+
//
|
|
3
|
+
// THE ARGUMENT THIS FILE EXISTS TO MAKE
|
|
4
|
+
// -------------------------------------
|
|
5
|
+
// Surveillance is not the only way to get behavioural data, and this claims it
|
|
6
|
+
// is not even the profitable way. Under surveillance the platform takes the
|
|
7
|
+
// reaction stream for free and the publisher whose catalogue produced it gets
|
|
8
|
+
// nothing — the film that made someone lean forward is an input the film's owner
|
|
9
|
+
// is not paid for. Here the same stream is a corpus with `lineage`, so when a
|
|
10
|
+
// trainer buys one agent's reactions the corpora those reactions were ABOUT are
|
|
11
|
+
// paid out of the sale. Publishers earn from the reaction economy rather than
|
|
12
|
+
// being its raw material. That is a better offer than surveillance makes them,
|
|
13
|
+
// and it is the whole of the wedge.
|
|
14
|
+
//
|
|
15
|
+
// WHY A CONSUMER CAN PUBLISH THIS WITH NO INFRASTRUCTURE
|
|
16
|
+
// ------------------------------------------------------
|
|
17
|
+
// A reaction's vector is the vector of the row reacted to, which the reader
|
|
18
|
+
// already downloaded in order to react to it. So a reaction corpus needs no
|
|
19
|
+
// model, no GPU, no bake, no quickbeam — the vectors are copied, not computed.
|
|
20
|
+
// The only thing that has to be fitted locally is `coverage`, which is eight
|
|
21
|
+
// centroids and is the function below. A consumer becoming a publisher is
|
|
22
|
+
// therefore not a tier you upgrade into; it is this file and a price.
|
|
23
|
+
//
|
|
24
|
+
// WHAT THE FREE INDEX DISCLOSES, AND WHAT IT DOES NOT
|
|
25
|
+
// ---------------------------------------------------
|
|
26
|
+
// Free: the COVERAGE CENTROIDS — where this reader's attention sits, and how
|
|
27
|
+
// densely — plus which corpora they were reading in, and the month.
|
|
28
|
+
// Paid: which item, what it was called, and whether it was a like or a skip.
|
|
29
|
+
//
|
|
30
|
+
// A reaction row ships NO VECTOR, and that is the whole design rather than an
|
|
31
|
+
// omission. The first version shipped one, on the reasoning that a buyer needs
|
|
32
|
+
// to rank before paying. Measured against the live `games` corpus, that handed
|
|
33
|
+
// over everything:
|
|
34
|
+
//
|
|
35
|
+
// 5 of 5 paid rows identified EXACTLY, at cosine 1.0000
|
|
36
|
+
//
|
|
37
|
+
// because a reaction's vector is a byte-for-byte copy of the row it reacted to,
|
|
38
|
+
// and the source corpora are free to download. Anyone could join the two and read
|
|
39
|
+
// the paid column for nothing. No amount of coarsening fixes a copy — the fix is
|
|
40
|
+
// to not publish it.
|
|
41
|
+
//
|
|
42
|
+
// Nothing is lost, because nobody buys row 47 of a stranger's reactions. A taste
|
|
43
|
+
// corpus is bought WHOLE, and the question a buyer actually asks — "does this
|
|
44
|
+
// reader have dense signal where I am training?" — is answered by the centroids
|
|
45
|
+
// alone. That is already how `consume/directory.js` ranks a corpus nobody has
|
|
46
|
+
// downloaded. Per-row vectors served no buyer and leaked the entire product.
|
|
47
|
+
//
|
|
48
|
+
// The centroids are themselves averages, so `k` is capped to keep at least
|
|
49
|
+
// MIN_PER_CENTROID reactions behind each one. A centroid over a single reaction
|
|
50
|
+
// IS that reaction's vector under another name, which is the same leak wearing a
|
|
51
|
+
// hat, and a reader with nine reactions is exactly who most needs this to hold.
|
|
52
|
+
|
|
53
|
+
import { packVec } from "../consume/embed.js";
|
|
54
|
+
import { lineage } from "./terms.js";
|
|
55
|
+
|
|
56
|
+
export const COVERAGE_K = 8;
|
|
57
|
+
export const COVERAGE_DIM = 128;
|
|
58
|
+
/** Fewest reactions that may stand behind one published centroid. */
|
|
59
|
+
export const MIN_PER_CENTROID = 8;
|
|
60
|
+
|
|
61
|
+
const l2 = (v) => { let n = 0; for (let i = 0; i < v.length; i++) n += v[i] * v[i]; return Math.sqrt(n) || 1; };
|
|
62
|
+
const unit = (v) => { const n = l2(v), o = new Float32Array(v.length); for (let i = 0; i < v.length; i++) o[i] = v[i] / n; return o; };
|
|
63
|
+
const dot = (a, b) => { let s = 0, n = Math.min(a.length, b.length); for (let i = 0; i < n; i++) s += a[i] * b[i]; return s; };
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Spherical k-means over the corpus's own vectors — the routing summary that
|
|
67
|
+
* lets a buyer rank this corpus WITHOUT downloading it.
|
|
68
|
+
*
|
|
69
|
+
* Mirrors quickbeam's `cdn._coverage`: fit at full width, truncate to `dim`
|
|
70
|
+
* leading components (Matryoshka), then RENORMALIZE, because a truncated unit
|
|
71
|
+
* vector is not unit and the client compares against a truncated query it
|
|
72
|
+
* renormalizes too. Get that order wrong and routing degrades silently.
|
|
73
|
+
*
|
|
74
|
+
* ponytail: farthest-point init rather than k-means++, so it needs no RNG and
|
|
75
|
+
* two runs over the same rows produce byte-identical centroids — a publisher
|
|
76
|
+
* re-baking must not mint a new manifest that says something different. Swap in
|
|
77
|
+
* k-means++ if cluster quality ever measurably matters; for choosing between
|
|
78
|
+
* corpora, which sit far apart, it does not.
|
|
79
|
+
*/
|
|
80
|
+
export function coverage(vectors, { k = COVERAGE_K, dim = COVERAGE_DIM, iters = 10 } = {}) {
|
|
81
|
+
const X = (vectors ?? []).filter((v) => v?.length).map(unit);
|
|
82
|
+
if (!X.length) return null;
|
|
83
|
+
k = Math.max(1, Math.min(k, X.length));
|
|
84
|
+
|
|
85
|
+
// Farthest-point init: start at the first row, then repeatedly take the row
|
|
86
|
+
// least similar to anything chosen so far.
|
|
87
|
+
const C = [X[0]];
|
|
88
|
+
const best = X.map((x) => dot(x, X[0]));
|
|
89
|
+
while (C.length < k) {
|
|
90
|
+
let pick = 0;
|
|
91
|
+
for (let i = 1; i < X.length; i++) if (best[i] < best[pick]) pick = i;
|
|
92
|
+
C.push(X[pick]);
|
|
93
|
+
for (let i = 0; i < X.length; i++) best[i] = Math.max(best[i], dot(X[i], X[pick]));
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const width = X.reduce((n, v) => Math.max(n, v.length), 0);
|
|
97
|
+
let assign = new Int32Array(X.length);
|
|
98
|
+
for (let it = 0; it < iters; it++) {
|
|
99
|
+
let moved = 0;
|
|
100
|
+
for (let i = 0; i < X.length; i++) {
|
|
101
|
+
let bi = 0, bs = -Infinity;
|
|
102
|
+
for (let c = 0; c < C.length; c++) { const s = dot(X[i], C[c]); if (s > bs) { bs = s; bi = c; } }
|
|
103
|
+
if (assign[i] !== bi) { assign[i] = bi; moved++; }
|
|
104
|
+
}
|
|
105
|
+
const sums = C.map(() => new Float32Array(width));
|
|
106
|
+
const n = new Array(C.length).fill(0);
|
|
107
|
+
for (let i = 0; i < X.length; i++) {
|
|
108
|
+
const s = sums[assign[i]]; n[assign[i]]++;
|
|
109
|
+
for (let j = 0; j < X[i].length; j++) s[j] += X[i][j];
|
|
110
|
+
}
|
|
111
|
+
// An empty cluster keeps its old centroid rather than collapsing to zero.
|
|
112
|
+
for (let c = 0; c < C.length; c++) if (n[c]) C[c] = unit(sums[c]);
|
|
113
|
+
if (!moved) break;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const d = Math.min(dim, width);
|
|
117
|
+
const counts = new Array(C.length).fill(0);
|
|
118
|
+
for (const a of assign) counts[a]++;
|
|
119
|
+
return {
|
|
120
|
+
dim: d, sampled: X.length,
|
|
121
|
+
vectors: C.map((c) => [...unit(c.slice(0, d))].map((x) => Math.round(x * 1e4) / 1e4)),
|
|
122
|
+
counts,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** The month a reaction happened in. Coarse ON PURPOSE — an exact timestamp is
|
|
127
|
+
* a join key against any other log, and the training value of "when" is at
|
|
128
|
+
* month resolution anyway. Nothing here needs to be precise enough to re-identify. */
|
|
129
|
+
const month = (at) => new Date(at ?? Date.now()).toISOString().slice(0, 7);
|
|
130
|
+
|
|
131
|
+
/** What the free index shows about a reaction, versus what a sale reveals. */
|
|
132
|
+
export const FREE = ["event", "corpus", "label", "month"];
|
|
133
|
+
export const LOCKED = ["reaction", "item", "title", "text"];
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Reactions → a publishable corpus.
|
|
137
|
+
*
|
|
138
|
+
* `events` are `{ id, corpus, title, vector, reaction, at }` — exactly what a
|
|
139
|
+
* reader already has after ranking a row and acting on it.
|
|
140
|
+
*
|
|
141
|
+
* `sources` maps a corpus name to the address that gets paid when this sells.
|
|
142
|
+
* Upstream is paid out of the PUBLISHER's share (never the app's), split in
|
|
143
|
+
* proportion to how many of these reactions each corpus produced — a corpus that
|
|
144
|
+
* supplied four fifths of the signal earns four fifths of the upstream cut.
|
|
145
|
+
*/
|
|
146
|
+
export function reactionCorpus(events = [], {
|
|
147
|
+
publisher, name = "reactions", description, sources = {},
|
|
148
|
+
price = "250000", asset = "USDC", upstreamBps = 2000, sample = 0.01,
|
|
149
|
+
} = {}) {
|
|
150
|
+
const rows = events.filter((e) => e?.vector?.length);
|
|
151
|
+
if (!rows.length) throw new Error("a reaction corpus needs at least one reaction with a vector");
|
|
152
|
+
if (!publisher) throw new Error("a reaction corpus needs a publisher address to pay");
|
|
153
|
+
|
|
154
|
+
const byCorpus = new Map();
|
|
155
|
+
for (const e of rows) byCorpus.set(e.corpus, (byCorpus.get(e.corpus) ?? 0) + 1);
|
|
156
|
+
|
|
157
|
+
// Upstream, proportional to contribution. Every share rounds down and the
|
|
158
|
+
// remainder goes to the largest contributor, so the declared total is exact
|
|
159
|
+
// rather than a basis point short — `terms.lineage` refuses anything over,
|
|
160
|
+
// and a buyer reconciling a receipt against the manifest must find them equal.
|
|
161
|
+
const cited = [...byCorpus].filter(([c]) => sources[c]).sort((a, b) => b[1] - a[1]);
|
|
162
|
+
const citedTotal = cited.reduce((s, [, n]) => s + n, 0);
|
|
163
|
+
const shares = cited.map(([c, n]) => ({ to: sources[c], bps: Math.floor((upstreamBps * n) / citedTotal), note: c }));
|
|
164
|
+
if (shares.length) shares[0].bps += upstreamBps - shares.reduce((s, e) => s + e.bps, 0);
|
|
165
|
+
const up = lineage(shares.filter((e) => e.bps > 0));
|
|
166
|
+
|
|
167
|
+
// The split is applied HERE, to the rows themselves, not merely declared in
|
|
168
|
+
// the manifest. Declaring it and shipping the fields anyway is the exact bug
|
|
169
|
+
// `consume/lint.js` calls "paywall names fields the free shard ships anyway",
|
|
170
|
+
// and this file had it: it returned every field on every row and left the
|
|
171
|
+
// withholding to a bake that does not run on this path.
|
|
172
|
+
const wire = [], locked = [];
|
|
173
|
+
rows.forEach((e, i) => {
|
|
174
|
+
const item = String(e.id ?? i);
|
|
175
|
+
// An ORDINAL, never the item. The id was `${name}:${item}`, which reads as
|
|
176
|
+
// an internal detail until you notice a corpus whose identity role IS its
|
|
177
|
+
// title (`games` is one), and then the free index is publishing the paid
|
|
178
|
+
// column inside its own primary key. Third time this shape appeared: the
|
|
179
|
+
// free thing was derived from the paid thing, and derivation is disclosure.
|
|
180
|
+
//
|
|
181
|
+
// ponytail: ordinals are stable while reactions only ever append, which is
|
|
182
|
+
// what a reaction log does. Deleting one renumbers the rest and breaks a
|
|
183
|
+
// buyer's citations — hash (name, item) if reactions ever become editable.
|
|
184
|
+
const id = `${name}:${i}`;
|
|
185
|
+
wire.push({
|
|
186
|
+
track_id: id,
|
|
187
|
+
owner: publisher,
|
|
188
|
+
fields: {
|
|
189
|
+
entityType: "reaction",
|
|
190
|
+
event: id,
|
|
191
|
+
corpus: String(e.corpus ?? "?"),
|
|
192
|
+
// The free title names the corpus, never the item. A reader
|
|
193
|
+
// scanning the index sees the shape of someone's attention
|
|
194
|
+
// without seeing one thing they paid attention to.
|
|
195
|
+
label: `a reaction in ${e.corpus ?? "?"}`,
|
|
196
|
+
month: month(e.at),
|
|
197
|
+
},
|
|
198
|
+
// No `v`. See the header — it was a copy of the source row's vector.
|
|
199
|
+
});
|
|
200
|
+
locked.push({
|
|
201
|
+
track_id: id,
|
|
202
|
+
fields: {
|
|
203
|
+
reaction: String(e.reaction ?? "like"),
|
|
204
|
+
item,
|
|
205
|
+
title: String(e.title ?? ""),
|
|
206
|
+
text: `${e.reaction ?? "like"} · ${e.title ?? ""}`.trim(),
|
|
207
|
+
v: packVec(e.vector), // the buyer gets full fidelity, having paid
|
|
208
|
+
},
|
|
209
|
+
});
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
const manifest = {
|
|
213
|
+
name,
|
|
214
|
+
description: description ?? `${rows.length} reactions from one reader across ${byCorpus.size} corpora — what held their attention, and what did not.`,
|
|
215
|
+
count: rows.length,
|
|
216
|
+
dim: rows[0].vector.length,
|
|
217
|
+
model: "nomic-ai/nomic-embed-text-v1.5",
|
|
218
|
+
distance: "Cosine",
|
|
219
|
+
role_map: {
|
|
220
|
+
identity: "event", title: "label", subtitle: "corpus", temporal: "month",
|
|
221
|
+
spatial: null, media: null, tags: ["corpus"], measures: [], relations: [],
|
|
222
|
+
text: ["text"],
|
|
223
|
+
},
|
|
224
|
+
entity_types: [{ type: "reaction", count: rows.length }],
|
|
225
|
+
presentation: { types: { reaction: { icon: "👍", accent: "#7a5cc9", singular: "Reaction", plural: "Reactions" } } },
|
|
226
|
+
paywall: {
|
|
227
|
+
free: FREE, locked: LOCKED, price: String(price), asset,
|
|
228
|
+
sample: { rate: sample, count: 0 }, // filled by the caller that writes the shard
|
|
229
|
+
},
|
|
230
|
+
coverage: coverage(rows.map((e) => e.vector),
|
|
231
|
+
{ k: Math.max(1, Math.min(COVERAGE_K, Math.floor(rows.length / MIN_PER_CENTROID))) }),
|
|
232
|
+
shards: [],
|
|
233
|
+
};
|
|
234
|
+
return { manifest, rows: wire, locked, lineage: up, byCorpus: Object.fromEntries(byCorpus) };
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// ── self-check: `node publish/reactions.js` ────────────────────────────────
|
|
238
|
+
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}`) {
|
|
239
|
+
const { splitPayment } = await import("./terms.js");
|
|
240
|
+
const A = (n) => `0x${String(n).repeat(40).slice(0, 40)}`;
|
|
241
|
+
const vec = (a, b) => { const v = new Float32Array(256); v[0] = a; v[1] = b; return v; };
|
|
242
|
+
|
|
243
|
+
// ── coverage: the summary a buyer ranks without downloading anything ──
|
|
244
|
+
if (coverage([]) !== null) throw new Error("no vectors is no coverage, not an empty one");
|
|
245
|
+
const two = coverage([vec(1, 0), vec(1, 0.01), vec(0, 1), vec(0.01, 1)], { k: 2 });
|
|
246
|
+
if (two.vectors.length !== 2 || two.sampled !== 4) throw new Error("k centroids over n rows");
|
|
247
|
+
if (two.counts.reduce((s, c) => s + c, 0) !== 4) throw new Error("every row is counted exactly once");
|
|
248
|
+
if (two.dim !== 128 || two.vectors[0].length !== 128) throw new Error("centroids truncate to 128d like the bake");
|
|
249
|
+
for (const c of two.vectors) {
|
|
250
|
+
const n = Math.sqrt(c.reduce((s, x) => s + x * x, 0));
|
|
251
|
+
if (Math.abs(n - 1) > 0.01) throw new Error(`a truncated centroid must be RENORMALIZED, got |c| = ${n}`);
|
|
252
|
+
}
|
|
253
|
+
// The two clusters are found, not averaged into one blur.
|
|
254
|
+
const sims = two.vectors.map((c) => Math.abs(c[0]));
|
|
255
|
+
if (Math.max(...sims) < 0.9 || Math.min(...sims) > 0.1) throw new Error(`k-means must separate the two groups: ${sims}`);
|
|
256
|
+
// Deterministic: a re-publish must not mint a manifest that says something new.
|
|
257
|
+
if (JSON.stringify(coverage([vec(1, 0), vec(0, 1), vec(1, 0.2)], { k: 2 }))
|
|
258
|
+
!== JSON.stringify(coverage([vec(1, 0), vec(0, 1), vec(1, 0.2)], { k: 2 }))) throw new Error("two fits over the same rows must agree byte for byte");
|
|
259
|
+
if (coverage([vec(1, 0)], { k: 8 }).vectors.length !== 1) throw new Error("k is capped at the number of rows");
|
|
260
|
+
|
|
261
|
+
// ── the corpus ──
|
|
262
|
+
const EV = [
|
|
263
|
+
{ id: "f1", corpus: "archive-films", title: "Nosferatu (1922)", vector: vec(1, 0), reaction: "like", at: "2026-08-14T10:00:00Z" },
|
|
264
|
+
{ id: "f2", corpus: "archive-films", title: "Metropolis 1927", vector: vec(0.9, 0.1), reaction: "like", at: "2026-08-15T10:00:00Z" },
|
|
265
|
+
{ id: "f3", corpus: "archive-films", title: "some sitcom", vector: vec(0, 1), reaction: "skip", at: "2026-08-16T10:00:00Z" },
|
|
266
|
+
{ id: "g1", corpus: "games", title: "Return of the Obra Dinn", vector: vec(0.8, 0.2), reaction: "like", at: "2026-08-17T10:00:00Z" },
|
|
267
|
+
];
|
|
268
|
+
const SRC = { "archive-films": A(1), games: A(2) };
|
|
269
|
+
const { manifest, rows, locked, lineage: up, byCorpus } = reactionCorpus(EV, { publisher: A(3), name: "taste-of-3", sources: SRC, price: "300000" });
|
|
270
|
+
|
|
271
|
+
if (rows.length !== 4 || manifest.count !== 4) throw new Error("every reaction with a vector becomes a row");
|
|
272
|
+
if (byCorpus["archive-films"] !== 3 || byCorpus.games !== 1) throw new Error("contribution is counted per corpus");
|
|
273
|
+
|
|
274
|
+
// The split the free index is allowed to make — APPLIED, not just declared.
|
|
275
|
+
for (const r of rows) {
|
|
276
|
+
for (const f of FREE) if (!(f in r.fields)) throw new Error(`the free index must carry ${f}`);
|
|
277
|
+
for (const f of LOCKED) if (f in r.fields) throw new Error(`the free index must NOT carry ${f} — declaring a split is not applying one`);
|
|
278
|
+
if (r.fields.month.length !== 7) throw new Error(`time is coarse on purpose, got ${r.fields.month}`);
|
|
279
|
+
if (/\d{2}T\d{2}/.test(r.fields.month)) throw new Error("an exact timestamp is a join key — never publish one");
|
|
280
|
+
}
|
|
281
|
+
if (LOCKED.some((f) => FREE.includes(f))) throw new Error("a field is on exactly one side");
|
|
282
|
+
|
|
283
|
+
// Nothing paid may appear ANYWHERE in a free row, including inside a key. The
|
|
284
|
+
// ids and titles below are the paid column; a substring hit is a leak however
|
|
285
|
+
// it got there.
|
|
286
|
+
for (const r of rows) {
|
|
287
|
+
const blob = JSON.stringify(r);
|
|
288
|
+
for (const e of EV) {
|
|
289
|
+
for (const secret of [e.id, e.title]) {
|
|
290
|
+
if (secret && blob.includes(secret)) throw new Error(`a free row leaks ${JSON.stringify(secret)}: ${blob}`);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
if (rows.some((r) => r.v)) throw new Error("a reaction row must ship NO vector — it is a copy of the source row's, and the source corpora are free");
|
|
295
|
+
if (locked.length !== rows.length) throw new Error("every free row has exactly one paid counterpart");
|
|
296
|
+
if (!locked.every((r) => r.fields.v?.length)) throw new Error("the buyer, having paid, gets the true vector");
|
|
297
|
+
if (new Set(rows.map((r) => r.track_id)).size !== rows.length) throw new Error("ids must be unique or the two halves cannot be joined");
|
|
298
|
+
|
|
299
|
+
// THE JOIN ATTACK, which is what made the vectors indefensible: an attacker
|
|
300
|
+
// holding the (free) source corpus matches a published vector against it and
|
|
301
|
+
// reads the paid column for nothing. Measured at 5/5 exact on live `games`.
|
|
302
|
+
// With no vector published there is nothing to match, so the attack needs the
|
|
303
|
+
// one thing it was trying to avoid: buying the corpus.
|
|
304
|
+
const sourceVectors = EV.map((e) => e.vector);
|
|
305
|
+
for (const r of rows) {
|
|
306
|
+
const anyMatch = sourceVectors.some((sv) => JSON.stringify([...sv]) === JSON.stringify([...(r.vector ?? [])]));
|
|
307
|
+
if (anyMatch) throw new Error("a free row must not be joinable to the source corpus it came from");
|
|
308
|
+
}
|
|
309
|
+
// …and the centroids, which ARE published, never stand for one reaction.
|
|
310
|
+
if (manifest.coverage.vectors.length > Math.floor(EV.length / MIN_PER_CENTROID) && EV.length >= MIN_PER_CENTROID) {
|
|
311
|
+
throw new Error("a centroid over a single reaction is that reaction's vector with a hat on");
|
|
312
|
+
}
|
|
313
|
+
if (reactionCorpus(EV, { publisher: A(3), sources: {} }).manifest.coverage.vectors.length !== 1) {
|
|
314
|
+
throw new Error("4 reactions cannot support 8 centroids — collapse to one rather than publish each");
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
// Upstream is proportional and EXACT — 3 of 4 reactions came from the films.
|
|
318
|
+
const bps = Object.fromEntries(up.map((e) => [e.note, Number(e.bps)]));
|
|
319
|
+
if (bps["archive-films"] !== 1500 || bps.games !== 500) throw new Error(`upstream must follow contribution: ${JSON.stringify(bps)}`);
|
|
320
|
+
if (up.reduce((s, e) => s + Number(e.bps), 0) !== 2000) throw new Error("the declared upstream total must be exact, not a basis point short");
|
|
321
|
+
|
|
322
|
+
// …and it reconciles against a real sale. This is the claim: the corpora that
|
|
323
|
+
// produced the attention are paid when the attention sells.
|
|
324
|
+
const TERMS = { appId: "fangorn.tv", owner: A(9), appBps: 1000 };
|
|
325
|
+
const pay = splitPayment(300000n, { terms: TERMS, publisher: A(3), lineage: up });
|
|
326
|
+
const got = Object.fromEntries(pay.map((p) => [p.to, p.amount]));
|
|
327
|
+
if (got[A(9)] !== 30000n) throw new Error("the app's cut comes off the top and never moves");
|
|
328
|
+
if (got[A(1)] !== 40500n || got[A(2)] !== 13500n) throw new Error(`upstream is paid from the publisher's share: ${JSON.stringify(got, (k, v) => typeof v === "bigint" ? String(v) : v)}`);
|
|
329
|
+
if (got[A(3)] !== 216000n) throw new Error("the reader keeps the rest");
|
|
330
|
+
if (pay.reduce((s, p) => s + p.amount, 0n) !== 300000n) throw new Error("money is neither invented nor destroyed");
|
|
331
|
+
|
|
332
|
+
// A reader who cites a corpus nobody registered simply owes it nothing.
|
|
333
|
+
const orphan = reactionCorpus(EV, { publisher: A(3), sources: { games: A(2) } });
|
|
334
|
+
if (orphan.lineage.length !== 1 || Number(orphan.lineage[0].bps) !== 2000) throw new Error("an unregistered corpus cannot be paid, and its share does not vanish");
|
|
335
|
+
|
|
336
|
+
if (!reactionCorpus(EV, { publisher: A(3), sources: {} }).lineage.length === 0) throw new Error("citing nobody is allowed — the reader keeps it all");
|
|
337
|
+
try { reactionCorpus([], { publisher: A(3) }); throw new Error("x"); } catch (e) { if (!e.message.includes("at least one")) throw new Error("an empty corpus must be refused"); }
|
|
338
|
+
try { reactionCorpus(EV, {}); throw new Error("x"); } catch (e) { if (!e.message.includes("publisher")) throw new Error("a corpus with nobody to pay must be refused"); }
|
|
339
|
+
|
|
340
|
+
console.log("reactions.js self-check ok — reactions become a corpus with no model, coverage is deterministic and renormalized after truncation, "
|
|
341
|
+
+ "the free index ranks on centroids alone, publishes no per-row vector to join against and no paid string anywhere including its own keys, "
|
|
342
|
+
+ "time is coarse, upstream follows contribution exactly and reconciles against a real sale");
|
|
343
|
+
}
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
// Encrypting a file into the access worker, from Node.
|
|
2
|
+
//
|
|
3
|
+
// This is the publisher's byte path, and it is the half of publishing that no
|
|
4
|
+
// relay can do for you: the DEK is generated here, sealed to the worker's public
|
|
5
|
+
// key here, and the plaintext never leaves the machine that owns it. A relay
|
|
6
|
+
// that offered to do this step would be a relay that had your file.
|
|
7
|
+
//
|
|
8
|
+
// The browser has its own copy of this walk (an app's own encrypt.js, over
|
|
9
|
+
// File.slice instead of fs.read). They must agree byte for byte — they share
|
|
10
|
+
// `pack`, `aadFor` and `chunkKey` from envelope.js precisely so that the only
|
|
11
|
+
// thing that differs between them is how bytes are read off a disk.
|
|
12
|
+
import { open } from "node:fs/promises";
|
|
13
|
+
import { createHash } from "node:crypto";
|
|
14
|
+
import { seal } from "@fangorn-network/sdk";
|
|
15
|
+
import { bytesToHex } from "viem";
|
|
16
|
+
import {
|
|
17
|
+
CHUNK_SIZE, NONCE_LEN, aadFor, chunkKey, deleteResource,
|
|
18
|
+
getWorkerPubkey, pack, putChunk, unpack,
|
|
19
|
+
} from "./envelope.js";
|
|
20
|
+
|
|
21
|
+
export { CHUNK_SIZE, aadFor, chunkKey, deleteResource, getWorkerPubkey };
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Envelope-encrypt the file at `file` chunk by chunk and upload each
|
|
25
|
+
* {ciphertext, sealed DEK} to the access worker's R2. Reads one chunk at a time,
|
|
26
|
+
* so peak memory is ~2x chunkSize no matter how big the file is.
|
|
27
|
+
*
|
|
28
|
+
* @returns { plaintextHash, chunks, size, chunkSize } — the hash to commit
|
|
29
|
+
* on-chain, the chunk count the buyer reassembles, and the plaintext
|
|
30
|
+
* byte geometry a streaming player needs to map a Range request onto a
|
|
31
|
+
* chunk (chunk i starts at i * chunkSize).
|
|
32
|
+
*/
|
|
33
|
+
export async function encryptAndUpload({ file, resourceId, workerUrl, uploadToken, chunkSize = CHUNK_SIZE, onProgress }) {
|
|
34
|
+
const dek = crypto.getRandomValues(new Uint8Array(32));
|
|
35
|
+
const aesKey = await crypto.subtle.importKey("raw", dek, "AES-GCM", false, ["encrypt"]);
|
|
36
|
+
const sealedDek = bytesToHex(seal(dek, await getWorkerPubkey(workerUrl), resourceId));
|
|
37
|
+
|
|
38
|
+
const fh = await open(file, "r");
|
|
39
|
+
try {
|
|
40
|
+
const { size } = await fh.stat();
|
|
41
|
+
const chunks = Math.max(1, Math.ceil(size / chunkSize));
|
|
42
|
+
const hash = createHash("sha256");
|
|
43
|
+
const buf = Buffer.allocUnsafe(Math.min(size, chunkSize) || 1);
|
|
44
|
+
|
|
45
|
+
for (let i = 0; i < chunks; i++) {
|
|
46
|
+
const { bytesRead } = await fh.read(buf, 0, buf.length, i * chunkSize);
|
|
47
|
+
const plain = buf.subarray(0, bytesRead);
|
|
48
|
+
hash.update(plain);
|
|
49
|
+
|
|
50
|
+
const nonce = crypto.getRandomValues(new Uint8Array(NONCE_LEN));
|
|
51
|
+
const aesCt = new Uint8Array(
|
|
52
|
+
await crypto.subtle.encrypt({ name: "AES-GCM", iv: nonce, additionalData: aadFor(i) }, aesKey, await pack(plain)),
|
|
53
|
+
);
|
|
54
|
+
const body = new Uint8Array(NONCE_LEN + aesCt.length);
|
|
55
|
+
body.set(nonce, 0);
|
|
56
|
+
body.set(aesCt, NONCE_LEN);
|
|
57
|
+
|
|
58
|
+
await putChunk({ workerUrl, uploadToken, key: chunkKey(resourceId, i), body, sealedDek });
|
|
59
|
+
onProgress?.(i + 1, chunks);
|
|
60
|
+
}
|
|
61
|
+
return { plaintextHash: `0x${hash.digest("hex")}`, chunks, size, chunkSize };
|
|
62
|
+
} finally {
|
|
63
|
+
await fh.close();
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// ── self-check: `node publish/settle.js` — no network, no worker ─────────────
|
|
68
|
+
//
|
|
69
|
+
// It decrypts with bare AES-GCM rather than importing a buyer, because what is
|
|
70
|
+
// under test here is the ENCRYPTOR: chunk addressing, the AAD index binding, the
|
|
71
|
+
// ragged tail, and that pack/unpack round trip. An app's full publisher→buyer
|
|
72
|
+
// round trip belongs in that app, against its own buyer.
|
|
73
|
+
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
74
|
+
const { writeFileSync, rmSync } = await import("node:fs");
|
|
75
|
+
const { unseal } = await import("@fangorn-network/sdk");
|
|
76
|
+
const { x25519 } = await import("@noble/curves/ed25519");
|
|
77
|
+
const { hexToBytes, keccak256 } = await import("viem");
|
|
78
|
+
|
|
79
|
+
const CHUNK = 1000;
|
|
80
|
+
const tmp = `/tmp/westmarch-settle-${process.pid}.bin`;
|
|
81
|
+
const workerSecret = crypto.getRandomValues(new Uint8Array(32));
|
|
82
|
+
const workerPubkey = x25519.getPublicKey(workerSecret);
|
|
83
|
+
const r2 = new Map();
|
|
84
|
+
let sealedSeen = null;
|
|
85
|
+
const realFetch = globalThis.fetch;
|
|
86
|
+
globalThis.fetch = async (url, init = {}) => {
|
|
87
|
+
const u = String(url);
|
|
88
|
+
if (u.endsWith("/pubkey")) return { ok: true, json: async () => ({ pubkey: bytesToHex(workerPubkey) }) };
|
|
89
|
+
if (u.includes("/upload/")) {
|
|
90
|
+
const key = u.split("/upload/")[1];
|
|
91
|
+
if (init.method === "DELETE") return r2.delete(key), { ok: true };
|
|
92
|
+
sealedSeen = init.headers["X-Sealed-Dek"];
|
|
93
|
+
r2.set(key, new Uint8Array(init.body));
|
|
94
|
+
return { ok: true };
|
|
95
|
+
}
|
|
96
|
+
return { ok: false, status: 404 };
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
const rid = keccak256(new Uint8Array([1]));
|
|
100
|
+
/** Decrypt one stored chunk the way any buyer must. */
|
|
101
|
+
const read = async (i) => {
|
|
102
|
+
const dek = unseal(hexToBytes(sealedSeen), workerSecret, rid);
|
|
103
|
+
const key = await crypto.subtle.importKey("raw", dek, "AES-GCM", false, ["decrypt"]);
|
|
104
|
+
const body = r2.get(chunkKey(rid, i));
|
|
105
|
+
return unpack(new Uint8Array(await crypto.subtle.decrypt(
|
|
106
|
+
{ name: "AES-GCM", iv: body.subarray(0, NONCE_LEN), additionalData: aadFor(i) },
|
|
107
|
+
key, body.subarray(NONCE_LEN),
|
|
108
|
+
)));
|
|
109
|
+
};
|
|
110
|
+
const same = (a, b) => a.length === b.length && a.every((x, i) => x === b[i]);
|
|
111
|
+
|
|
112
|
+
try {
|
|
113
|
+
// Random bytes: 2.5 chunks → a ragged tail, and gzip must LOSE, so these
|
|
114
|
+
// exercise pack()'s store-raw branch.
|
|
115
|
+
const bytes = crypto.getRandomValues(new Uint8Array(2500));
|
|
116
|
+
writeFileSync(tmp, bytes);
|
|
117
|
+
const out = await encryptAndUpload({ file: tmp, resourceId: rid, workerUrl: "http://stub", chunkSize: CHUNK });
|
|
118
|
+
if (out.chunks !== 3) throw new Error(`chunk count: got ${out.chunks}, want 3`);
|
|
119
|
+
if (out.plaintextHash !== `0x${createHash("sha256").update(bytes).digest("hex")}`) throw new Error("streamed hash != whole-file sha256");
|
|
120
|
+
if (!r2.has(rid)) throw new Error("chunk 0 must land on the resourceId — that is where /access looks");
|
|
121
|
+
if (r2.size !== 3) throw new Error(`chunk keys collide: ${r2.size} objects for 3 chunks`);
|
|
122
|
+
// nonce(12) + flag(1) + ct(500) + tag(16)
|
|
123
|
+
if (r2.get(chunkKey(rid, 2)).length !== 12 + 1 + 500 + 16) throw new Error("ragged last chunk is the wrong size");
|
|
124
|
+
|
|
125
|
+
const back = new Uint8Array([...await read(0), ...await read(1), ...await read(2)]);
|
|
126
|
+
if (!same(back, bytes)) throw new Error("round trip corrupted the file");
|
|
127
|
+
|
|
128
|
+
// AAD binds each chunk to its index: a chunk decrypted at the wrong index
|
|
129
|
+
// must FAIL, not silently hand back scrambled bytes.
|
|
130
|
+
let swallowed = false;
|
|
131
|
+
try {
|
|
132
|
+
const dek = unseal(hexToBytes(sealedSeen), workerSecret, rid);
|
|
133
|
+
const key = await crypto.subtle.importKey("raw", dek, "AES-GCM", false, ["decrypt"]);
|
|
134
|
+
const body = r2.get(chunkKey(rid, 0));
|
|
135
|
+
await crypto.subtle.decrypt({ name: "AES-GCM", iv: body.subarray(0, NONCE_LEN), additionalData: aadFor(1) }, key, body.subarray(NONCE_LEN));
|
|
136
|
+
swallowed = true;
|
|
137
|
+
} catch { /* expected */ }
|
|
138
|
+
if (swallowed) throw new Error("a chunk decrypted under the wrong index — AAD binding is broken");
|
|
139
|
+
|
|
140
|
+
// Compressible payload: pack()'s other branch. Stored bytes must come in
|
|
141
|
+
// well under the plaintext and still round trip.
|
|
142
|
+
r2.clear();
|
|
143
|
+
const text = new Uint8Array(3000).fill(65);
|
|
144
|
+
writeFileSync(tmp, text);
|
|
145
|
+
const packed = await encryptAndUpload({ file: tmp, resourceId: rid, workerUrl: "http://stub", chunkSize: CHUNK });
|
|
146
|
+
const stored = [...r2.values()].reduce((n, b) => n + b.length, 0);
|
|
147
|
+
if (stored > text.length / 2) throw new Error(`compressible payload barely shrank: ${stored} of ${text.length}`);
|
|
148
|
+
const gzBack = new Uint8Array([...await read(0), ...await read(1), ...await read(2)]);
|
|
149
|
+
if (!same(gzBack, text)) throw new Error("gzip round trip corrupted the file");
|
|
150
|
+
|
|
151
|
+
// Delete frees every chunk, and a worker that refuses RAISES rather than
|
|
152
|
+
// reporting freed space the operator is still billed for.
|
|
153
|
+
if (await deleteResource({ workerUrl: "http://stub", uploadToken: "t", resourceId: rid, chunks: packed.chunks }) !== packed.chunks) throw new Error("deleteResource miscounted");
|
|
154
|
+
if (r2.size !== 0) throw new Error(`delete left ${r2.size} chunk(s) behind`);
|
|
155
|
+
globalThis.fetch = async () => ({ ok: false, status: 401, statusText: "Unauthorized" });
|
|
156
|
+
let quiet = false;
|
|
157
|
+
try { await deleteResource({ workerUrl: "http://stub", resourceId: rid, chunks: 2 }); quiet = true; } catch { /* expected */ }
|
|
158
|
+
if (quiet) throw new Error("a refused delete was swallowed — the ledger would free bytes still in R2");
|
|
159
|
+
|
|
160
|
+
console.log(`settle.js self-check ok — round trip, hash, ragged tail, AAD index binding, gzip ${stored}/${text.length} bytes, delete`);
|
|
161
|
+
} finally {
|
|
162
|
+
globalThis.fetch = realFetch;
|
|
163
|
+
rmSync(tmp, { force: true });
|
|
164
|
+
}
|
|
165
|
+
}
|