@intellectual-frontiers/grc 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,453 @@
1
+ // Spec: 0001-grc FR-006
2
+ /**
3
+ * Evidence: JSON's canonical form and the digest it gives, a hash-chained ledger, a manifest of files and a deterministic bundle (commons
4
+ * 0001-grc FR-006 to FR-009). The kernel holds no key: a manifest's signature is made and checked by functions the caller gives.
5
+ *
6
+ * - `canonicalize` writes RFC 8785's JSON Canonicalization Scheme: object members sorted by the UTF-16 code units of their names, numbers
7
+ * as ECMAScript writes them, strings with JSON's minimal escapes, no white space; `evidenceDigest` is the SHA-256 of its UTF-8 (FR-006).
8
+ * - A ledger is append-only and hash-chained: entry `seq` (from 1) holds the digest of the entry before (64 zeros for the first), its time,
9
+ * kind, subject and evidence digest, and its own `digest`, the SHA-256 of the canonical form of the rest. It is written as JSON lines,
10
+ * and `verifyLedger` names the first entry that was changed, removed, inserted or reordered (FR-007). Removing the last entries leaves a
11
+ * valid chain, so a signature covers a head (`LedgerHead`) that `verifyLedger` is then given.
12
+ * - A manifest lists files by path, size and SHA-256 in the byte order of their paths; `verifyManifest` names each file missing, extra or
13
+ * different, and `verifyManifestSignature` checks its bytes with a `VerifySignature` the caller gives (FR-008).
14
+ * - A bundle is a gzip of a ustar archive whose first entry is the manifest (then its signature, if any), the same bytes every time for the
15
+ * same files; `readBundle` gives the files back only where the manifest (and the signature, if a verifier is given) holds (FR-009).
16
+ *
17
+ * @example
18
+ * ```ts
19
+ * import { canonicalize, evidenceDigest } from "./evidence.ts";
20
+ * canonicalize({ b: 1, a: 2.0 }); // { kind: "ok", value: '{"a":2,"b":1}' }
21
+ * await evidenceDigest({ a: 2, b: 1 }); // the same digest as for { b: 1, a: 2.0 }
22
+ * ```
23
+ *
24
+ * @module
25
+ */
26
+ import { absurd, array, formatDecodeError, integer, object, refine, string, err, ok, } from "@intellectual-frontiers/core";
27
+ import { concat, filesOf, gunzip, gzip, isSafePath, sortedPaths, tar, tarHeader, untar, } from "@intellectual-frontiers/formats/archive";
28
+ import { instantDecoder } from "./grc.js";
29
+ import { sha256, sha256Of } from "@intellectual-frontiers/crypto/sha256";
30
+ function memberPath(path, key) {
31
+ return /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key) ? `${path}.${key}` : `${path}[${JSON.stringify(key)}]`;
32
+ }
33
+ function isPlainObject(v) {
34
+ const proto = Object.getPrototypeOf(v);
35
+ return proto === Object.prototype || proto === null;
36
+ }
37
+ function canonicalText(value, path, parents, out) {
38
+ const fail = (reason) => ({ kind: "canonical", path, reason });
39
+ if (value === null)
40
+ out.push("null");
41
+ else if (typeof value === "boolean")
42
+ out.push(value ? "true" : "false");
43
+ else if (typeof value === "number") {
44
+ if (!Number.isFinite(value))
45
+ return fail(`${String(value)} is not a JSON number`);
46
+ out.push(JSON.stringify(value));
47
+ }
48
+ else if (typeof value === "string") {
49
+ if (!value.isWellFormed())
50
+ return fail("the string holds a lone surrogate");
51
+ out.push(JSON.stringify(value));
52
+ }
53
+ else if (typeof value === "object") {
54
+ if (parents.has(value))
55
+ return fail("the value refers to itself");
56
+ parents.add(value);
57
+ if (Array.isArray(value)) {
58
+ out.push("[");
59
+ for (let i = 0; i < value.length; i++) {
60
+ if (i > 0)
61
+ out.push(",");
62
+ const item = value[i];
63
+ if (item === undefined)
64
+ return { kind: "canonical", path: `${path}[${i}]`, reason: "an array holds nothing here" };
65
+ const e = canonicalText(item, `${path}[${i}]`, parents, out);
66
+ if (e !== undefined)
67
+ return e;
68
+ }
69
+ out.push("]");
70
+ }
71
+ else {
72
+ if (!isPlainObject(value))
73
+ return fail("only plain objects are JSON objects");
74
+ const keys = Object.keys(value).sort();
75
+ out.push("{");
76
+ let first = true;
77
+ for (const key of keys) {
78
+ const member = Reflect.get(value, key);
79
+ if (member === undefined)
80
+ continue;
81
+ if (!key.isWellFormed())
82
+ return fail(`the member name ${JSON.stringify(key)} holds a lone surrogate`);
83
+ out.push(first ? "" : ",", JSON.stringify(key), ":");
84
+ first = false;
85
+ const e = canonicalText(member, memberPath(path, key), parents, out);
86
+ if (e !== undefined)
87
+ return e;
88
+ }
89
+ out.push("}");
90
+ }
91
+ parents.delete(value);
92
+ }
93
+ else
94
+ return fail(`a ${typeof value} is not JSON`);
95
+ return undefined;
96
+ }
97
+ /**
98
+ * A value's canonical JSON text (RFC 8785): plain objects, arrays, strings, finite numbers, booleans and `null`. An object member whose
99
+ * value is `undefined` is left out, as `JSON.stringify` leaves it; anything else that is not JSON (a non-finite number, a lone surrogate,
100
+ * a `Date`, a class instance, a cycle, a hole) is an error naming where it is.
101
+ */
102
+ export function canonicalize(value) {
103
+ const out = [];
104
+ const e = canonicalText(value, "$", new Set(), out);
105
+ return e === undefined ? ok(out.join("")) : err(e);
106
+ }
107
+ /** A value's canonical JSON as UTF-8 bytes. */
108
+ export function canonicalBytes(value) {
109
+ const r = canonicalize(value);
110
+ return r.kind === "ok" ? ok(new TextEncoder().encode(r.value)) : r;
111
+ }
112
+ /** The address of a piece of evidence: the SHA-256 of its canonical JSON, the same on every runtime. */
113
+ export async function evidenceDigest(value) {
114
+ const r = canonicalize(value);
115
+ return r.kind === "ok" ? ok(await sha256(r.value)) : r;
116
+ }
117
+ // ---------------------------------------------------------------------------------------------------------------------------- ledger (FR-007)
118
+ /** The digest the first entry of a ledger names as the one before it: 64 zeros. */
119
+ export const GENESIS = "0".repeat(64); // 64 hexadecimal digits by construction
120
+ /** The digest an entry must carry: the SHA-256 of the canonical JSON of every field but `digest`. */
121
+ export async function entryDigest(e) {
122
+ return evidenceDigest({
123
+ seq: e.seq,
124
+ prev: e.prev,
125
+ at: e.at,
126
+ kind: e.kind,
127
+ subject: { kind: e.subject.kind, id: e.subject.id },
128
+ evidence: e.evidence,
129
+ });
130
+ }
131
+ /** The head of a ledger: its last entry's place and digest. */
132
+ export function ledgerHead(entries) {
133
+ const last = entries.at(-1);
134
+ return last === undefined ? { seq: 0, digest: GENESIS } : { seq: last.seq, digest: last.digest };
135
+ }
136
+ /** The ledger with one record appended; an error where the record has no canonical form (a lone surrogate in its kind or subject). */
137
+ export async function appendEntry(entries, record) {
138
+ const head = ledgerHead(entries);
139
+ const body = {
140
+ seq: head.seq + 1,
141
+ prev: head.digest,
142
+ at: record.at,
143
+ kind: record.kind,
144
+ subject: record.subject,
145
+ evidence: record.evidence,
146
+ };
147
+ const d = await entryDigest(body);
148
+ return d.kind === "ok" ? ok([...entries, { ...body, digest: d.value }]) : d;
149
+ }
150
+ /** A ledger as JSON lines: each entry's canonical JSON and a line feed. */
151
+ export function ledgerLines(entries) {
152
+ return entries
153
+ .map((e) => {
154
+ const r = canonicalize(e);
155
+ return `${r.kind === "ok" ? r.value : JSON.stringify(e)}\n`;
156
+ })
157
+ .join("");
158
+ }
159
+ const sha256Decoder = (input, path = "$") => {
160
+ const s = typeof input === "string" && /^[0-9a-f]{64}$/.test(input) ? sha256Of(input) : undefined;
161
+ return s === undefined
162
+ ? err({ kind: "decode", path, expected: "a SHA-256 (64 lower-case hexadecimal digits)", found: typeof input })
163
+ : ok(s);
164
+ };
165
+ /** A ledger entry as its JSON holds it. */
166
+ export const ledgerEntryDecoder = object({
167
+ seq: refine(integer, "positive", (n) => n >= 1),
168
+ prev: sha256Decoder,
169
+ at: instantDecoder,
170
+ kind: string,
171
+ subject: object({ kind: string, id: string }),
172
+ evidence: sha256Decoder,
173
+ digest: sha256Decoder,
174
+ });
175
+ /** A ledger parse error as one sentence. */
176
+ export function formatLedgerParseError(e) {
177
+ return `line ${e.line}: ${formatDecodeError(e.error)}`;
178
+ }
179
+ /** A ledger from its JSON lines (blank lines skipped), each decoded; the chain is checked by `verifyLedger`. */
180
+ export function parseLedger(text) {
181
+ const entries = [];
182
+ const lines = text.split("\n");
183
+ for (const [i, raw] of lines.entries()) {
184
+ const line = raw.replace(/\r$/, "");
185
+ if (line.trim() === "")
186
+ continue;
187
+ let value;
188
+ try {
189
+ value = JSON.parse(line);
190
+ }
191
+ catch (e) {
192
+ return err({
193
+ kind: "ledger-line",
194
+ line: i + 1,
195
+ error: { kind: "decode", path: "$", expected: "JSON", found: e instanceof Error ? e.message : String(e) },
196
+ });
197
+ }
198
+ const r = ledgerEntryDecoder(value);
199
+ if (r.kind === "err")
200
+ return err({ kind: "ledger-line", line: i + 1, error: r.error });
201
+ entries.push(r.value);
202
+ }
203
+ return ok(entries);
204
+ }
205
+ /** A ledger break as one sentence. */
206
+ export function formatLedgerBreak(b) {
207
+ return `entry ${b.seq} was ${b.kind}: ${b.reason}`;
208
+ }
209
+ /**
210
+ * The ledger's head where every entry's digest is the digest of its content, its `seq` its place and its `prev` the digest of the entry
211
+ * before it (and, where a `head` is given, the ledger reaches it with that digest); else the first break. Entry N edited in place is
212
+ * `changed` at N (edited and re-hashed, it is found at N + 1's link and still named N); entry N gone is `removed` at N; a forged entry
213
+ * slipped in at N is `inserted` at N; two entries swapped are `reordered` at the first of them.
214
+ */
215
+ export async function verifyLedger(entries, head) {
216
+ const brk = (kind, seq, reason) => err({ kind, seq, reason });
217
+ for (const [i, e] of entries.entries()) {
218
+ const place = i + 1;
219
+ const d = await entryDigest(e);
220
+ if (d.kind === "err" || d.value !== e.digest)
221
+ return brk("changed", place, "its digest is not the SHA-256 of its content");
222
+ if (e.seq > place) {
223
+ return entries.slice(i + 1).some((x) => x.seq === place)
224
+ ? brk("reordered", place, `entry ${e.seq} stands where entry ${place} should`)
225
+ : brk("removed", place, e.seq === place + 1 ? `entry ${place} is missing` : `entries ${place} to ${e.seq - 1} are missing`);
226
+ }
227
+ if (e.seq < place) {
228
+ const beforePrevious = i >= 2 ? entries[i - 2]?.digest : GENESIS;
229
+ return e.prev === beforePrevious
230
+ ? brk("inserted", Math.max(place - 1, 1), `it is not in the chain: entry ${place} links past it`)
231
+ : brk("inserted", place, `it repeats the place ${e.seq}`);
232
+ }
233
+ const previous = i === 0 ? GENESIS : (entries[i - 1]?.digest ?? GENESIS);
234
+ if (e.prev !== previous)
235
+ return i === 0
236
+ ? brk("changed", 1, "it does not begin the chain")
237
+ : brk("changed", place - 1, `entry ${place} names another digest for it`);
238
+ }
239
+ if (head !== undefined) {
240
+ if (entries.length < head.seq)
241
+ return brk("removed", entries.length + 1, `the ledger ends before its head, entry ${head.seq}`);
242
+ const at = head.seq === 0 ? GENESIS : entries[head.seq - 1]?.digest;
243
+ if (at !== head.digest)
244
+ return brk("changed", Math.max(head.seq, 1), "its digest is not the head's");
245
+ }
246
+ return ok(ledgerHead(entries));
247
+ }
248
+ /** The manifest of some files. */
249
+ export async function manifestOf(files) {
250
+ const out = [];
251
+ for (const path of sortedPaths(files.keys())) {
252
+ const data = files.get(path) ?? new Uint8Array();
253
+ out.push({ path, size: data.length, sha256: await sha256(data) });
254
+ }
255
+ return { files: out };
256
+ }
257
+ /** A manifest's bytes: its canonical JSON (RFC 8785) and a line feed. These are the bytes a signature covers. */
258
+ export function manifestBytes(m) {
259
+ const r = canonicalize({ files: m.files.map((f) => ({ path: f.path, size: f.size, sha256: f.sha256 })) });
260
+ // A manifest's strings are paths and digests; a path with a lone surrogate is written as UTF-8 would replace it.
261
+ return new TextEncoder().encode(`${r.kind === "ok" ? r.value : JSON.stringify(m)}\n`);
262
+ }
263
+ /** A manifest error as one sentence. */
264
+ export function formatManifestError(e) {
265
+ switch (e.kind) {
266
+ case "manifest-decode":
267
+ return `the manifest does not decode: ${formatDecodeError(e.error)}`;
268
+ case "manifest-order":
269
+ return `the manifest lists ${JSON.stringify(e.path)} out of byte order or twice`;
270
+ case "manifest-unsafe-path":
271
+ return `the manifest lists a path that would leave its folder: ${JSON.stringify(e.path)}`;
272
+ default:
273
+ return absurd(e);
274
+ }
275
+ }
276
+ const manifestDecoder = object({
277
+ files: array(object({ path: string, size: refine(integer, "size", (n) => n >= 0), sha256: sha256Decoder })),
278
+ });
279
+ /** A manifest from its bytes (or text), each path checked safe and the list checked in strict byte order. */
280
+ export function parseManifest(input) {
281
+ const text = typeof input === "string" ? input : new TextDecoder().decode(input);
282
+ let value;
283
+ try {
284
+ value = JSON.parse(text);
285
+ }
286
+ catch (e) {
287
+ return err({
288
+ kind: "manifest-decode",
289
+ error: { kind: "decode", path: "$", expected: "JSON", found: e instanceof Error ? e.message : String(e) },
290
+ });
291
+ }
292
+ const r = manifestDecoder(value);
293
+ if (r.kind === "err")
294
+ return err({ kind: "manifest-decode", error: r.error });
295
+ const paths = r.value.files.map((f) => f.path);
296
+ const sorted = sortedPaths(paths);
297
+ for (const [i, p] of paths.entries()) {
298
+ if (!isSafePath(p))
299
+ return err({ kind: "manifest-unsafe-path", path: p });
300
+ if (sorted[i] !== p || (i > 0 && paths[i - 1] === p))
301
+ return err({ kind: "manifest-order", path: p });
302
+ }
303
+ return r;
304
+ }
305
+ /** A mismatch as one sentence. */
306
+ export function formatManifestMismatch(m) {
307
+ switch (m.kind) {
308
+ case "missing":
309
+ return `${m.path} is in the manifest but not among the files`;
310
+ case "extra":
311
+ return `${m.path} is among the files but not in the manifest`;
312
+ case "different":
313
+ return `${m.path} is not the file the manifest lists (${m.actual.size} bytes, ${m.actual.sha256})`;
314
+ default:
315
+ return absurd(m);
316
+ }
317
+ }
318
+ /** Whether files are exactly what the manifest lists; else every file missing, extra or different, in the byte order of their paths. */
319
+ export async function verifyManifest(m, files) {
320
+ const listed = new Map(m.files.map((f) => [f.path, f]));
321
+ const out = [];
322
+ for (const path of sortedPaths(new Set([...listed.keys(), ...files.keys()]))) {
323
+ const want = listed.get(path);
324
+ const data = files.get(path);
325
+ if (want === undefined)
326
+ out.push({ kind: "extra", path });
327
+ else if (data === undefined)
328
+ out.push({ kind: "missing", path });
329
+ else {
330
+ const sum = await sha256(data);
331
+ if (sum !== want.sha256 || data.length !== want.size)
332
+ out.push({
333
+ kind: "different",
334
+ path,
335
+ expected: { size: want.size, sha256: want.sha256 },
336
+ actual: { size: data.length, sha256: sum },
337
+ });
338
+ }
339
+ }
340
+ return out.length === 0 ? ok(m) : err(out);
341
+ }
342
+ /** The manifest its bytes hold, where the signature over those exact bytes verifies; a verifier that throws is a bad signature. */
343
+ export async function verifyManifestSignature(bytes, signature, verify) {
344
+ let good;
345
+ try {
346
+ good = await verify(bytes, signature);
347
+ }
348
+ catch (e) {
349
+ return err({ kind: "bad-signature", reason: e instanceof Error ? e.message : String(e) });
350
+ }
351
+ if (!good)
352
+ return err({ kind: "bad-signature", reason: "the signature does not verify" });
353
+ return parseManifest(bytes);
354
+ }
355
+ // ---------------------------------------------------------------------------------------------------------------------------- bundle (FR-009)
356
+ /** The path of a bundle's manifest, its first entry. */
357
+ export const BUNDLE_MANIFEST = "MANIFEST.json";
358
+ /** The path of a bundle manifest's signature, its second entry where there is one. */
359
+ export const BUNDLE_SIGNATURE = "MANIFEST.json.sig";
360
+ /** A bundle error as one sentence. */
361
+ export function formatBundleError(e) {
362
+ switch (e.kind) {
363
+ case "archive":
364
+ return `the bundle's archive: ${e.error.kind}`;
365
+ case "reserved-path":
366
+ return `${e.path} is the bundle's own and cannot be one of its files`;
367
+ case "no-manifest":
368
+ return `the bundle does not begin with ${BUNDLE_MANIFEST}`;
369
+ case "mismatch":
370
+ return `the bundle's files are not its manifest's: ${e.mismatches.map(formatManifestMismatch).join("; ")}`;
371
+ case "unsigned":
372
+ return "the bundle has no signature";
373
+ case "bad-signature":
374
+ return `the bundle's signature: ${e.reason}`;
375
+ case "manifest-decode":
376
+ case "manifest-order":
377
+ case "manifest-unsafe-path":
378
+ return formatManifestError(e);
379
+ default:
380
+ return absurd(e);
381
+ }
382
+ }
383
+ function tarEntry(path, data) {
384
+ const h = tarHeader(path, data.length, false);
385
+ if (h === undefined)
386
+ return err({ kind: "archive", error: { kind: "too-long", path } });
387
+ return ok(concat([h, data, new Uint8Array((512 - (data.length % 512)) % 512)]));
388
+ }
389
+ /**
390
+ * A bundle of files: a gzip of a ustar archive whose first entry is the manifest, then its signature where a signer is given, then the
391
+ * files (their folders first) in byte order, every time and date fixed, so the same files (and a deterministic signer) give the same bytes.
392
+ */
393
+ export async function writeBundle(files, options = {}) {
394
+ for (const reserved of [BUNDLE_MANIFEST, BUNDLE_SIGNATURE])
395
+ if (files.has(reserved))
396
+ return err({ kind: "reserved-path", path: reserved });
397
+ const manifest = manifestBytes(await manifestOf(files));
398
+ const head = tarEntry(BUNDLE_MANIFEST, manifest);
399
+ if (head.kind === "err")
400
+ return head;
401
+ const parts = [head.value];
402
+ if (options.sign !== undefined) {
403
+ const sig = tarEntry(BUNDLE_SIGNATURE, await options.sign(manifest));
404
+ if (sig.kind === "err")
405
+ return sig;
406
+ parts.push(sig.value);
407
+ }
408
+ const rest = tar(files);
409
+ if (rest.kind === "err")
410
+ return err({ kind: "archive", error: rest.error });
411
+ parts.push(rest.value);
412
+ return ok(gzip(concat(parts)));
413
+ }
414
+ /**
415
+ * A bundle's files, where its first entry is a manifest the files match exactly (one added, removed or changed is named); with `verify`,
416
+ * also where its signature verifies over the manifest's bytes (a bundle with none is then refused as unsigned).
417
+ */
418
+ export async function readBundle(data, options = {}) {
419
+ const unzipped = await gunzip(data);
420
+ if (unzipped.kind === "err")
421
+ return err({ kind: "archive", error: unzipped.error });
422
+ const entries = untar(unzipped.value);
423
+ if (entries.kind === "err")
424
+ return err({ kind: "archive", error: entries.error });
425
+ const first = entries.value[0];
426
+ if (first === undefined || first.kind !== "file" || first.path !== BUNDLE_MANIFEST)
427
+ return err({ kind: "no-manifest" });
428
+ const all = filesOf(entries.value);
429
+ const signature = all.get(BUNDLE_SIGNATURE);
430
+ let manifest;
431
+ if (options.verify !== undefined) {
432
+ if (signature === undefined)
433
+ return err({ kind: "unsigned" });
434
+ manifest = await verifyManifestSignature(first.data, signature, options.verify);
435
+ }
436
+ else
437
+ manifest = parseManifest(first.data);
438
+ if (manifest.kind === "err")
439
+ return manifest;
440
+ const files = new Map([...all].filter(([p]) => p !== BUNDLE_MANIFEST && p !== BUNDLE_SIGNATURE));
441
+ const checked = await verifyManifest(manifest.value, files);
442
+ if (checked.kind === "err")
443
+ return err({ kind: "mismatch", mismatches: checked.error });
444
+ return ok({
445
+ manifest: manifest.value,
446
+ files,
447
+ signature: signature === undefined
448
+ ? { kind: "none" }
449
+ : options.verify !== undefined
450
+ ? { kind: "verified" }
451
+ : { kind: "unverified", bytes: signature },
452
+ });
453
+ }