@sanning/proof 0.10.0 → 0.11.0

package/src/verifier.ts

@@ -9,7 +9,7 @@
 
 import canonicalize from "canonicalize";
 
-import { ed25519Verify, sha256Hex, utf8 } from "./crypto.js";
+import { ed25519Verify, isHexField, sha256Hex, utf8 } from "./crypto.js";
 import type { ContentRole, Envelope, VerificationResult, VerifyOptions } from "./types.js";
 
 // Fail-closed accepted-profile registry (envelope-spec §2): exactly the
@@ -191,6 +191,82 @@ export function jcs(value: unknown): string {
   return canonical;
 }
 
+// The byte-order mark, as a CHARACTER. RFC 8785 output never begins with
+// U+FEFF, so bytes that do are not conformant whatever a JSON reader makes of
+// them.
+const JCS_BOM = "";
+
+/**
+ * Read bytes that claim to be `jcs(value)` (RFC 8785), strictly. The Python
+ * twin is `jcs_loads` in `src/sanning_proof/canonicalize.py`, and every site in
+ * either kernel that reads bytes a producer signed goes through one of the two.
+ *
+ * ONE RULE, SPELLED OUT, BECAUSE TWO BUILT-INS ARE NOT ONE RULE. Four checks
+ * carry it, none of them left to a decoder's defaults: strict UTF-8, no leading
+ * byte-order mark, no non-finite literal (this kernel's `JSON.parse` rejects
+ * `NaN`, `Infinity` and `-Infinity` natively, and Python names them in a
+ * `parse_constant` hook), and no unpaired surrogate in the parsed value.
+ *
+ * THE FOURTH CHECK IS NOT THE FIRST ONE TWICE. The strict decode refuses a
+ * surrogate written as raw bytes; it cannot see one written as the ASCII
+ * escape `\ud800`, which `JSON.parse` turns into a lone code unit AFTER the
+ * decode has passed. `"c\ud800x"` is pure ASCII on the wire, so this reader
+ * accepted it while its own text above said it does not, and one poisoned
+ * record then deleted the Python kernel's whole chain reading (the gate on
+ * proof#154). Both spellings are the same malformed record, and both are
+ * refused here.
+ *
+ * `new TextDecoder()` alone is none of that. It replaces every invalid sequence
+ * with U+FFFD and strips a BOM. Python's `json.loads(<bytes>)` is not it
+ * either, in the opposite direction: CPython sniffs the encoding and decodes
+ * with `errors="surrogatepass"`, so it strips a UTF-8 BOM, accepts a UTF-16 or
+ * UTF-32 body, and passes a lone surrogate and a CESU-8 pair through. Both
+ * defaults have shipped a split verdict on one pack: a raw `0x80` inside a
+ * stamp body read `verified` here and `failed` there (gate 4), and then a
+ * `BOM + JCS(record)` checkpoint read `verified` there and `mixed` here (gate
+ * 5), both on proof#140. JCS is defined over UTF-8 and I-JSON (RFC 7493)
+ * forbids a lone surrogate, so each of those inputs is MALFORMED rather than
+ * another spelling of the same record.
+ *
+ * Throws for anything that is not conformant; every call site treats a throw as
+ * "this reader cannot read these bytes" and names the check that failed.
+ */
+export function jcsParse(bytes: Uint8Array): unknown {
+  const text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes);
+  if (text.startsWith(JCS_BOM)) throw new Error("jcsParse: leading byte-order mark");
+  const value: unknown = JSON.parse(text);
+  rejectUnpairedSurrogates(value);
+  return value;
+}
+
+/**
+ * Refuse a parsed value carrying an unpaired surrogate, in a key or a value, at
+ * any depth. The Python twin is `_reject_unpaired_surrogates`, and the two name
+ * the failure with the same words.
+ *
+ * Deliberately ITERATIVE. `rejectLoneSurrogates` below walks a value this
+ * kernel is about to serialize, which `boundedInput` has already depth-bounded.
+ * This one walks whatever `JSON.parse` returned for a producer's bytes, with no
+ * bound in front of it, so a recursive walk would turn a deeply-nested record
+ * into a `RangeError` out of the reader itself.
+ */
+function rejectUnpairedSurrogates(value: unknown): void {
+  const stack: unknown[] = [value];
+  while (stack.length > 0) {
+    const node = stack.pop();
+    if (typeof node === "string") {
+      if (hasLoneSurrogate(node)) throw new Error("jcsParse: unpaired surrogate");
+    } else if (Array.isArray(node)) {
+      for (const child of node) stack.push(child);
+    } else if (node !== null && typeof node === "object") {
+      for (const [key, child] of Object.entries(node)) {
+        if (hasLoneSurrogate(key)) throw new Error("jcsParse: unpaired surrogate");
+        stack.push(child);
+      }
+    }
+  }
+}
+
 // Walk every string in the value (keys and values). The check must run on
 // the INPUT: `canonicalize` escapes a lone surrogate as `\udXXX` text in its
 // output, so the malformed code unit is invisible after serialization.
@@ -280,11 +356,34 @@ export function isRfc3339UtcInstant(value: unknown): boolean {
 // present: §2 flags an absent `signed_at` as non-conformant rather than
 // rejecting it, and `environment` is optional in the skeleton.
 //
+// `public_key` AND `signature` ARE CHECKED HERE TOO (proof#146, under
+// proof#135 option B). The strict `hexToBytes` inside `ed25519Verify` is what
+// makes the three kernels agree on the VERDICT; naming the spelling is what
+// makes them agree on the REASON. On its own a respelled key reads `Ed25519
+// signature verification failed`, which sends an auditor looking for a forged
+// signature that is not there. proof#135 was opened against this function's
+// Python twin, for covering `signed_at` and `environment` and not `public_key`.
+//
+// ONLY A STRING IS CHECKED, the same rule `envelopeHexError` follows one layer
+// up: an absent or non-string `public_key` is already a failed signature in all
+// three kernels (Go names it "missing signature or public_key"), and naming it
+// here as a spelling would state one fact twice and in the wrong words. CASE IS
+// NOT CHECKED: proof#135 pinned the charset and left the case open.
+//
 // Own properties only, because those are the members `jcs` signs over.
 function skeletonFieldErrors(env: Envelope): string[] {
   const fields = env as unknown as Record<string, unknown>;
   const own = (k: string): unknown => (Object.prototype.hasOwnProperty.call(fields, k) ? fields[k] : undefined);
   const out: string[] = [];
+  for (const [name, nibbles] of [
+    ["public_key", 64],
+    ["signature", 128],
+  ] as const) {
+    const value = own(name);
+    if (typeof value === "string" && !isHexField(value, nibbles)) {
+      out.push(`malformed ${name}: not ${nibbles} hex characters (envelope-spec §2)`);
+    }
+  }
   const signedAt = own("signed_at");
   if (signedAt !== undefined && !isRfc3339UtcInstant(signedAt)) {
     out.push("malformed signed_at: not an RFC 3339 UTC date-time with Z naming a real date and time (envelope-spec §2)");
@@ -381,6 +480,44 @@ export async function verifyEnvelope(
         "Option keys are camelCase in the TypeScript kernel; the Python kernel uses snake_case.",
     );
   }
+  // ── A verifier never raises on an input (proof#85 item 3) ──
+  //
+  // The option check above is the one exception and stays OUTSIDE this try: an
+  // unknown option key is the CALLER's mistake, not the envelope's, and turning
+  // a typo into a verdict is how a check silently never runs.
+  //
+  // Below, every field read is a read of attacker-shaped data. `env` is typed
+  // as an `Envelope`, but nothing enforces that at runtime, and an object whose
+  // property reads throw (a `get`-trap `Proxy`, a getter that raises, an
+  // exotic object from another realm) passed the null/non-object guard and then
+  // escaped as a raised exception. An auditor cannot tell a raised exception
+  // apart from a broken tool, which is the one thing this kernel exists not to
+  // hand them. CLAUDE.md states the rule flatly: malformed envelopes return a
+  // failed `VerificationResult`, they do not throw. This closes the class, not
+  // the one `Proxy` case: #77 hardened the anchor-trace path for the same
+  // reason, and a per-input patch would have to be written again for the next
+  // exotic object.
+  try {
+    return await verifyEnvelopeChecked(env, opts);
+  } catch (e) {
+    return {
+      ok: false,
+      specVersionOk: false,
+      // `null`, never `false`: nothing was read, so nothing was determined.
+      // The same rule the not-a-JSON-object branch below follows.
+      payloadHashOk: null,
+      signatureOk: false,
+      contentHashOk: opts.expectedContentHash === undefined ? null : false,
+      contentRole: null,
+      errors: [`envelope could not be read: ${stringifyErr(e)}`],
+    };
+  }
+}
+
+async function verifyEnvelopeChecked(
+  env: Envelope,
+  opts: VerifyOptions,
+): Promise<VerificationResult> {
   const expectedContentHash = opts.expectedContentHash;
   const errors: string[] = [];
 
@@ -437,6 +574,16 @@ export async function verifyEnvelope(
   // inline payload, supplied committed bytes, or both. With neither, the
   // binding is undetermined (null): "signature-valid, semantics-undetermined"
   // (§3.1/§6.2). null does not fail the verdict; absence does.
+  //
+  // THE ENVELOPE'S OWN `payload_hash` IS NOT ECHOED INTO THESE REASONS, and
+  // that is a change (proof#154 F4). It is producer-controlled text of any
+  // length, and it went into an auditor-facing error list verbatim: a producer
+  // could put a newline and a forged `payload_hash: OK  envelope verified`
+  // line, or an ANSI escape that paints `VERIFIED` green, into what the
+  // verifier prints. No other field in this file echoes its value, the Python
+  // twin names the same failure without one, and the value is in the envelope
+  // the auditor already holds. The RECOMPUTED digest stays: this kernel
+  // produced it, so it is 64 characters of lowercase hex by construction.
   let payloadHashOk: boolean | null;
   if (typeof env.payload_hash !== "string") {
     payloadHashOk = false;
@@ -449,7 +596,7 @@ export async function verifyEnvelope(
         checks.push(recomputed === env.payload_hash);
         if (recomputed !== env.payload_hash) {
           errors.push(
-            `payload_hash mismatch: envelope=${env.payload_hash} recomputed=${recomputed}`,
+            `payload_hash mismatch: the envelope's payload_hash is not SHA-256(JCS(payload)), recomputed=${recomputed}`,
           );
         }
       } catch (e) {
@@ -462,7 +609,7 @@ export async function verifyEnvelope(
       checks.push(external === env.payload_hash);
       if (external !== env.payload_hash) {
         errors.push(
-          `payload_hash does not match the committed bytes: envelope=${env.payload_hash} bytes=${external}`,
+          `payload_hash does not match the committed bytes: bytes=${external}`,
         );
       }
     }

package/vectors/README.md

@@ -26,6 +26,41 @@ on a run that contacted none — a false accusation manufactured by a missing
 state, and the defect this table was written after. Vendoring rules are the same
 three steps as below, against `vectors/stamp-on-chain.json`.
 
+## `jcs-reader.json`
+
+The rule that decides how a kernel reads bytes that claim to be `JCS(value)`
+(RFC 8785). Four checks, none of them left to a built-in decoder's defaults:
+**strict UTF-8**, **no leading byte-order mark**, **no non-finite literal**, and
+**no unpaired surrogate in the parsed value**. The fourth is not the first one
+twice: a surrogate written as the ASCII escape `\ud800` survives every
+byte-level check, because the bytes are ASCII, and the parser produces the
+malformed code unit afterwards (proof#154).
+Implemented by [`../src/verifier.ts`](../src/verifier.ts) as `jcsParse` and by
+`src/sanning_proof/canonicalize.py` as `jcs_loads`, and asserted against this
+table by [`../test/host-wording.test.ts`](../test/host-wording.test.ts) **and by
+the Python kernel against the same bytes**, in `tests/test_host_wording.py`, so
+neither kernel can pass by agreeing with the other.
+
+**The one thing an implementation must not do with it: reach for its language's
+default.** `new TextDecoder()` replaces invalid UTF-8 with U+FFFD and strips a
+BOM. Python's `json.loads(<bytes>)` is not `bytes.decode("utf-8")` then
+`json.loads`: CPython sniffs the encoding and decodes with
+`errors="surrogatepass"`, so it strips a BOM too, accepts a UTF-16 or UTF-32
+body, and passes a lone surrogate and a CESU-8 pair through. Both defaults have
+shipped a split verdict on one pack that verifies (proof#140, gates 4 and 5).
+
+**The accepted cases matter as much as the rejected ones.** An honest body
+carrying astral characters, a noncharacter, a U+FEFF *inside a string value* or
+combining marks is conformant, and a reader that rejects any of them has broken
+evidence someone already anchored. Seven of the 26 cases are there for that,
+including an astral character spelled as an escaped surrogate PAIR, which is the
+honest input the fourth check must not catch.
+
+Vendoring rules are the same three steps as below, against
+`vectors/jcs-reader.json`. Hand the reader the **bytes**, never a string: half
+the cases are about the decode, and a reader tested through a string has not
+been tested.
+
 ## `pack-grading.json`
 
 The rule that decides whether a **pack** — a signed bundle plus whatever
@@ -72,7 +107,7 @@ happened, and the reason this file moved here.
 
 | Key | Meaning |
 |---|---|
-| `ruleVersion` | Bumped when a case's expectation changes **or a case is added**. An implementation asserts the version it was written against, so a table it does not understand fails loudly rather than passing vacuously. Currently **2**. |
+| `ruleVersion` | Bumped when a case's expectation changes **or a case is added**. An implementation asserts the version it was written against, so a table it does not understand fails loudly rather than passing vacuously. Currently **3**. |
 | `kernelEventClasses` | The four per-event outcome classes a verification result can carry, named so a case can be written as counts rather than as a hand-built bundle. |
 | `cases[].kernel` | The verification result to grade, as class counts. |
 | `cases[].logs` | The disclosure account: how many records the bundle presents, and how many disclosed usable bytes. |

package/vectors/jcs-reader.json

@@ -0,0 +1,166 @@
+{
+  "rule": "jcs-reader",
+  "ruleVersion": 2,
+  "statement": "How a kernel reads bytes that claim to be JCS(value) (RFC 8785). Four checks, none of them left to a built-in decoder's defaults: strict UTF-8, no LEADING byte-order mark, no non-finite literal, and no UNPAIRED surrogate in the parsed value. The fourth is not the first one twice: a surrogate written as the ASCII escape `\\ud800` survives every byte-level check, because the bytes are ASCII, and the parser produces the malformed code unit afterwards. Everything else a JSON parser rejects is rejected. A reader that accepts any of the rejected cases here is reading a set of bytes some other reader will read differently, and a producer chooses those bytes: that is a split verdict on one pack, not a spelling preference. The accepted cases matter as much as the rejected ones -- an honest body carrying astral characters, noncharacters, a U+FEFF inside a string value or combining marks is conformant and must keep verifying.",
+  "home": "@sanning/proof -- ts/vectors/jcs-reader.json. This file is the RULE, not a test fixture of one implementation. TypeScript implements it in ts/src/verifier.ts as jcsParse and Python in src/sanning_proof/canonicalize.py as jcs_loads; ts/test/host-wording.test.ts and tests/test_host_wording.py hold BOTH kernels to this file, so neither can pass by agreeing with the other.",
+  "howToUse": "Base64-decode `bytesB64`, hand the bytes to your kernel's JCS reader, and assert that it returns a value where `accept` is true and throws or raises where it is false. Never decode to a string first: half of these cases are ABOUT the decode, and a reader tested through a string has not been tested.",
+  "whyItExists": "Round four of proof#140 put a strict decoder on the TypeScript side of three sites and left json.loads(<bytes>) on the Python side of two. CPython hands bytes to detect_encoding and decodes with errors='surrogatepass', so Python became the LENIENT kernel: a sanning.anchor.trace/v3 pack whose checkpoint record bytes were BOM + JCS(record), re-sealed so the pack verifies, read `verified` with zero exceptions in Python and `mixed` in TypeScript. The full ladder, the case leg and CI were green over it. Both guards were source assertions over different module sets, and the TypeScript one stated the false premise in its own comment. One table both kernels read is the fix for that.",
+  "cases": [
+    {
+      "id": "plain",
+      "bytesB64": "eyJldmVudCI6eyJtZXJrbGVfcm9vdCI6ImFhIn0sInByZXZpb3VzX2hhc2giOiJHRU5FU0lTIn0=",
+      "accept": true,
+      "why": "the control: RFC 8785 output, read as an object"
+    },
+    {
+      "id": "bom-leading",
+      "bytesB64": "77u/eyJldmVudCI6eyJtZXJrbGVfcm9vdCI6ImFhIn0sInByZXZpb3VzX2hhc2giOiJHRU5FU0lTIn0=",
+      "accept": false,
+      "why": "a UTF-8 byte-order mark before the object. RFC 8785 output never begins with U+FEFF. `json.loads(<bytes>)` STRIPS it and `new TextDecoder()` STRIPS it; both strict readers keep it and reject"
+    },
+    {
+      "id": "utf16le-bom",
+      "bytesB64": "//57ACIAZQB2AGUAbgB0ACIAOgB7ACIAbQBlAHIAawBsAGUAXwByAG8AbwB0ACIAOgAiAGEAYQAiAH0ALAAiAHAAcgBlAHYAaQBvAHUAcwBfAGgAYQBzAGgAIgA6ACIARwBFAE4ARQBTAEkAUwAiAH0A",
+      "accept": false,
+      "why": "the same object encoded UTF-16LE with a BOM. CPython's `detect_encoding` sniffs it and decodes happily; JCS is defined over UTF-8 and this is not it"
+    },
+    {
+      "id": "utf16le-no-bom",
+      "bytesB64": "ewAiAGUAdgBlAG4AdAAiADoAewAiAG0AZQByAGsAbABlAF8AcgBvAG8AdAAiADoAIgBhAGEAIgB9ACwAIgBwAHIAZQB2AGkAbwB1AHMAXwBoAGEAcwBoACIAOgAiAEcARQBOAEUAUwBJAFMAIgB9AA==",
+      "accept": false,
+      "why": "UTF-16LE with no BOM. CPython falls back to a NUL heuristic on the first four bytes and still decodes it"
+    },
+    {
+      "id": "utf32le-bom",
+      "bytesB64": "//4AAHsAAAAiAAAAZQAAAHYAAABlAAAAbgAAAHQAAAAiAAAAOgAAAHsAAAAiAAAAbQAAAGUAAAByAAAAawAAAGwAAABlAAAAXwAAAHIAAABvAAAAbwAAAHQAAAAiAAAAOgAAACIAAABhAAAAYQAAACIAAAB9AAAALAAAACIAAABwAAAAcgAAAGUAAAB2AAAAaQAAAG8AAAB1AAAAcwAAAF8AAABoAAAAYQAAAHMAAABoAAAAIgAAADoAAAAiAAAARwAAAEUAAABOAAAARQAAAFMAAABJAAAAUwAAACIAAAB9AAAA",
+      "accept": false,
+      "why": "the same object encoded UTF-32LE with a BOM. `detect_encoding` sniffs this one too"
+    },
+    {
+      "id": "lone-surrogate-high",
+      "bytesB64": "eyJhIjoieO2ggHkifQ==",
+      "accept": false,
+      "why": "a high surrogate alone, encoded as three UTF-8 bytes. CPython decodes bytes with `errors=\"surrogatepass\"` and passes it through as a character; I-JSON (RFC 7493) forbids it and it is not representable as UTF-8"
+    },
+    {
+      "id": "lone-surrogate-low",
+      "bytesB64": "eyJhIjoieO2wgHkifQ==",
+      "accept": false,
+      "why": "a low surrogate alone, the mirror of the case above"
+    },
+    {
+      "id": "escaped-lone-surrogate-high",
+      "bytesB64": "eyJrIjoiY1x1ZDgwMHgifQ==",
+      "accept": false,
+      "why": "a high surrogate written as the ASCII escape `\\ud800`, with no low surrogate after it. PURE ASCII ON THE WIRE, which is the whole point: the strict UTF-8 decode above cannot see it, and both kernels accepted it while their own text said they refuse a lone surrogate. I-JSON (RFC 7493 2.1) forbids an unpaired surrogate and RFC 8785 cannot emit one, since it serializes to UTF-8 and UTF-8 has no encoding for a surrogate code point. Accepted, it is not a lenience anyone can live with: the Python kernel could not then encode the value for a sort key, `read_chains` raised, and the CLI reported `no event chain was found to follow in this bundle` for a bundle whose chains TypeScript read and named a finding on (proof#154)"
+    },
+    {
+      "id": "escaped-lone-surrogate-low",
+      "bytesB64": "eyJrIjoiY1x1ZGMwMHgifQ==",
+      "accept": false,
+      "why": "a low surrogate written as an ASCII escape with no high surrogate before it: the mirror of the case above, and the one a reader that only looks ahead for a pair would miss"
+    },
+    {
+      "id": "escaped-lone-surrogate-in-key",
+      "bytesB64": "eyJjXHVkODAweCI6MX0=",
+      "accept": false,
+      "why": "the same escape in a MEMBER NAME rather than a value. RFC 8785 sorts on member names, so a reader that walks values only accepts a record whose keys no other kernel can represent"
+    },
+    {
+      "id": "escaped-surrogate-pair-astral",
+      "bytesB64": "eyJrIjoiY1x1ZDgzZFx1ZGUwMHgifQ==",
+      "accept": true,
+      "why": "U+1F600 written as the escaped surrogate PAIR `\\ud83d\\ude00`. LEGITIMATE and it must keep verifying: both hosts join the pair into the one astral character before any check sees it, so this is the honest input the three rejections above must not catch. A reader that refuses every `\\udXXX` escape rather than every UNPAIRED one refuses this, and what it refuses is the honest spelling"
+    },
+    {
+      "id": "cesu8-pair",
+      "bytesB64": "eyJhIjoieO2gve24gHkifQ==",
+      "accept": false,
+      "why": "CESU-8: one astral code point spelled as a surrogate PAIR in three-byte form. Passes through `surrogatepass` and recombines; UTF-8 spells U+1F600 in four bytes and nothing else"
+    },
+    {
+      "id": "raw-0x80",
+      "bytesB64": "eyJhIjoieIB5In0=",
+      "accept": false,
+      "why": "a continuation byte with no lead byte: not UTF-8 at all. The case gate 4 found, where `new TextDecoder()` substituted U+FFFD and read `verified`"
+    },
+    {
+      "id": "overlong-solidus",
+      "bytesB64": "eyJhIjoieMCveSJ9",
+      "accept": false,
+      "why": "an overlong encoding of `/`. A second spelling of one code point is a second set of bytes hashing to a second digest"
+    },
+    {
+      "id": "truncated-sequence",
+      "bytesB64": "eyJhIjoieOKCeSJ9",
+      "accept": false,
+      "why": "a three-byte sequence cut after two"
+    },
+    {
+      "id": "nan",
+      "bytesB64": "eyJhIjpOYU59",
+      "accept": false,
+      "why": "`NaN`. RFC 8259 does not define it and RFC 8785 cannot emit it; `json.loads` returns a float and `JSON.parse` throws"
+    },
+    {
+      "id": "infinity",
+      "bytesB64": "eyJhIjpJbmZpbml0eX0=",
+      "accept": false,
+      "why": "`Infinity`, the same class as `NaN`"
+    },
+    {
+      "id": "negative-infinity",
+      "bytesB64": "eyJhIjotSW5maW5pdHl9",
+      "accept": false,
+      "why": "`-Infinity`, the same class as `NaN`"
+    },
+    {
+      "id": "not-json",
+      "bytesB64": "aGVsbG8=",
+      "accept": false,
+      "why": "not JSON in any encoding"
+    },
+    {
+      "id": "empty",
+      "bytesB64": "",
+      "accept": false,
+      "why": "no bytes at all"
+    },
+    {
+      "id": "astral-emoji",
+      "bytesB64": "eyJhIjoiePCfmIB5In0=",
+      "accept": true,
+      "why": "U+1F600 spelled the one way UTF-8 spells it, four bytes. LEGITIMATE: an honest body may carry any astral character and must keep verifying"
+    },
+    {
+      "id": "u-fffe",
+      "bytesB64": "eyJhIjoieO+/vnkifQ==",
+      "accept": true,
+      "why": "U+FFFE, a noncharacter, inside a string. LEGITIMATE: RFC 8785 emits it and neither kernel may reject it"
+    },
+    {
+      "id": "bom-mid-string",
+      "bytesB64": "eyJhIjoieO+7v3kifQ==",
+      "accept": true,
+      "why": "U+FEFF inside a string value rather than before the object. LEGITIMATE: it is an ordinary code point there, and only a LEADING one is the malformed case"
+    },
+    {
+      "id": "combining-marks",
+      "bytesB64": "eyJhIjoiZcyBzKcifQ==",
+      "accept": true,
+      "why": "a base character with two combining marks. LEGITIMATE: RFC 8785 does not normalize, so the reader must not either"
+    },
+    {
+      "id": "nul-in-string",
+      "bytesB64": "eyJhIjoieAB5In0=",
+      "accept": false,
+      "why": "a raw NUL inside a string. Valid UTF-8 and invalid JSON: RFC 8259 requires the escape, and both kernels reject at the parse rather than at the decode"
+    },
+    {
+      "id": "array-root",
+      "bytesB64": "WzEsMiwzXQ==",
+      "accept": true,
+      "why": "a JSON array at the root. The reader returns it; whether a given site accepts a non-object is that site's check, not this one's"
+    }
+  ]
+}

package/vectors/pack-grading.json

@@ -1,10 +1,10 @@
 {
   "rule": "pack-grading",
-  "ruleVersion": 2,
-  "statement": "What a verification surface may CLAIM about a pack it was handed. verified is reachable ONLY when every record's own bytes bound to its own commitment and nothing is missing, unreadable, surplus or omitted. Absent or unreadable evidence is incomplete (UNKNOWN), never failed and never verified. A hard kernel verdict wins: malformed > failed > incomplete > the kernel's own headline.",
+  "ruleVersion": 5,
+  "statement": "What a verification surface may CLAIM about a pack it was handed. verified is reachable ONLY when every record's own bytes bound to its own commitment and nothing is missing, unreadable, surplus or omitted. Absent or unreadable evidence is incomplete (UNKNOWN), never failed and never verified. A hard kernel verdict wins: malformed > failed > incomplete > the kernel's own headline. A pack that DEVIATES from its own signed declaration (evidence-bundle.md \u00a75.4) is failed, and every deviation is a cause: a declared record that is absent, a present record nobody declared, or a record the pack declared withheld whose bytes its own SIGNED BODY supplies. A reading that rests instead on the UNSIGNED content handed in beside a pack is not a deviation and never reaches failed (proof#147): each content-dependent finding is its own cause and carries the pack to incomplete, because whoever assembles a case fills that map in, so an accuser holding an intact pack could otherwise reach failed with no key at all. A deviation and a content-dependent finding together are still failed, because the deviation reads off the signed body and a failing check wins. A declaration check that came back partial therefore splits in THREE, and only two of the three turn on the declaration's readability: no declaration member in the body at all moves nothing, because every pack built before \u00a75.4 is that case; a declaration the reader CANNOT READ makes the pack incomplete, because declaration_version is chosen by the party that assembles the pack and has no accept-set, so an unreadable claim must never reach a clean verdict; and a declaration the reader read perfectly well whose disclosure only unsigned bytes could settle makes the pack incomplete as well, named as what the reader could not establish rather than as what the holder did. The grade carries the declaration check verdict in its own output field, so a consumer that keeps only the grade can still tell a declared pack from an undeclared one.",
   "home": "@sanning/proof — ts/vectors/pack-grading.json. This file is the RULE, not a test fixture of one implementation. The kernel implements it in ts/src/grading.ts and is held to it by ts/test/grading.test.ts.",
   "consumers": "A surface that grades packs SHOULD call the kernel's gradePack rather than derive the rule again. A surface that cannot yet (an older pinned kernel, another language) vendors this file byte-for-byte, asserts its own derivation against it, and runs a drift check that compares its copy against the copy shipped in the @sanning/proof tarball at vectors/pack-grading.json. A vendored copy that is not byte-identical to this one is a defect, not a variant.",
-  "howToUse": "Adapt a kernel verification result and a disclosure account into the shapes below, call gradePack, and assert the expectations. `kernel` gives the per-event outcome classes the kernel returns; `logs` gives the account of which records disclosed bytes; `omissions` gives the signed-leaf_count completeness notes. Nothing here depends on how the bytes were transported — a directory read and a browser drop produce the same account and must produce the same verdict.",
+  "howToUse": "Adapt a kernel verification result and a disclosure account into the shapes below, call gradePack, and assert the expectations. `kernel` gives the per-event outcome classes the kernel returns; `logs` gives the account of which records disclosed bytes; `omissions` gives the signed-leaf_count completeness notes. Nothing here depends on how the bytes were transported: a directory read and a browser drop produce the same account and must produce the same verdict. `declaration`, where a case carries one, gives the declaration check's verdict, whether the body carried a declaration member at all, and how many deviations it found; pass a check of that shape as gradePack's optional fourth argument. A case with no `declaration` key calls gradePack with three arguments, which is the pre-§5.4 behavior and must not move, and its `expect.declaration` is absent.",
   "kernelEventClasses": {
     "okBound": "{ ok: true, contentOk: true } — the record's own bytes hashed to its own commitment",
     "okUnknown": "{ ok: true, contentOk: null } — structurally fine, but no bytes were disclosed",
@@ -52,6 +52,14 @@
       "omissions": [],
       "expect": { "verdict": "incomplete", "recordsVerified": 0, "recordsFailed": 0, "recordsUndetermined": 26, "causeCount": 1 }
     },
+    {
+      "id": "withheld-record-clean-account",
+      "why": "the case the terminal got wrong (proof#110). A bundle the kernel grades `verified` under evidence-bundle.md §3, holding one record the producer WITHHELD. The disclosure account is clean, and correctly so: no bytes were ever expected for a record that was never offered, so nothing is missing, unreadable, surplus or omitted and there is no cause to report. The record is still unhashed. The PACK is `incomplete` while the BUNDLE is `verified`: the two enums are distinct, and an undetermined record ON ITS OWN is enough to downgrade, with no cause beside it. Every other incomplete case here carries a cause, so a surface could satisfy all of them by keying off `causes` alone and still print a green verdict over a record nobody checked, which is exactly what `proof verify` did.",
+      "kernel": { "status": "verified", "okBound": 2, "okUnknown": 1, "mismatched": 0, "brokenNoContent": 0 },
+      "logs": { "expected": 2, "present": 2, "missing": 0, "unreadable": 0, "extra": 0 },
+      "omissions": [],
+      "expect": { "verdict": "incomplete", "recordsVerified": 2, "recordsFailed": 0, "recordsUndetermined": 1, "causeCount": 0 }
+    },
     {
       "id": "one-log-unreadable",
       "why": "a file we could not read is our fault, not the record's — UNKNOWN",
@@ -131,6 +139,126 @@
       "logs": { "expected": 26, "present": 26, "missing": 0, "unreadable": 0, "extra": 0 },
       "omissions": [],
       "expect": { "verdict": "incomplete", "recordsVerified": 26, "recordsFailed": 0, "recordsUndetermined": 0, "causeCount": 0 }
+    },
+    {
+      "id": "declaration-matches",
+      "why": "a pack that holds exactly what its declaration says it holds: the declaration adds no cause and moves no verdict",
+      "kernel": { "status": "verified", "okBound": 26, "okUnknown": 0, "mismatched": 0, "brokenNoContent": 0 },
+      "logs": { "expected": 26, "present": 26, "missing": 0, "unreadable": 0, "extra": 0 },
+      "omissions": [],
+      "declaration": { "verdict": "verified", "present": true, "deviations": 0 },
+      "expect": { "verdict": "verified", "recordsVerified": 26, "recordsFailed": 0, "recordsUndetermined": 0, "causeCount": 0, "declaration": "verified" }
+    },
+    {
+      "id": "declaration-deviates",
+      "why": "the trimmed hand-over. Every record present verifies and nothing is missing from the disclosure account, so every other check is green; the pack deviates from its own declaration and that is a FAILURE, never an absence",
+      "kernel": { "status": "verified", "okBound": 26, "okUnknown": 0, "mismatched": 0, "brokenNoContent": 0 },
+      "logs": { "expected": 26, "present": 26, "missing": 0, "unreadable": 0, "extra": 0 },
+      "omissions": [],
+      "declaration": { "verdict": "failed", "present": true, "deviations": 1 },
+      "expect": { "verdict": "failed", "recordsVerified": 26, "recordsFailed": 0, "recordsUndetermined": 0, "causeCount": 1, "declaration": "failed" }
+    },
+    {
+      "id": "declaration-content-dependent",
+      "why": "the pack agrees with its own claim everywhere its signature reaches, and does not line up with the UNSIGNED content handed in beside it. A NAMED cause and incomplete, never failed: an accuser holding an intact pack reaches this by dropping one file or adding one, with no key at all (proof#147)",
+      "kernel": {
+        "status": "verified",
+        "okBound": 26,
+        "okUnknown": 0,
+        "mismatched": 0,
+        "brokenNoContent": 0
+      },
+      "logs": {
+        "expected": 26,
+        "present": 26,
+        "missing": 0,
+        "unreadable": 0,
+        "extra": 0
+      },
+      "omissions": [],
+      "declaration": {
+        "verdict": "partial",
+        "present": true,
+        "deviations": 0,
+        "contentFindings": 1
+      },
+      "expect": {
+        "verdict": "incomplete",
+        "recordsVerified": 26,
+        "recordsFailed": 0,
+        "recordsUndetermined": 0,
+        "causeCount": 1,
+        "declaration": "partial"
+      }
+    },
+    {
+      "id": "declaration-deviation-beats-a-content-finding",
+      "why": "one of each, over different records. The deviation reads off the pack's own signed body so the pack FAILS, and the content-dependent finding is still its own cause: two classes, both reported, neither folded into the other",
+      "kernel": {
+        "status": "verified",
+        "okBound": 26,
+        "okUnknown": 0,
+        "mismatched": 0,
+        "brokenNoContent": 0
+      },
+      "logs": {
+        "expected": 26,
+        "present": 26,
+        "missing": 0,
+        "unreadable": 0,
+        "extra": 0
+      },
+      "omissions": [],
+      "declaration": {
+        "verdict": "failed",
+        "present": true,
+        "deviations": 1,
+        "contentFindings": 1
+      },
+      "expect": {
+        "verdict": "failed",
+        "recordsVerified": 26,
+        "recordsFailed": 0,
+        "recordsUndetermined": 0,
+        "causeCount": 2,
+        "declaration": "failed"
+      }
+    },
+    {
+      "id": "declaration-names-every-deviation",
+      "why": "three deviations in one pack are three causes. A rule that reported one shortfall would pass the case above and lose the other two findings",
+      "kernel": { "status": "verified", "okBound": 26, "okUnknown": 0, "mismatched": 0, "brokenNoContent": 0 },
+      "logs": { "expected": 26, "present": 26, "missing": 0, "unreadable": 0, "extra": 0 },
+      "omissions": [],
+      "declaration": { "verdict": "failed", "present": true, "deviations": 3 },
+      "expect": { "verdict": "failed", "recordsVerified": 26, "recordsFailed": 0, "recordsUndetermined": 0, "causeCount": 3, "declaration": "failed" }
+    },
+    {
+      "id": "declaration-deviation-beats-incomplete",
+      "why": "a proven mismatch between what a pack claimed and what it carries is never softened into absence, even when the pack is also missing disclosed bytes",
+      "kernel": { "status": "verified", "okBound": 20, "okUnknown": 6, "mismatched": 0, "brokenNoContent": 0 },
+      "logs": { "expected": 26, "present": 20, "missing": 6, "unreadable": 0, "extra": 0 },
+      "omissions": [],
+      "declaration": { "verdict": "failed", "present": true, "deviations": 1 },
+      "expect": { "verdict": "failed", "recordsVerified": 20, "recordsFailed": 0, "recordsUndetermined": 6, "causeCount": 2, "declaration": "failed" }
+    },
+    {
+      "id": "declaration-absent-moves-nothing",
+      "why": "a pack with no declaration: partial is the declaration check's own answer, and pack grading keeps its existing verdict. Every pack built before \u00a75.4 is this case, and re-grading them would be a spec change wearing an optional argument. `present` is false, which is what separates this case from the one below it",
+      "kernel": { "status": "verified", "okBound": 26, "okUnknown": 0, "mismatched": 0, "brokenNoContent": 0 },
+      "logs": { "expected": 26, "present": 26, "missing": 0, "unreadable": 0, "extra": 0 },
+      "omissions": [],
+      "declaration": { "verdict": "partial", "present": false, "deviations": 0 },
+      "expect": { "verdict": "verified", "recordsVerified": 26, "recordsFailed": 0, "recordsUndetermined": 0, "causeCount": 0, "declaration": "partial" }
+    },
+    {
+      "id": "declaration-unreadable-is-incomplete",
+      "why": "a pack that DOES carry a declaration and wrote it in a shape this reader cannot read. `declaration_version` is chosen by the same party that assembles the pack and has no accept-set, so a holder can trim a record, write one unreadable field and re-seal the whole body with its own key: every other check stays green and the evidence of the trim sits inside the signed body. `partial` keeps its meaning on the CHECK, and the PACK is incomplete, because a claim the reader cannot read is not a claim the reader may ignore. The case above and this one differ only in `present`, and a rule that reads the verdict alone passes both while grading one of them clean",
+      "kernel": { "status": "verified", "okBound": 26, "okUnknown": 0, "mismatched": 0, "brokenNoContent": 0 },
+      "logs": { "expected": 26, "present": 26, "missing": 0, "unreadable": 0, "extra": 0 },
+      "omissions": [],
+      "declaration": { "verdict": "partial", "present": true, "deviations": 0 },
+      "expect": { "verdict": "incomplete", "recordsVerified": 26, "recordsFailed": 0, "recordsUndetermined": 0, "causeCount": 0, "declaration": "partial" }
     }
   ]
 }