@sanning/proof 0.11.1 → 0.13.0

package/src/zip-reader.ts

@@ -0,0 +1,229 @@
+/**
+ * The one ZIP reader `case` uses, for a case file named on the command line.
+ *
+ * The TypeScript half of `src/sanning_proof/zip_reader.py`. Node has no ZIP
+ * reader, and Python's `zipfile` is not used, so the two kernels read one
+ * archive by one rule rather than by two libraries that disagree on the edges:
+ * a central directory that contradicts a local header, a name that is not
+ * UTF-8, data after the end of a deflate stream. Where either library would
+ * have made a choice, this module refuses instead.
+ *
+ * What it reads (evidence-bundle.md 5.6.1): the central directory, and each
+ * entry's data through its local header. Stored and deflate only. What it
+ * REFUSES, as a whole archive: no end-of-central-directory record, ZIP64, more
+ * than one disk, an entry the central directory places outside the archive, a
+ * local header that does not match its central entry, a size or CRC-32 that
+ * does not match the data, two entries with one name, a name that is not
+ * UTF-8, more than MAX_ZIP_ENTRIES entries or MAX_ZIP_BYTES bytes in all, and
+ * a name nested deeper than MAX_ZIP_DEPTH folders. What it REPORTS and never
+ * reads, entry by entry: an encrypted entry, a symbolic link, a method other
+ * than stored or deflate, and a name that is absolute, holds a backslash or a
+ * NUL, or has an empty, `.` or `..` segment.
+ *
+ * Pure: bytes in, entries out. The caller turns them into the tree `case`
+ * splits.
+ */
+
+import { inflateRawSync } from "node:zlib";
+
+import { utf8Strict } from "./verifier.js";
+
+/** More entries than this refuse the archive: the walk's own bound on names. */
+export const MAX_ZIP_ENTRIES = 50_000;
+/** More uncompressed bytes than this, across every entry, refuse the archive. */
+export const MAX_ZIP_BYTES = 2 ** 30;
+/** A name nested deeper than this many folders refuses the archive. */
+export const MAX_ZIP_DEPTH = 32;
+
+const EOCD = 0x06054b50;
+const CENTRAL = 0x02014b50;
+const LOCAL = 0x04034b50;
+const S_IFMT = 0o170000;
+const S_IFLNK = 0o120000;
+const UNIX = 3;
+
+/** The archive cannot be read as a case file. `message` is the whole report, identical in both kernels. */
+export class ZipReadError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ZipReadError";
+  }
+}
+
+/**
+ * One file entry. `refused` is why the entry is reported and not read, or
+ * `null`; `data` is its bytes where it was read.
+ */
+export interface ZipEntry {
+  name: string;
+  refused: string | null;
+  data: Uint8Array | null;
+}
+
+const CRC_TABLE = (() => {
+  const table = new Uint32Array(256);
+  for (let n = 0; n < 256; n++) {
+    let c = n;
+    for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
+    table[n] = c >>> 0;
+  }
+  return table;
+})();
+
+function crc32(bytes: Uint8Array): number {
+  let c = 0xffffffff;
+  for (let i = 0; i < bytes.length; i++) c = CRC_TABLE[(c ^ bytes[i]) & 0xff] ^ (c >>> 8);
+  return (c ^ 0xffffffff) >>> 0;
+}
+
+function refusal(name: string): string | null {
+  if (name.startsWith("/")) return "an absolute name";
+  if (name.includes("\\")) return "a name with a backslash";
+  if (name.includes("\u0000")) return "a name with a NUL";
+  for (const segment of name.split("/")) {
+    if (segment === "" || segment === "." || segment === "..") {
+      return "a name with an empty, . or .. segment";
+    }
+  }
+  return null;
+}
+
+function sameBytes(a: Uint8Array, b: Uint8Array): boolean {
+  if (a.length !== b.length) return false;
+  for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
+  return true;
+}
+
+/**
+ * Every FILE entry of a ZIP archive, in central-directory order. A directory
+ * entry (a name ending in `/`) is not a file and is not returned. Throws
+ * `ZipReadError`.
+ */
+export function readZip(data: Uint8Array): ZipEntry[] {
+  const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
+  const n = data.length;
+  const u32 = (at: number) => view.getUint32(at, true);
+  const u16 = (at: number) => view.getUint16(at, true);
+  // The end-of-central-directory record: 22 bytes plus a comment of at most
+  // 65535, searched backwards so the LAST signature wins.
+  let eocd = -1;
+  const lowest = Math.max(0, n - 22 - 0xffff);
+  for (let at = n - 22; at >= lowest; at--) {
+    if (u32(at) === EOCD && at + 22 + u16(at + 20) === n) {
+      eocd = at;
+      break;
+    }
+  }
+  if (eocd < 0) throw new ZipReadError("is not a readable .zip archive");
+  const disk = u16(eocd + 4);
+  const cdDisk = u16(eocd + 6);
+  const hereCount = u16(eocd + 8);
+  const count = u16(eocd + 10);
+  const cdSize = u32(eocd + 12);
+  const cdOffset = u32(eocd + 16);
+  if (count === 0xffff || cdSize === 0xffffffff || cdOffset === 0xffffffff) {
+    throw new ZipReadError("is a ZIP64 archive, which case does not read");
+  }
+  if (disk !== 0 || cdDisk !== 0 || hereCount !== count) {
+    throw new ZipReadError("spans more than one disk, which case does not read");
+  }
+  if (count > MAX_ZIP_ENTRIES) {
+    throw new ZipReadError(`holds more than ${MAX_ZIP_ENTRIES} entries; refusing to read it`);
+  }
+  if (cdOffset + cdSize > eocd) throw new ZipReadError("is not a readable .zip archive");
+
+  const entries: ZipEntry[] = [];
+  const names = new Set<string>();
+  let total = 0;
+  let at = cdOffset;
+  for (let e = 0; e < count; e++) {
+    if (at + 46 > eocd || u32(at) !== CENTRAL) {
+      throw new ZipReadError("is not a readable .zip archive");
+    }
+    const madeBy = u16(at + 4);
+    const flags = u16(at + 8);
+    const method = u16(at + 10);
+    const crc = u32(at + 16);
+    const csize = u32(at + 20);
+    const usize = u32(at + 24);
+    const nameLen = u16(at + 28);
+    const extraLen = u16(at + 30);
+    const commentLen = u16(at + 32);
+    const external = u32(at + 38);
+    const localAt = u32(at + 42);
+    const rawName = data.subarray(at + 46, at + 46 + nameLen);
+    at += 46 + nameLen + extraLen + commentLen;
+    if (at > eocd) throw new ZipReadError("is not a readable .zip archive");
+    let name: string;
+    try {
+      name = utf8Strict(rawName);
+    } catch {
+      throw new ZipReadError(
+        "holds a name that is not valid UTF-8; rename it, then run case again",
+      );
+    }
+    if (names.has(name)) throw new ZipReadError(`holds ${name} twice`);
+    names.add(name);
+    if (name.endsWith("/")) continue;
+    if (name.split("/").length - 1 > MAX_ZIP_DEPTH) {
+      throw new ZipReadError(
+        `is nested more than ${MAX_ZIP_DEPTH} folders deep; refusing to read it`,
+      );
+    }
+    let refused = refusal(name);
+    if (refused === null && flags & 0x1) refused = "an encrypted entry";
+    if (
+      refused === null &&
+      madeBy >> 8 === UNIX &&
+      ((external >>> 16) & S_IFMT) === S_IFLNK
+    ) {
+      refused = "a symbolic link";
+    }
+    if (refused === null && method !== 0 && method !== 8) {
+      refused = "a compression method other than stored or deflate";
+    }
+    if (refused !== null) {
+      entries.push({ name, refused, data: null });
+      continue;
+    }
+
+    total += usize;
+    if (total > MAX_ZIP_BYTES) {
+      throw new ZipReadError(
+        `holds more than ${MAX_ZIP_BYTES} bytes uncompressed; refusing to read it`,
+      );
+    }
+    const unreadable = () => new ZipReadError(`holds ${name}, which cannot be read`);
+    if (localAt + 30 > cdOffset || u32(localAt) !== LOCAL) throw unreadable();
+    const localNameLen = u16(localAt + 26);
+    const localExtraLen = u16(localAt + 28);
+    if (!sameBytes(data.subarray(localAt + 30, localAt + 30 + localNameLen), rawName)) {
+      throw unreadable();
+    }
+    const start = localAt + 30 + localNameLen + localExtraLen;
+    if (start + csize > cdOffset) throw unreadable();
+    const packed = data.subarray(start, start + csize);
+    let body: Uint8Array;
+    if (method === 0) {
+      body = packed;
+    } else {
+      // `info: true` hands back the engine, whose `bytesWritten` is how many
+      // input bytes the stream consumed. Anything left over is data after the
+      // end of the deflate stream, which Python's `unused_data` refuses too.
+      let result: { buffer: Buffer; engine: { bytesWritten: number } };
+      try {
+        result = inflateRawSync(packed, {
+          info: true,
+          maxOutputLength: usize + 1,
+        }) as unknown as { buffer: Buffer; engine: { bytesWritten: number } };
+      } catch {
+        throw unreadable();
+      }
+      if (result.engine.bytesWritten !== packed.length) throw unreadable();
+      body = new Uint8Array(result.buffer.buffer, result.buffer.byteOffset, result.buffer.length);
+    }
+    if (body.length !== usize || crc32(body) !== crc) throw unreadable();
+    entries.push({ name, refused: null, data: body.slice() });
+  }
+  return entries;
+}

package/vectors/README.md

@@ -107,13 +107,66 @@ happened, and the reason this file moved here.
 
 | Key | Meaning |
 |---|---|
-| `ruleVersion` | Bumped when a case's expectation changes **or a case is added**. An implementation asserts the version it was written against, so a table it does not understand fails loudly rather than passing vacuously. Currently **3**. |
-| `kernelEventClasses` | The four per-event outcome classes a verification result can carry, named so a case can be written as counts rather than as a hand-built bundle. |
+| `ruleVersion` | Bumped when a case's expectation changes **or a case is added**. An implementation asserts the version it was written against, so a table it does not understand fails loudly rather than passing vacuously. Currently **6**: `evidence-bundle.md` §5.4.5, where a record withheld by a readable declaration is counted apart and holds the pack at nothing. |
+| `kernelEventClasses` | The six per-event outcome classes a verification result can carry, named so a case can be written as counts rather than as a hand-built bundle. The two added at rule version 6, `withheldBound` and `withheldUnbound`, carry an `eventId` and a `payloadBindingOk`; `howToUse` says how an adapter builds them. |
 | `cases[].kernel` | The verification result to grade, as class counts. |
 | `cases[].logs` | The disclosure account: how many records the bundle presents, and how many disclosed usable bytes. |
 | `cases[].omissions` | Completeness notes — a checkpoint's signed `leaf_count` against the records the pack presents for it. |
-| `cases[].expect` | The verdict, the three counts, and how many causes the grade must carry. |
+| `cases[].expect` | The verdict, the three counts, how many causes the grade must carry, and, from rule version 6, `recordsWithheld` where a case pins it. |
 
 Nothing in a case names a transport. A directory read and a browser drop produce
 the same account and must produce the same verdict; a table that encoded one
 surface's mechanism would let the other one drift while staying green.
+
+## `case-file-index.json`
+
+How a reader compares a case file's index, `contents.json`, with the folder it
+sits at the root of (`evidence-bundle.md` §5.6.3, v1.20 A25). One pure function
+in both kernels, `readCaseFileIndex` in [`../src/case-file.ts`](../src/case-file.ts)
+and `read_case_file_index` in `src/sanning_proof/case_file.py`, asserted against
+this table by [`../test/case-file-index.test.ts`](../test/case-file-index.test.ts)
+**and by the Python kernel against the same bytes**, in
+`tests/test_case_file_index.py`. Each row gives the index bytes as hex (or
+`null` for a folder with no index) and every file in the folder with its
+SHA-256, and expects the index state, its `case_id` label and the findings in
+the pinned order. **No finding moves a verdict**: the index is never an input
+to the case reading, and a surface that took one from it would be trusting
+whoever assembled the drop. Written by
+`tools/gen-vectors/gen_case_file_index_vectors.py`, whose expectations are
+literals.
+
+## `case-split.json`
+
+How `case` turns the folders it was handed into packs: the verify page's
+splitting rule (`sanning-io/verify-page` `src/lib/case.ts` `splitCase`,
+`src/lib/pack.ts` `assemble`) over a walk whose rules are the command's own,
+restated in [`../src/cli.ts`](../src/cli.ts) and `src/sanning_proof/cli.py`
+because the page's code lives in the page. Two tables and the limits:
+
+- `trees`: a directory tree to create (files, folders, symbolic links, a name
+  given as bytes, or a folder filled with N empty files), the arguments, and
+  the packs, the outside paths and the paths that are not regular files, or
+  the refusal. The rows include the decision file unpacked, a symbolic link
+  under `logs/`, each side of the depth and entry limits, names that are not
+  UTF-8, and (v1.20, A25) a case file unpacked and as a `.zip` argument, with
+  the entries a reader lists and never reads and the archives it refuses.
+- `parses`: how `bundle.json` bytes become JSON through the reader `verify`
+  and `case` share, including each side of the nesting limit.
+- `limits`: the walk depth (32), the entry count (50,000) and the JSON nesting
+  depth (512), which both kernels assert they state.
+
+Asserted by [`../test/case-cli.test.ts`](../test/case-cli.test.ts) **and by
+the Python kernel against the same trees**, in `tests/test_case_cli.py`, and
+run through both commands by `cross-kernel/run_case_cli.sh`.
+
+Every expectation is a literal written in
+`tools/gen-vectors/gen_case_split_vectors.py`, not the output of either
+kernel. The splitting rows were graded against the page's own code when the
+table was first written. The walk rows (links, depth, entry count, names, a
+`.zip` argument) and the reader rows are the command's own rules, and the page
+does not share them: it opens a dropped `.zip`, drops a leading byte-order
+mark, and reads whatever the browser hands it for a link, a bad name or a deep
+tree. One splitting row disagrees with the page on purpose: `record-file-names` keeps
+a record named `__proto__`, which the page's `assemble` loses because it keys
+disclosed content by a plain object. A page that vendors this table fails that
+row until it keys content by a `Map`.