@openwop/openwop-conformance 2.37.0 → 2.38.0

package/src/lib/jcs.ts

@@ -0,0 +1,229 @@
+/**
+ * RFC 0212 — canonical JSON is RFC 8785 (JCS), and the input MUST be I-JSON.
+ *
+ * Every signature and digest in the corpus is over these bytes: pack
+ * signatures (`ed25519-canonical-json`), the certification-bundle attestation,
+ * `discovery.sha256`, `witnessSha256`, and the RFC 0150 semantic request digest.
+ * `conformance/vectors/jcs-v1.json` is the normative test of this module and of
+ * any other-language implementation.
+ *
+ * Two entry points, because two of the refusals are invisible after a parse:
+ *
+ *   - `canonicalJSON(value)` — the VALUE boundary. Refuses what a native value
+ *     can still show: non-finite numbers, non-JSON values (undefined, functions,
+ *     bigint, symbols, class instances such as Date) and lone surrogates.
+ *   - `parseIJson(text)` — the TEXT boundary. `JSON.parse` keeps the last of two
+ *     duplicate names and rounds `9007199254740993` to `…992` silently; both are
+ *     refused here, before coercion. Read a document you will re-canonicalize
+ *     through this, not `JSON.parse`.
+ *
+ * Why not `Object.keys(v).sort()` + `JSON.stringify` with no checks (the code
+ * this replaces): it is JCS for well-formed input — the default sort IS UTF-16
+ * code-unit order and ES number/string serialization IS JCS §3.2.2 — but it
+ * turns NaN into `null`, emits the text `undefined`, and signs a rounded
+ * integer. Each of those is a document two verifiers read differently with no
+ * error on either side.
+ */
+
+export type JcsRefusalKind = 'duplicate-name' | 'lone-surrogate' | 'integer-out-of-range' | 'non-finite' | 'not-json';
+
+export class JcsRefusal extends Error {
+  readonly kind: JcsRefusalKind;
+  constructor(kind: JcsRefusalKind, message: string) {
+    super(`RFC 0212 §B refusal (${kind}): ${message}`);
+    this.name = 'JcsRefusal';
+    this.kind = kind;
+  }
+}
+
+/** A JSON number literal, matched in place (sticky) so parsing stays linear. */
+const NUMBER = /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?/y;
+
+/** ±(2^53 − 1), for the integer-literal refusal at the text boundary. */
+const MAX_EXACT_BIG = 2n ** 53n - 1n;
+
+/**
+ * RFC 8785 §3.2.3 — compare by UTF-16 code units. Explicit rather than the
+ * `Array.prototype.sort` default so the rule is visible at every call site, and
+ * never `localeCompare`: collation is locale-dependent (Czech sorts `ch` after
+ * `h`; English puts `a` before `A` and ignores `-`), so a digest computed with
+ * it depends on the machine that computed it.
+ */
+export function codeUnitCompare(a: string, b: string): number {
+  const n = Math.min(a.length, b.length);
+  for (let i = 0; i < n; i += 1) {
+    const d = a.charCodeAt(i) - b.charCodeAt(i);
+    if (d !== 0) return d;
+  }
+  return a.length - b.length;
+}
+
+function assertWellFormed(s: string, where: string): void {
+  for (let i = 0; i < s.length; i += 1) {
+    const u = s.charCodeAt(i);
+    if (u >= 0xd800 && u <= 0xdbff) {
+      const next = s.charCodeAt(i + 1);
+      if (next >= 0xdc00 && next <= 0xdfff) { i += 1; continue; }
+      throw new JcsRefusal('lone-surrogate', `lone high surrogate U+${u.toString(16).toUpperCase()} in ${where}`);
+    }
+    if (u >= 0xdc00 && u <= 0xdfff) throw new JcsRefusal('lone-surrogate', `lone low surrogate U+${u.toString(16).toUpperCase()} in ${where}`);
+  }
+}
+
+function serializeNumber(n: number): string {
+  if (!Number.isFinite(n)) throw new JcsRefusal('non-finite', `${String(n)} is not a JSON number`);
+  // No magnitude check here: a double is already exact, and JCS serializes it
+  // (RFC 8785 Appendix B includes 9007199254740994). The integer-range refusal
+  // is about a LITERAL that no double holds, which only the text shows — see
+  // `parseIJson`.
+  return Object.is(n, -0) ? '0' : String(n);
+}
+
+function isPlainObject(v: object): boolean {
+  const proto = Object.getPrototypeOf(v) as unknown;
+  return proto === Object.prototype || proto === null;
+}
+
+/** RFC 8785 serialization of an I-JSON value. Throws `JcsRefusal` instead of coercing. */
+export function canonicalJSON(value: unknown): string {
+  if (value === null) return 'null';
+  switch (typeof value) {
+    case 'boolean': return value ? 'true' : 'false';
+    case 'number': return serializeNumber(value);
+    case 'string': assertWellFormed(value, 'a string'); return JSON.stringify(value);
+    case 'object': break;
+    default: throw new JcsRefusal('not-json', `a ${typeof value} is not a JSON value`);
+  }
+  if (Array.isArray(value)) {
+    const parts: string[] = [];
+    for (let i = 0; i < value.length; i += 1) {
+      if (!(i in value)) throw new JcsRefusal('not-json', 'a sparse array hole is not a JSON value');
+      parts.push(canonicalJSON(value[i]));
+    }
+    return `[${parts.join(',')}]`;
+  }
+  if (!isPlainObject(value)) throw new JcsRefusal('not-json', `a ${(value as object).constructor?.name ?? 'non-plain'} object is not a JSON value`);
+  const obj = value as Record<string, unknown>;
+  const keys = Object.keys(obj).sort(codeUnitCompare);
+  return `{${keys.map((k) => { assertWellFormed(k, 'a member name'); return `${JSON.stringify(k)}:${canonicalJSON(obj[k])}`; }).join(',')}}`;
+}
+
+/**
+ * Parse JSON text, refusing what `JSON.parse` would silently accept or change:
+ * duplicate member names, integer literals outside ±(2^53 − 1), literals that
+ * overflow to ±Infinity, and lone surrogates. Members are defined, not
+ * assigned, so a member named `__proto__` is data, not a prototype write.
+ */
+export function parseIJson(text: string): unknown {
+  let i = 0;
+  const fail = (m: string): never => { throw new JcsRefusal('not-json', `${m} at offset ${i}`); };
+  const ws = (): void => { while (i < text.length && (text[i] === ' ' || text[i] === '\t' || text[i] === '\n' || text[i] === '\r')) i += 1; };
+
+  const str = (): string => {
+    if (text[i] !== '"') fail('expected a string');
+    i += 1;
+    let out = '';
+    for (;;) {
+      if (i >= text.length) fail('unterminated string');
+      const c = text[i];
+      i += 1;
+      if (c === '"') break;
+      if (c === '\\') {
+        const e = text[i];
+        i += 1;
+        switch (e) {
+          case '"': out += '"'; break;
+          case '\\': out += '\\'; break;
+          case '/': out += '/'; break;
+          case 'b': out += '\b'; break;
+          case 'f': out += '\f'; break;
+          case 'n': out += '\n'; break;
+          case 'r': out += '\r'; break;
+          case 't': out += '\t'; break;
+          case 'u': {
+            const h = text.slice(i, i + 4);
+            if (!/^[0-9a-fA-F]{4}$/.test(h)) fail('bad \\u escape');
+            out += String.fromCharCode(parseInt(h, 16));
+            i += 4;
+            break;
+          }
+          default: fail('bad escape');
+        }
+      } else {
+        if ((c as string).charCodeAt(0) < 0x20) fail('raw control character in a string');
+        out += c;
+      }
+    }
+    assertWellFormed(out, 'a string');
+    return out;
+  };
+
+  const num = (): number => {
+    NUMBER.lastIndex = i;
+    const m = NUMBER.exec(text);
+    if (m === null) return fail('bad number');
+    i += m[0].length;
+    if (m[2] === undefined && m[3] === undefined) {
+      const b = BigInt(m[0]);
+      if (b > MAX_EXACT_BIG || b < -MAX_EXACT_BIG) throw new JcsRefusal('integer-out-of-range', `integer literal ${m[0]} is outside ±(2^53 − 1)`);
+    }
+    const v = Number(m[0]);
+    if (!Number.isFinite(v)) throw new JcsRefusal('non-finite', `${m[0]} overflows to a non-finite number`);
+    return v;
+  };
+
+  const val = (): unknown => {
+    ws();
+    const c = text[i];
+    if (c === '{') {
+      i += 1;
+      const obj: Record<string, unknown> = {};
+      ws();
+      if (text[i] === '}') { i += 1; return obj; }
+      for (;;) {
+        ws();
+        const k = str();
+        if (Object.prototype.hasOwnProperty.call(obj, k)) throw new JcsRefusal('duplicate-name', `duplicate member name ${JSON.stringify(k)}`);
+        ws();
+        if (text[i] !== ':') fail('expected :');
+        i += 1;
+        Object.defineProperty(obj, k, { value: val(), enumerable: true, writable: true, configurable: true });
+        ws();
+        const d = text[i];
+        i += 1;
+        if (d === '}') return obj;
+        if (d !== ',') fail('expected , or }');
+      }
+    }
+    if (c === '[') {
+      i += 1;
+      const arr: unknown[] = [];
+      ws();
+      if (text[i] === ']') { i += 1; return arr; }
+      for (;;) {
+        arr.push(val());
+        ws();
+        const d = text[i];
+        i += 1;
+        if (d === ']') return arr;
+        if (d !== ',') fail('expected , or ]');
+      }
+    }
+    if (c === '"') return str();
+    if (text.startsWith('true', i)) { i += 4; return true; }
+    if (text.startsWith('false', i)) { i += 5; return false; }
+    if (text.startsWith('null', i)) { i += 4; return null; }
+    if (c === '-' || (c !== undefined && c >= '0' && c <= '9')) return num();
+    return fail('unexpected token');
+  };
+
+  const v = val();
+  ws();
+  if (i !== text.length) fail('trailing data');
+  return v;
+}
+
+/** JCS bytes of a JSON text, with every §B refusal applied. */
+export function canonicalizeText(text: string): string {
+  return canonicalJSON(parseIJson(text));
+}

package/src/lib/llm-cache-key-recipe.ts

@@ -10,30 +10,20 @@
  *     RFC 0041 §E SECURITY-invariant probe (intra-host reproducibility +
  *     non-recipe-field invariance + Phase 4 advertisement alignment).
  *
- * `canonicalize` mirrors RFC 8785 JCS-style output (sorted keys, no
- * whitespace, preserved array order). Hosts that have a real JCS library
- * available SHOULD prefer it; this helper is for the conformance side,
- * not the host side. Keep in sync with `spec/v1/replay.md` §B.
+ * `canonicalize` is RFC 8785 JCS with the RFC 0212 I-JSON refusal set (the
+ * suite's one implementation, `./jcs.ts`), and `tools[]` sorts by UTF-16 code
+ * units — never `localeCompare`, which orders `get_weather` / `getWeather`
+ * differently per locale and so breaks the TS/Python/Go agreement RFC 0150 §C
+ * asks for. Keep in sync with `spec/v1/replay.md` §B.
  */
 
 import { createHash } from 'node:crypto';
 import { driver } from './driver.js';
+import { canonicalJSON, codeUnitCompare } from './jcs.js';
 
-/** RFC 8785 JCS-style canonicalization (subset suitable for the recipe
- *  fields). Sorted keys recursively; no whitespace; preserved array order;
- *  strings JSON-encoded verbatim (no NFC normalization — the recipe
- *  inputs in our test seam are ASCII). */
+/** RFC 8785 JCS (RFC 0212). Throws `JcsRefusal` on a non-I-JSON value. */
 export function canonicalize(value: unknown): string {
-  if (value === null) return 'null';
-  if (typeof value === 'boolean' || typeof value === 'number') return JSON.stringify(value);
-  if (typeof value === 'string') return JSON.stringify(value);
-  if (Array.isArray(value)) return '[' + value.map((v) => canonicalize(v)).join(',') + ']';
-  if (typeof value === 'object') {
-    const obj = value as Record<string, unknown>;
-    const keys = Object.keys(obj).sort();
-    return '{' + keys.map((k) => `${JSON.stringify(k)}:${canonicalize(obj[k])}`).join(',') + '}';
-  }
-  return JSON.stringify(value);
+  return canonicalJSON(value);
 }
 
 /** @deprecated RETIRED v1 projection (pre RFC 0150 §C). Kept ONLY so a
@@ -44,7 +34,7 @@ export function canonicalize(value: unknown): string {
 export function projectRecipe(raw: Record<string, unknown>): Record<string, unknown> {
   const out: Record<string, unknown> = { provider: raw.provider, model: raw.model, messages: raw.messages };
   if (Array.isArray(raw.tools) && raw.tools.length > 0) {
-    out.tools = [...(raw.tools as Array<{ name: string }>)].sort((a, b) => a.name.localeCompare(b.name));
+    out.tools = [...(raw.tools as Array<{ name: string }>)].sort((a, b) => codeUnitCompare(a.name, b.name));
   }
   if (typeof raw.temperature === 'number') out.temperature = raw.temperature;
   if (typeof raw.topP === 'number') out.topP = raw.topP;
@@ -76,7 +66,7 @@ export const SEMANTIC_REQUEST_RECIPE_V2 = 'openwop-semantic-request-v2';
 export function projectSemanticRequestV2(raw: Record<string, unknown>): Record<string, unknown> {
   const request: Record<string, unknown> = { messages: raw.messages };
   if (Array.isArray(raw.tools) && raw.tools.length > 0) {
-    request.tools = [...(raw.tools as Array<{ name: string }>)].sort((a, b) => a.name.localeCompare(b.name));
+    request.tools = [...(raw.tools as Array<{ name: string }>)].sort((a, b) => codeUnitCompare(a.name, b.name));
   }
   for (const k of ['temperature', 'topP', 'topK', 'maxOutputTokens', 'seed'] as const) {
     if (typeof raw[k] === 'number') request[k] = raw[k];

package/src/lib/requirement-ledger.ts

@@ -87,15 +87,68 @@ const journal: LedgerEntry[] = [];
  * dispositions throws: RFC 0148 §A says **exactly one** disposition per
  * requirement, and a silent last-write-wins would let a later soft-skip
  * overwrite an earlier real failure — the failure mode in reverse.
+ *
+ * `extras.fold` is the one exception, for the per-`it` rows `setup.ts` records
+ * when several `it` legs witness ONE requirement id; see its docblock below.
+ */
+/**
+ * How certifiable each disposition is, least first. `readLedgerFile` has always
+ * resolved a cross-worker disagreement this way — "one worker said it failed"
+ * outranks "another said it passed", and an unresolvable disagreement must never
+ * round toward certification. `fold` below applies the SAME rule in-worker.
  */
+const CERTIFIABILITY_RANK: Record<Disposition, number> = {
+  'executed-fail': 0,
+  blocked: 1,
+  'executed-pass': 2,
+  skipped: 3,
+  inapplicable: 4,
+};
+
 export function recordRequirement(
   requirementId: string,
   disposition: Disposition,
   detail?: string,
-  extras?: { assertionCount?: number; scenarioFile?: string; evidence?: RowEvidence },
+  extras?: {
+    assertionCount?: number;
+    scenarioFile?: string;
+    evidence?: RowEvidence;
+    /**
+     * FOLD instead of throw when this id already has a disposition (2.37.0).
+     *
+     * The strict contract above — one disposition per requirement per run,
+     * contradiction throws — is right for a scenario that classifies ITSELF:
+     * two hand-written verdicts for one id is an authoring bug, and the throw
+     * is how it surfaces.
+     *
+     * It is wrong for the per-`it` rows `setup.ts` records, because the corpus
+     * deliberately witnesses ONE requirement with SEVERAL `it` legs: 27
+     * scenario files hand `req()` a module-level `const ID`, so every leg in
+     * the file records under the same id. There the throw is not a guard —
+     * `setup.ts` wraps the call in `try {} catch {}` ("never fail a test for
+     * bookkeeping"), so the second verdict was silently DISCARDED, and, worse,
+     * discarded before it reached the JSONL sink.
+     *
+     * MEASURED, `v2-run-bulk-cancel.test.ts` on a tier-2 host: leg 1 passed
+     * (3 assertions), leg 2 failed on its 7th. The file row recorded
+     * `executed-fail` with 10 assertions and the detail "one or more assertions
+     * in the file failed"; `openwop.requirement.0170.run-bulk-cancel` recorded
+     * `executed-pass` with 3. The failing leg's message — which names the
+     * requirement AND prints the offending entry — existed, was computed by
+     * `resolveItRecord`, and was thrown away here. The operator had to
+     * hand-probe every assertion in the file against production to find out
+     * what had failed.
+     *
+     * Folding by `CERTIFIABILITY_RANK` makes the surviving row the least
+     * certifiable of the legs, carrying THAT leg's detail, with the legs'
+     * assertion counts summed — the same answer `readLedgerFile` would reach
+     * from the sink lines, so the in-memory ledger and the file agree.
+     */
+    fold?: true;
+  },
 ): void {
   const prior = ledger.get(requirementId);
-  if (prior !== undefined && prior.disposition !== disposition) {
+  if (prior !== undefined && prior.disposition !== disposition && extras?.fold !== true) {
     throw new Error(
       `RFC 0148 §A: ${requirementId} already recorded as '${prior.disposition}', now '${disposition}'. ` +
         'Exactly one disposition per requirement per run.',
@@ -107,7 +160,7 @@ export function recordRequirement(
         'Anything other than executed-pass MUST say why, or the ledger records an outcome nobody can act on.',
     );
   }
-  const entry: LedgerEntry = {
+  let entry: LedgerEntry = {
     requirementId,
     disposition,
     ...(detail === undefined ? {} : { detail }),
@@ -115,6 +168,27 @@ export function recordRequirement(
     ...(extras?.scenarioFile === undefined ? {} : { scenarioFile: extras.scenarioFile }),
     ...(extras?.evidence === undefined || disposition !== 'executed-pass' ? {} : { evidence: extras.evidence }),
   };
+  if (prior !== undefined && extras?.fold === true) {
+    // The least-certifiable leg wins the disposition and keeps its own detail;
+    // a tie keeps whichever side actually said something. Counts sum, because
+    // both legs really did assert against the target for this one requirement.
+    const keepPrior = CERTIFIABILITY_RANK[prior.disposition] <= CERTIFIABILITY_RANK[disposition];
+    const winner = keepPrior ? prior : entry;
+    const loser = keepPrior ? entry : prior;
+    const count = (prior.assertionCount ?? 0) + (extras?.assertionCount ?? 0);
+    const keptDetail = winner.detail ?? loser.detail;
+    const hasCount = prior.assertionCount !== undefined || extras?.assertionCount !== undefined;
+    entry = {
+      requirementId,
+      disposition: winner.disposition,
+      ...(keptDetail === undefined ? {} : { detail: keptDetail }),
+      ...(hasCount ? { assertionCount: count } : {}),
+      ...(winner.scenarioFile === undefined ? {} : { scenarioFile: winner.scenarioFile }),
+      // `evidence` is only meaningful on a pass; a fold that lands anywhere
+      // else drops it, exactly as the constructor above does.
+      ...(winner.disposition === 'executed-pass' && winner.evidence !== undefined ? { evidence: winner.evidence } : {}),
+    };
+  }
   ledger.set(requirementId, entry);
   journal.push(entry);
   // File sink (RFC 0148 acceptance item 2, S6). The in-memory map lives in a
@@ -159,7 +233,11 @@ export function readLedgerFile(path: string): readonly LedgerEntry[] {
     }
     if (typeof e.requirementId !== 'string' || !DISPOSITIONS.includes(e.disposition)) continue;
     const prior = merged.get(e.requirementId);
-    if (prior === undefined || rank[e.disposition] < rank[prior.disposition]) merged.set(e.requirementId, e);
+    // `<=`, not `<`: a per-`it` FOLD (2.37.0) appends the cumulative row after
+    // the leg rows it folded, so on an equal disposition the LAST line is the
+    // one carrying the summed `assertionCount`. For genuinely duplicate lines
+    // the two are identical and the choice is a no-op.
+    if (prior === undefined || rank[e.disposition] <= rank[prior.disposition]) merged.set(e.requirementId, e);
   }
   return [...merged.values()].sort((a, b) => a.requirementId.localeCompare(b.requirementId));
 }

package/src/lib/scenario-disposition.ts

@@ -118,8 +118,13 @@ export function resolveItRecord(
    * sites were not measured and v1 bundles are read through its EOS.
    */
   blockedStands = false,
+  /** The `it` title, so a failed row says WHICH leg of the requirement failed. */
+  testName?: string,
 ): { disposition: Disposition; detail?: string } {
-  if (state === 'fail') return { disposition: 'executed-fail', detail: `the test executed and failed: ${(firstError ?? 'no message').slice(0, 300)}` };
+  if (state === 'fail') {
+    const where = testName === undefined || testName.trim() === '' ? '' : ` in "${testName.slice(0, 120)}"`;
+    return { disposition: 'executed-fail', detail: `the test executed and failed${where}: ${(firstError ?? 'no message').slice(0, 300)}` };
+  }
   if (state === 'pass' && assertionCalls > 0) {
     // `blockedDespiteAssertions` (soft-skip.ts): the leg says its setup
     // assertions are not the requirement, and the requirement went unobserved.
@@ -142,14 +147,45 @@ export function resolveItRecord(
   return { disposition: 'skipped', detail: 'vitest skipped the test (ctx.skip / it.skip) without a recorded gate reason' };
 }
 
+/** One failed `it` in a file: its title and its first error message. */
+export interface TestFailure {
+  readonly name: string;
+  readonly message?: string;
+}
+
+/**
+ * The `executed-fail` detail for a file row: WHICH cases failed, and what the
+ * first one said.
+ *
+ * Until 2.37.0 this was the fixed string "one or more assertions in the file
+ * failed". A tier-2 host read exactly that for `v2-run-bulk-cancel` — 10
+ * assertions, no case name, no message, and nothing else in the record for that
+ * file — and had to hand-probe every assertion in the file against production to
+ * find out what had happened. A bundle row whose only detail is that sentence is
+ * undiagnosable by construction, and every future flicker in any scenario had
+ * the same problem.
+ */
+export function failureDetail(failures: readonly TestFailure[], failedCount: number): string {
+  if (failures.length === 0) {
+    return `${failedCount} test(s) in the file failed; the runner captured no message`;
+  }
+  const head = failures[0]!;
+  const named = failures.slice(0, 3).map((f) => `"${f.name.slice(0, 120)}"`).join(', ');
+  const more = failures.length > 3 ? ` (+${failures.length - 3} more)` : '';
+  const msg = head.message === undefined || head.message.trim() === '' ? 'no message' : head.message.slice(0, 400);
+  return `${failures.length} test(s) failed — ${named}${more}; first failure: ${msg}`;
+}
+
 /** Worker half: fold a file's per-test states (+ any gate-recorded reason) into
  *  the ONE disposition the file records. */
 export function fileDisposition(
   states: readonly FileTestState[],
   gateReason: 'inapplicable' | 'skipped' | undefined,
   assertionCount?: number,
+  failures: readonly TestFailure[] = [],
 ): { disposition: Disposition; detail?: string } {
-  if (states.some((s) => s === 'fail')) return { disposition: 'executed-fail', detail: 'one or more assertions in the file failed' };
+  const failed = states.filter((s) => s === 'fail').length;
+  if (failed > 0) return { disposition: 'executed-fail', detail: failureDetail(failures, failed) };
   if (states.some((s) => s === 'pass')) {
     // A test that early-returned through `behaviorGate` is reported by vitest
     // as a pass with zero assertions. When EVERY passing test in the file did
@@ -195,6 +231,8 @@ export function resolveFileRecord(
   assertionCount: number,
   noted: { kind: 'inapplicable' | 'skipped' | 'blocked'; reason: string } | null,
   specCoherenceFile?: string,
+  /** The failed cases, so an `executed-fail` row NAMES them (2.37.0). */
+  failures: readonly TestFailure[] = [],
 ): { disposition: Disposition; detail?: string } {
   // A scenario whose subject is the CORPUS, skipped because the published
   // tarball does not bundle spec/v1/. RFC 0148 §A: `blocked` is defined over
@@ -211,7 +249,7 @@ export function resolveFileRecord(
   ) {
     return { disposition: 'inapplicable', detail: SPEC_COHERENCE_DETAIL };
   }
-  let { disposition, detail } = fileDisposition(states, gateReason, assertionCount);
+  let { disposition, detail } = fileDisposition(states, gateReason, assertionCount, failures);
   if (disposition === 'executed-pass' && assertionCount === 0) {
     if (noted !== null) {
       disposition = noted.kind;