@openwop/openwop-conformance 2.46.6 → 2.46.8

package/src/lib/eval-trajectory.ts

@@ -0,0 +1,146 @@
+/**
+ * RFC 0244 — the suite's reference grader for eval trajectories and trials.
+ *
+ * Pure functions, no I/O. `v2-eval-trajectory-trials.test.ts` uses them to
+ * recompute what a host must report, so a host's grade is checked against the
+ * normative definition rather than against itself:
+ *
+ *   gradeTrajectory   §A — the four match modes, `forbidden`, `argsMatch`,
+ *                     and the mode-independent `matchedCount` (a maximum
+ *                     one-to-one matching, ignoring order).
+ *   passAtK / passHatK / scoreStderr   §B — the normative formulas.
+ *
+ * @see RFCS/0244-eval-trajectories-trials-and-recorded-fixtures.md
+ * @see schemas/v2/agent-eval-suite.schema.json, schemas/v2/eval-summary.schema.json
+ */
+
+export type ArgsMatch = 'ignore' | 'exact' | 'subset';
+export type MatchMode = 'exact' | 'in-order' | 'any-order' | 'subset';
+export interface ExpectedCall { readonly tool: string; readonly args?: unknown; readonly argsMatch?: ArgsMatch }
+export interface ObservedCall { readonly tool: string; readonly inputs?: unknown }
+export interface TrajectorySpec { readonly match: MatchMode; readonly calls: readonly ExpectedCall[]; readonly forbidden?: readonly string[] }
+export interface TrajectoryGrade {
+  readonly mode: MatchMode;
+  readonly matched: boolean;
+  readonly expectedCount: number;
+  readonly observedCount: number;
+  readonly matchedCount: number;
+  readonly forbiddenCount: number;
+}
+
+const isRecord = (v: unknown): v is Record<string, unknown> => v !== null && typeof v === 'object' && !Array.isArray(v);
+
+/** JSON deep equality, key-order insensitive. */
+export function jsonEqual(a: unknown, b: unknown): boolean {
+  if (a === b) return true;
+  if (Array.isArray(a) && Array.isArray(b)) return a.length === b.length && a.every((x, i) => jsonEqual(x, b[i]));
+  if (isRecord(a) && isRecord(b)) {
+    const ka = Object.keys(a);
+    return ka.length === Object.keys(b).length && ka.every((k) => k in b && jsonEqual(a[k], b[k]));
+  }
+  return false;
+}
+
+/** `subset`: every key path in `expected` is present in `actual` with a JSON-deep-equal value. */
+export function jsonSubset(expected: unknown, actual: unknown): boolean {
+  if (isRecord(expected)) return isRecord(actual) && Object.keys(expected).every((k) => k in actual && jsonSubset(expected[k], actual[k]));
+  return jsonEqual(expected, actual);
+}
+
+export function callMatches(e: ExpectedCall, o: ObservedCall): boolean {
+  if (e.tool !== o.tool) return false;
+  const mode = e.argsMatch ?? 'ignore';
+  if (mode === 'ignore' || e.args === undefined) return true;
+  return mode === 'exact' ? jsonEqual(e.args, o.inputs) : jsonSubset(e.args, o.inputs);
+}
+
+/** Size of a maximum one-to-one matching (Kuhn's augmenting paths). */
+function maxMatching(expected: readonly ExpectedCall[], observed: readonly ObservedCall[]): number {
+  const owner: (number | undefined)[] = new Array(observed.length).fill(undefined);
+  const augment = (i: number, seen: boolean[]): boolean => {
+    for (let j = 0; j < observed.length; j++) {
+      if (seen[j] || !callMatches(expected[i]!, observed[j]!)) continue;
+      seen[j] = true;
+      if (owner[j] === undefined || augment(owner[j]!, seen)) { owner[j] = i; return true; }
+    }
+    return false;
+  };
+  let n = 0;
+  for (let i = 0; i < expected.length; i++) if (augment(i, new Array(observed.length).fill(false))) n++;
+  return n;
+}
+
+function inOrder(expected: readonly ExpectedCall[], observed: readonly ObservedCall[]): boolean {
+  let j = 0;
+  for (const e of expected) {
+    while (j < observed.length && !callMatches(e, observed[j]!)) j++;
+    if (j === observed.length) return false;
+    j++;
+  }
+  return true;
+}
+
+export function gradeTrajectory(spec: TrajectorySpec, observed: readonly ObservedCall[]): TrajectoryGrade {
+  const forbidden = new Set(spec.forbidden ?? []);
+  const forbiddenCount = observed.filter((o) => forbidden.has(o.tool)).length;
+  const matchedCount = maxMatching(spec.calls, observed);
+  let modeOk: boolean;
+  switch (spec.match) {
+    case 'exact': modeOk = observed.length === spec.calls.length && spec.calls.every((e, i) => callMatches(e, observed[i]!)); break;
+    case 'in-order': modeOk = inOrder(spec.calls, observed); break;
+    case 'any-order': modeOk = matchedCount === spec.calls.length; break;
+    case 'subset': modeOk = observed.every((o) => spec.calls.some((e) => callMatches(e, o))); break;
+  }
+  return { mode: spec.match, matched: modeOk && forbiddenCount === 0, expectedCount: spec.calls.length, observedCount: observed.length, matchedCount, forbiddenCount };
+}
+
+/**
+ * C(a, k) / C(b, k) as an exact rational, converted to a double once. The
+ * ratio telescopes to Π (a−i)/(b−i) for i < k, so no factorial is formed and
+ * the result is stable for large n (pass^k(200, 100, 50) ≈ 2.2e-19).
+ */
+function binomRatio(a: number, b: number, k: number): number {
+  if (k < 0 || k > b) throw new RangeError(`k=${k} outside 0..${b}`);
+  if (k > a) return 0;
+  let num = 1n;
+  let den = 1n;
+  for (let i = 0; i < k; i++) { num *= BigInt(a - i); den *= BigInt(b - i); }
+  return ratioToNumber(num, den);
+}
+
+/** num/den (num ≤ den, both > 0) to the nearest double, without overflowing either. */
+function ratioToNumber(num: bigint, den: bigint): number {
+  if (num === 0n) return 0;
+  const shift = den.toString(2).length - num.toString(2).length + 64;
+  const q = (num << BigInt(shift)) / den;
+  let r = Number(q);
+  let e = shift;
+  while (e > 1000) { r *= 2 ** -1000; e -= 1000; }
+  return r * 2 ** -e;
+}
+
+/** pass@k = 1 − C(n−c, k) / C(n, k); 0 when c = 0. */
+export function passAtK(n: number, c: number, k: number): number {
+  if (c === 0) return 0;
+  return 1 - binomRatio(n - c, n, k);
+}
+
+/** pass^k = C(c, k) / C(n, k); 0 when c < k. */
+export function passHatK(n: number, c: number, k: number): number {
+  if (c < k) return 0;
+  return binomRatio(c, n, k);
+}
+
+/** RFC 0244 §B tolerance: a host value agrees with the formula when |host − ref| ≤ 1e-9 · max(1, |ref|). */
+export function withinTolerance(host: number, ref: number): boolean {
+  return Number.isFinite(host) && Math.abs(host - ref) <= 1e-9 * Math.max(1, Math.abs(ref));
+}
+
+/** Sample sd of the per-task mean scores over √T; undefined when T < 2. */
+export function scoreStderr(taskMeans: readonly number[]): number | undefined {
+  const t = taskMeans.length;
+  if (t < 2) return undefined;
+  const mean = taskMeans.reduce((a, b) => a + b, 0) / t;
+  const variance = taskMeans.reduce((a, b) => a + (b - mean) ** 2, 0) / (t - 1);
+  return Math.sqrt(variance) / Math.sqrt(t);
+}

package/src/scenarios/v2-eval-trajectory-trials.test.ts

@@ -0,0 +1,253 @@
+/**
+ * v2 — eval suites grade the tool path, run repeated trials, and fail closed
+ * on a fixture miss (RFC 0244; the rules are the `agents.evalSuite` facet
+ * description in `spec/v2/facets/agents.schema.json` and the
+ * `agent-eval-suite` / `eval-summary` / `run-event-payloads` descriptions).
+ *
+ * Server-free legs (always on):
+ *   1. The RFC 0244 suite, summary and `eval.scored` / `eval.started` fields
+ *      validate, and the RFC's negatives are refused — including the
+ *      content-free ones that are the v2 public test for SECURITY invariant
+ *      `eval-summary-no-content-leak` (a trajectory block naming a tool, a
+ *      prose `scoreLabel`).
+ *   2. The three eval-suite fixtures validate against the v2 suite schema.
+ *
+ * Host legs (gated on the facet member AND the fixture in discovery
+ * `fixtures`). Each fixture is an `AgentEvalSuite` under
+ * `fixtures/eval-suites/`, run against the scripted agent
+ * `core.conformance.agents.scripted` (contract in `fixtures.md`): every task's
+ * `input.script` is replayed as tool calls, so the trajectory is known to the
+ * suite in advance and the host's grade is checked against the reference
+ * grader in `lib/eval-trajectory.ts`, not against itself.
+ *   3. `conformance-eval-trajectory` — per-task `passed` and the trajectory
+ *      counts equal the reference grade; the log carries the scripted calls.
+ *   4. `conformance-eval-trials` — `trialCount` on `eval.started`, one
+ *      `eval.scored` per task per trial with contiguous `trialIndex`, and the
+ *      per-task `trials` block and `trialStats` equal the §B formulas.
+ *   5. `conformance-eval-fixture-miss` — the missed task reports
+ *      `fixtureMissCount: 1` and `passed: false`; the served one passes.
+ *      That no live tool ran is seam-gated (RFC 0244 G2) and not asserted.
+ *   6. Refusal — a host advertising `agents.evalSuite` without `trials` that
+ *      lists `conformance-eval-trials` refuses it at createRun with
+ *      `400 validation_error`.
+ *
+ * @see RFCS/0244-eval-trajectories-trials-and-recorded-fixtures.md
+ */
+
+import { describe, it, expect } from 'vitest';
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { driver, type OpenWOPResponse } from '../lib/driver.js';
+import { v2Discovery, v2Validator, v2RefValidator } from '../lib/v2.js';
+import { softSkip } from '../lib/soft-skip.js';
+import { req } from '../lib/requirement-ids.js';
+import { readErrorCode } from '../lib/error-envelope.js';
+import { majorProfile } from '../lib/major-profile.js';
+import { isFixtureAdvertised } from '../lib/fixtures.js';
+import { FIXTURES_DIR } from '../lib/paths.js';
+import { pollTerminal, readEvents, isRecord, type Ev } from '../lib/fixture-run-observer.js';
+import { gradeTrajectory, passAtK, passHatK, scoreStderr, withinTolerance, type TrajectorySpec } from '../lib/eval-trajectory.js';
+
+const PROFILE = majorProfile(2);
+const UNREADABLE = 'v2 discovery unreachable — /.well-known/openwop did not answer 200 JSON under OpenWOP-Version: 2.0';
+const DOC = 'agents.schema.json §evalSuite (RFC 0244)';
+const AGENT = 'core.conformance.agents.scripted';
+const SUITE_URI = (id: string) => `https://conformance.openwop.dev/eval-suites/${id}`;
+const ID_SHAPE = 'openwop.requirement.0244.wire-shape';
+const ID_CONTENT_FREE = 'openwop.requirement.0244.content-free';
+const ID_FIXTURES = 'openwop.requirement.0244.fixtures-valid';
+const ID_TRAJECTORY = 'openwop.requirement.0244.trajectory-graded';
+const ID_TRIALS = 'openwop.requirement.0244.trials-per-task';
+const ID_TRIAL_STATS = 'openwop.requirement.0244.trial-stats-formula';
+const ID_MISS = 'openwop.requirement.0244.fixture-miss-fails';
+const ID_REFUSED = 'openwop.requirement.0244.unadvertised-feature-refused';
+
+interface Task { taskId: string; input: { script: Array<{ tool: string; arguments?: unknown }>; output: string }; expected: { kind: string; trajectory?: TrajectorySpec } }
+interface Suite { tasks: Task[]; trials?: { count: number; k?: number; passRule?: string } }
+const fixture = (id: string): Suite => JSON.parse(readFileSync(join(FIXTURES_DIR, 'eval-suites', `${id}.json`), 'utf8')) as Suite;
+
+const suiteSchema = v2Validator('agent-eval-suite');
+const summarySchema = v2Validator('eval-summary');
+const scored = v2RefValidator('run-event-payloads.schema.json#/$defs/evalScored');
+const started = v2RefValidator('run-event-payloads.schema.json#/$defs/evalStarted');
+
+async function http(fn: () => Promise<OpenWOPResponse>): Promise<OpenWOPResponse | null> {
+  try { return await fn(); } catch { return null; }
+}
+
+async function evalSuiteFacet(): Promise<{ doc: Record<string, unknown> | null; facet: Record<string, unknown> | null }> {
+  let doc: Record<string, unknown> | null;
+  try { doc = await v2Discovery(); } catch { doc = null; }
+  const agents = doc?.['agents'];
+  const facet = isRecord(agents) && isRecord(agents['evalSuite']) ? agents['evalSuite'] : null;
+  return { doc, facet };
+}
+
+type Observed = { kind: 'skip'; disposition: 'inapplicable' | 'blocked'; reason: string } | { kind: 'ok'; events: Ev[]; summary: Record<string, unknown> };
+
+const runs = new Map<string, Promise<Observed>>();
+/** One eval run per fixture per worker: two legs that read the same run share it. */
+function runSuite(id: string): Promise<Observed> {
+  let p = runs.get(id);
+  if (p === undefined) { p = runSuiteOnce(id); runs.set(id, p); }
+  return p;
+}
+
+async function runSuiteOnce(id: string): Promise<Observed> {
+  const res = await http(() => driver.post(PROFILE.runsPath, { mode: 'eval', evalSuiteRef: SUITE_URI(id), agentId: AGENT }));
+  if (res === null) return { kind: 'skip', disposition: 'blocked', reason: 'POST /runs unreachable (fetch failed)' };
+  if (res.status === 429) return { kind: 'skip', disposition: 'blocked', reason: 'POST /runs answered 429 — the run budget, not the wire' };
+  const runId = isRecord(res.json) && typeof res.json['runId'] === 'string' ? res.json['runId'] : undefined;
+  if ((res.status !== 201 && res.status !== 202) || runId === undefined) return { kind: 'skip', disposition: 'blocked', reason: `createRun of eval suite ${id} answered ${res.status} ${readErrorCode(res.json) ?? ''}`.trim() };
+  const terminal = await pollTerminal(PROFILE, runId, { timeoutMs: 60_000 });
+  if (terminal.snapshot['status'] !== 'completed') return { kind: 'skip', disposition: 'blocked', reason: `eval run ${runId} ended ${String(terminal.snapshot['status'])}, not completed` };
+  const s = await http(() => driver.get(`${PROFILE.runsPath}/${PROFILE.idSegment(runId)}/eval-summary`));
+  if (s === null || s.status !== 200 || !isRecord(s.json)) return { kind: 'skip', disposition: 'blocked', reason: `GET …/eval-summary answered ${s?.status ?? 'nothing'}` };
+  return { kind: 'ok', events: await readEvents(PROFILE, runId), summary: s.json };
+}
+
+const taskEntry = (summary: Record<string, unknown>, taskId: string): Record<string, unknown> | undefined =>
+  (Array.isArray(summary['tasks']) ? (summary['tasks'] as unknown[]) : []).filter(isRecord).find((t) => t['taskId'] === taskId);
+
+describe('v2 RFC 0244 — wire shape (server-free)', () => {
+  it('the RFC 0244 suite, summary and event fields validate and the negatives are refused', () => {
+    const traj = fixture('conformance-eval-trajectory');
+    const bad = (mut: (s: Suite) => void): boolean => { const s = structuredClone(traj) as Suite; mut(s); return suiteSchema(s).ok; };
+    expect(bad((s) => { (s.tasks[0]!.expected.trajectory as { match: string }).match = 'superset'; }), req(ID_SHAPE, DOC, 'trajectory.match "superset" MUST be refused (use any-order)')).toBe(false);
+    expect(bad((s) => { s.trials = { count: 0 }; }), req(ID_SHAPE, DOC, 'trials.count 0 MUST be refused')).toBe(false);
+    expect(bad((s) => { (s.tasks[0]!.expected.trajectory as unknown as Record<string, unknown>)['weight'] = 0.5; }), req(ID_SHAPE, DOC, 'an unknown trajectory member MUST be refused')).toBe(false);
+
+    const good = { taskId: 't', score: 1, passed: true, trialIndex: 2, scoreLabel: 'pass', evaluatorType: 'deterministic' };
+    expect(scored(good).ok, req(ID_SHAPE, DOC, `eval.scored with trialIndex/scoreLabel/evaluatorType MUST validate: ${scored(good).errors}`)).toBe(true);
+    expect(started({ suiteId: 'core.openwop.evals.x', suiteVersion: '1.0.0', taskCount: 2, modes: ['golden'], trialCount: 3 }).ok, req(ID_SHAPE, DOC, 'eval.started.trialCount MUST validate')).toBe(true);
+    expect(scored({ ...good, evaluatorType: 'gpt' }).ok, req(ID_SHAPE, DOC, 'an unknown evaluatorType MUST be refused')).toBe(false);
+
+    const summary = {
+      suiteId: 'core.openwop.evals.x', suiteVersion: '1.0.0', aggregateScore: 0.5, passed: false, taskCount: 1, passedCount: 0,
+      tasks: [{ taskId: 't', score: 0.8, passed: false,
+        trajectory: { mode: 'in-order', matched: false, expectedCount: 2, observedCount: 3, matchedCount: 1, forbiddenCount: 1 },
+        trials: { n: 5, passedCount: 4, meanScore: 0.8, passAtK: 1, passHatK: 0.4 },
+        efficiency: { toolCallCount: 3, modelCallCount: 1, inputTokens: 10, outputTokens: 5 }, fixtureMissCount: 0 }],
+      trialStats: { k: 3, passAtK: 1, passHatK: 0.4 },
+    };
+    expect(summarySchema(summary).ok, req(ID_SHAPE, DOC, `an RFC 0244 EvalSummary MUST validate: ${summarySchema(summary).errors}`)).toBe(true);
+  });
+
+  it('the RFC 0244 additions are content-free (eval-summary-no-content-leak)', () => {
+    const base = { suiteId: 'core.openwop.evals.x', suiteVersion: '1.0.0', aggregateScore: 0, passed: false, taskCount: 1, passedCount: 0 };
+    const leak = { ...base, tasks: [{ taskId: 't', score: 0, passed: false, trajectory: { mode: 'exact', matched: false, expectedCount: 1, observedCount: 1, matchedCount: 0, forbiddenCount: 1, tools: ['billing:refund'] } }] };
+    expect(summarySchema(leak).ok, req(ID_CONTENT_FREE, DOC, 'a trajectory block naming tools MUST be refused')).toBe(false);
+    expect(scored({ taskId: 't', score: 1, passed: true, scoreLabel: 'Looks good to me' }).ok, req(ID_CONTENT_FREE, DOC, 'a prose scoreLabel MUST be refused')).toBe(false);
+  });
+
+  it('the eval-suite fixtures validate against the v2 suite schema', () => {
+    for (const id of ['conformance-eval-trajectory', 'conformance-eval-trials', 'conformance-eval-fixture-miss']) {
+      const r = suiteSchema(fixture(id));
+      expect(r.ok, req(ID_FIXTURES, 'fixtures.md §eval-suites', `${id} MUST validate: ${r.errors}`)).toBe(true);
+    }
+  });
+});
+
+describe('v2 RFC 0244 — host behaviour', () => {
+  it('a trajectory suite is graded per the reference grader', async () => {
+    const { doc, facet } = await evalSuiteFacet();
+    if (!doc) return softSkip('blocked', UNREADABLE);
+    if (facet?.['trajectory'] !== true) return softSkip('inapplicable', 'the host does not advertise agents.evalSuite.trajectory');
+    const id = 'conformance-eval-trajectory';
+    if (!isFixtureAdvertised(id)) return softSkip('inapplicable', `the host does not advertise fixture ${id}`);
+    const o = await runSuite(id);
+    if (o.kind === 'skip') return softSkip(o.disposition, o.reason);
+    const suite = fixture(id);
+    for (const t of suite.tasks) {
+      const want = gradeTrajectory(t.expected.trajectory!, t.input.script.map((c) => ({ tool: c.tool, inputs: c.arguments })));
+      const entry = taskEntry(o.summary, t.taskId);
+      expect(entry, req(ID_TRAJECTORY, DOC, `the summary MUST carry task ${t.taskId}`)).toBeDefined();
+      const got = isRecord(entry?.['trajectory']) ? entry['trajectory'] : {};
+      expect(got, req(ID_TRAJECTORY, DOC, `task ${t.taskId}: the trajectory counts MUST equal the reference grade`)).toEqual(want);
+      if (!want.matched) expect(entry?.['passed'], req(ID_TRAJECTORY, DOC, `task ${t.taskId}: an unmatched trajectory MUST NOT pass`)).toBe(false);
+      if (t.expected.kind === 'trajectory') expect(entry?.['score'], req(ID_TRAJECTORY, DOC, `task ${t.taskId}: a trajectory-only task scores 1 when matched, else 0`)).toBe(want.matched ? 1 : 0);
+    }
+    const calls = o.events.filter((e) => e.type === 'agent.tool-called' && e.payload['agentId'] === AGENT).length;
+    const scripted = suite.tasks.reduce((a, t) => a + t.input.script.length, 0);
+    expect(calls, req(ID_TRAJECTORY, DOC, `the log MUST carry one agent.tool-called per scripted call (${scripted})`)).toBe(scripted);
+  });
+
+  async function trialsRun(): Promise<{ skip: [Parameters<typeof softSkip>[0], string] } | { o: Extract<Observed, { kind: 'ok' }>; suite: Suite; n: number; k: number }> {
+    const { doc, facet } = await evalSuiteFacet();
+    if (!doc) return { skip: ['blocked', UNREADABLE] };
+    if (!isRecord(facet?.['trials'])) return { skip: ['inapplicable', 'the host does not advertise agents.evalSuite.trials'] };
+    const id = 'conformance-eval-trials';
+    if (!isFixtureAdvertised(id)) return { skip: ['inapplicable', `the host does not advertise fixture ${id}`] };
+    const o = await runSuite(id);
+    if (o.kind === 'skip') return { skip: [o.disposition, o.reason] };
+    const suite = fixture(id);
+    const n = suite.trials!.count;
+    return { o, suite, n, k: suite.trials!.k ?? n };
+  }
+
+  it('repeated trials emit one eval.scored per task per trial', async () => {
+    const r = await trialsRun();
+    if ('skip' in r) return softSkip(...r.skip);
+    const { o, suite, n } = r;
+    expect(o.events.find((e) => e.type === 'eval.started')?.payload['trialCount'], req(ID_TRIALS, DOC, 'eval.started MUST carry trialCount')).toBe(n);
+    for (const t of suite.tasks) {
+      const rows = o.events.filter((e) => e.type === 'eval.scored' && e.payload['taskId'] === t.taskId);
+      expect(rows.map((x) => x.payload['trialIndex']).sort(), req(ID_TRIALS, DOC, `task ${t.taskId}: trialIndex MUST be 0..${n - 1}, once each`)).toEqual([...Array(n).keys()]);
+    }
+  });
+
+  it('the per-task trials block and trialStats follow the §B formulas', async () => {
+    const r = await trialsRun();
+    if ('skip' in r) return softSkip(...r.skip);
+    const { o, suite, n, k } = r;
+    const means: number[] = [];
+    for (const t of suite.tasks) {
+      const rows = o.events.filter((e) => e.type === 'eval.scored' && e.payload['taskId'] === t.taskId);
+      const c = rows.filter((x) => x.payload['passed'] === true).length;
+      const mean = rows.length === 0 ? 0 : rows.reduce((a, x) => a + Number(x.payload['score']), 0) / rows.length;
+      means.push(mean);
+      const trials = taskEntry(o.summary, t.taskId)?.['trials'];
+      const got = isRecord(trials) ? trials : {};
+      expect(got['n'], req(ID_TRIAL_STATS, DOC, `task ${t.taskId}: trials.n`)).toBe(n);
+      expect(got['passedCount'], req(ID_TRIAL_STATS, DOC, `task ${t.taskId}: trials.passedCount MUST count the passed trials`)).toBe(c);
+      expect(withinTolerance(Number(got['passAtK']), passAtK(n, c, k)), req(ID_TRIAL_STATS, DOC, `task ${t.taskId}: passAtK MUST equal 1 − C(n−c,k)/C(n,k) within 1e-9 relative (got ${String(got['passAtK'])}, want ${passAtK(n, c, k)})`)).toBe(true);
+      expect(withinTolerance(Number(got['passHatK']), passHatK(n, c, k)), req(ID_TRIAL_STATS, DOC, `task ${t.taskId}: passHatK MUST equal C(c,k)/C(n,k) within 1e-9 relative (got ${String(got['passHatK'])}, want ${passHatK(n, c, k)})`)).toBe(true);
+      expect(withinTolerance(Number(got['meanScore']), mean), req(ID_TRIAL_STATS, DOC, `task ${t.taskId}: meanScore MUST equal the mean trial score within 1e-9 relative (got ${String(got['meanScore'])}, want ${mean})`)).toBe(true);
+    }
+    const stats = isRecord(o.summary['trialStats']) ? o.summary['trialStats'] : {};
+    const se = scoreStderr(means);
+    if (se !== undefined) expect(withinTolerance(Number(stats['scoreStderr']), se), req(ID_TRIAL_STATS, DOC, `trialStats.scoreStderr MUST equal sd(per-task means)/√T within 1e-9 relative (got ${String(stats['scoreStderr'])}, want ${se})`)).toBe(true);
+  });
+
+  it('a fixture miss under onMiss "fail" fails the trial and is counted', async () => {
+    const { doc, facet } = await evalSuiteFacet();
+    if (!doc) return softSkip('blocked', UNREADABLE);
+    if (facet?.['fixtureMatching'] !== true) return softSkip('inapplicable', 'the host does not advertise agents.evalSuite.fixtureMatching');
+    const id = 'conformance-eval-fixture-miss';
+    if (!isFixtureAdvertised(id)) return softSkip('inapplicable', `the host does not advertise fixture ${id}`);
+    const o = await runSuite(id);
+    if (o.kind === 'skip') return softSkip(o.disposition, o.reason);
+    const missed = taskEntry(o.summary, 'missed');
+    const served = taskEntry(o.summary, 'served');
+    expect(missed?.['fixtureMissCount'], req(ID_MISS, DOC, 'the missed task MUST report fixtureMissCount 1')).toBe(1);
+    expect(missed?.['passed'], req(ID_MISS, DOC, 'a miss under onMiss "fail" MUST fail the trial')).toBe(false);
+    expect(served?.['passed'], req(ID_MISS, DOC, 'a served task MUST still pass')).toBe(true);
+    expect(served?.['fixtureMissCount'] ?? 0, req(ID_MISS, DOC, 'a served task MUST NOT report a miss')).toBe(0);
+  });
+
+  it('a suite using an unadvertised RFC 0244 feature is refused at createRun', async () => {
+    const { doc, facet } = await evalSuiteFacet();
+    if (!doc) return softSkip('blocked', UNREADABLE);
+    if (facet === null) return softSkip('inapplicable', 'the host does not advertise agents.evalSuite (v2-eval-mode-unadvertised-refused covers it)');
+    if (isRecord(facet['trials'])) return softSkip('inapplicable', 'the host advertises agents.evalSuite.trials, so the trials suite is served');
+    const id = 'conformance-eval-trials';
+    if (!isFixtureAdvertised(id)) return softSkip('inapplicable', `the host does not advertise fixture ${id}`);
+    const res = await http(() => driver.post(PROFILE.runsPath, { mode: 'eval', evalSuiteRef: SUITE_URI(id), agentId: AGENT }));
+    if (res === null) return softSkip('blocked', 'POST /runs unreachable (fetch failed)');
+    if (res.status === 429) return softSkip('blocked', 'POST /runs answered 429 — the run budget, not the wire');
+    const runId = isRecord(res.json) && typeof res.json['runId'] === 'string' ? res.json['runId'] : undefined;
+    if (res.status === 201 && runId !== undefined) await http(() => driver.post(`${PROFILE.runsPath}/${PROFILE.idSegment(runId)}/cancel`, {}));
+    expect(res.status, req(ID_REFUSED, DOC, `a trials suite on a host without agents.evalSuite.trials MUST be refused with 400 (got ${res.status})`)).toBe(400);
+    expect(readErrorCode(res.json), req(ID_REFUSED, DOC, 'the refusal MUST carry validation_error')).toBe('validation_error');
+  });
+});