mlx-bun 0.0.10 → 0.0.12

package/src/eval/ifeval.ts

@@ -1,138 +1,81 @@
-// IFEval — programmatically-verifiable instruction-following scoring (Zhou et
-// al. 2023, "Instruction-Following Evaluation for Large Language Models"). Each
-// instance pairs a prompt with one or more *verifiable* instructions; a response
-// is scored by deterministic checks (no LLM judge), so before/after ORPO
-// comparisons are exact and reproducible.
+// Compatibility facade for the canonical IFEval scorer.
 //
-// This implements the high-frequency instruction families (a working subset of
-// the ~25 in the reference suite, extensible via INSTRUCTIONS). Strict scoring:
-// an instance passes (strict) iff every instruction passes; we also report the
-// per-instruction pass rate. The official metrics are prompt-level and
-// instruction-level accuracy — both derived from `scoreInstance`.
+// The complete verifier registry and strict/loose aggregation contract live in
+// `tasks/ifeval.ts`, which is also used by `scripts/eval.ts`. Keep this module
+// as the stable import surface used by `scripts/run-ifeval.ts` and older
+// callers; it must not grow a second verifier implementation.
 
-export interface IFEvalInstance {
-  /** The prompt text (used by instructions that reference it, e.g. "repeat the request"). */
-  prompt: string;
-  /** Instruction ids, e.g. "keywords:existence". */
-  instruction_id_list: string[];
-  /** Per-instruction kwargs, aligned with instruction_id_list. */
-  kwargs: Array<Record<string, unknown>>;
-}
+import {
+  scoreIfevalInstance,
+  scoreIfevalPairs,
+  type IfevalInstance,
+  type IfevalResult,
+} from "./tasks/ifeval";
+
+export {
+  SUPPORTED_INSTRUCTIONS,
+  looseClean,
+  scoreIfevalInstance,
+  scoreIfevalPairs,
+  stripThinking,
+  verifyResponse,
+  type IfevalCoverage,
+  type IfevalInstance,
+  type IfevalInstanceScore,
+  type IfevalPair,
+  type IfevalResult,
+  type IfevalVerification,
+} from "./tasks/ifeval";
+
+/** Historical spelling retained for callers of this module. */
+export type IFEvalInstance = IfevalInstance;
 
 export interface InstanceResult {
-  /** One pass/fail per instruction (aligned with instruction_id_list). */
+  /** Unknown instructions are true here per the canonical prompt contract. */
   perInstruction: boolean[];
-  /** Strict: all instructions passed. */
   followedAll: boolean;
+  /** Consult this field rather than interpreting unknown IDs as verified. */
+  unhandled: string[];
 }
 
-type Checker = (response: string, kw: Record<string, unknown>, inst: IFEvalInstance) => boolean;
-
-const words = (s: string): string[] => s.trim().split(/\s+/).filter(Boolean);
-const sentences = (s: string): string[] =>
-  s.split(/(?<=[.!?])\s+/).map((x) => x.trim()).filter(Boolean);
-const paragraphs = (s: string): string[] =>
-  s.split(/\n\s*\n/).map((x) => x.trim()).filter(Boolean);
-const norm = (s: string): string => s.toLowerCase();
-
-const cmp = (n: number, rel: string | undefined, target: number): boolean => {
-  switch (rel) {
-    case "at least": return n >= target;
-    case "at most": return n <= target;
-    case "less than": return n < target;
-    case "exactly": return n === target;
-    default: return n >= target; // IFEval default for "around"/unspecified is lenient ≥
-  }
-};
-
-/** The verifiable-instruction registry. Keyed by IFEval instruction_id. */
-export const INSTRUCTIONS: Record<string, Checker> = {
-  // ---- keywords ----
-  "keywords:existence": (r, kw) =>
-    (kw.keywords as string[]).every((k) => norm(r).includes(norm(k))),
-  "keywords:frequency": (r, kw) => {
-    const k = norm(kw.keyword as string);
-    const count = norm(r).split(k).length - 1;
-    return cmp(count, kw.relation as string, kw.frequency as number);
-  },
-  "keywords:forbidden_words": (r, kw) =>
-    !(kw.forbidden_words as string[]).some((k) => norm(r).includes(norm(k))),
-
-  // ---- length constraints ----
-  "length_constraints:number_words": (r, kw) =>
-    cmp(words(r).length, kw.relation as string, kw.num_words as number),
-  "length_constraints:number_sentences": (r, kw) =>
-    cmp(sentences(r).length, kw.relation as string, kw.num_sentences as number),
-  "length_constraints:number_paragraphs": (r, kw) =>
-    cmp(paragraphs(r).length, kw.relation as string, kw.num_paragraphs as number),
-
-  // ---- detectable format ----
-  "detectable_format:number_bullet_lists": (r, kw) => {
-    const bullets = r.split("\n").filter((l) => /^\s*[*\-]\s+/.test(l)).length;
-    return cmp(bullets, kw.relation as string, kw.num_bullets as number);
-  },
-  "detectable_format:number_highlighted_sections": (r, kw) => {
-    const hi = (r.match(/\*[^*\n]+\*/g) ?? []).length;
-    return cmp(hi, kw.relation as string, kw.num_highlights as number);
-  },
-  "detectable_format:title": (r) => /<<[^>\n]+>>/.test(r),
-  "detectable_format:json_format": (r) => {
-    const t = r.trim().replace(/^```(?:json)?\s*|\s*```$/g, "").trim();
-    try { JSON.parse(t); return true; } catch { return false; }
-  },
-
-  // ---- case ----
-  "change_case:english_lowercase": (r) => r === r.toLowerCase(),
-  "change_case:english_capital": (r) => r === r.toUpperCase(),
-
-  // ---- start / end ----
-  "startend:end_checker": (r, kw) =>
-    norm(r.trimEnd()).endsWith(norm((kw.end_phrase as string).trim())),
-  "startend:quotation": (r) => {
-    const t = r.trim();
-    return t.startsWith('"') && t.endsWith('"') && t.length >= 2;
-  },
-
-  // ---- punctuation ----
-  "punctuation:no_comma": (r) => !r.includes(","),
-};
-
-/** Score one IFEval instance (strict). Unknown instruction ids count as failed
- *  (so coverage gaps never inflate the score) — filter the dataset to the
- *  supported ids, or extend INSTRUCTIONS, to avoid that. */
-export function scoreInstance(instance: IFEvalInstance, response: string): InstanceResult {
-  const perInstruction = instance.instruction_id_list.map((id, i) => {
-    const check = INSTRUCTIONS[id];
-    if (!check) return false;
-    try { return check(response, instance.kwargs[i] ?? {}, instance); } catch { return false; }
-  });
-  return { perInstruction, followedAll: perInstruction.every(Boolean) };
+/**
+ * Historical strict-only helper, now delegated to the canonical scorer.
+ * New code should prefer `scoreIfevalInstance` for both strict and loose modes.
+ */
+export function scoreInstance(
+  instance: IFEvalInstance,
+  response: string,
+): InstanceResult {
+  const { strict } = scoreIfevalInstance(instance, response);
+  return {
+    perInstruction: strict.instructionPasses.map((result) => result ?? true),
+    followedAll: strict.pass,
+    unhandled: strict.unhandled,
+  };
 }
 
-export interface IFEvalReport {
+export interface IFEvalReport extends IfevalResult {
+  /** Historical alias for nTotal. */
   n: number;
-  /** Fraction of instances where ALL instructions were followed. */
+  /** Historical alias for strictAcc (prompt-level strict). */
   promptAccuracy: number;
-  /** Fraction of individual instructions followed. */
+  /** Historical alias for strictInstructionAcc. */
   instructionAccuracy: number;
 }
 
-/** Aggregate the official prompt-level + instruction-level accuracies over a set
- *  of (instance, response) pairs. */
-export function aggregate(pairs: Array<{ instance: IFEvalInstance; response: string }>): IFEvalReport {
-  let prompts = 0, instrFollowed = 0, instrTotal = 0;
-  for (const { instance, response } of pairs) {
-    const r = scoreInstance(instance, response);
-    if (r.followedAll) prompts++;
-    instrFollowed += r.perInstruction.filter(Boolean).length;
-    instrTotal += r.perInstruction.length;
-  }
+/**
+ * Historical aggregation helper with unambiguous canonical fields plus aliases.
+ * Unknown instructions do not fail prompt accuracy, are excluded from
+ * instruction accuracy, and remain visible in `coverage`.
+ */
+export function aggregate(
+  pairs: Array<{ instance: IFEvalInstance; response: string }>,
+): IFEvalReport {
+  const result = scoreIfevalPairs(pairs);
   return {
-    n: pairs.length,
-    promptAccuracy: pairs.length ? prompts / pairs.length : 0,
-    instructionAccuracy: instrTotal ? instrFollowed / instrTotal : 0,
+    ...result,
+    n: result.nTotal,
+    promptAccuracy: result.strictAcc,
+    instructionAccuracy: result.strictInstructionAcc,
   };
 }
-
-/** Instruction ids this implementation supports (filter datasets to these). */
-export const SUPPORTED_INSTRUCTIONS = new Set(Object.keys(INSTRUCTIONS));

package/src/eval/kl.ts

@@ -26,7 +26,7 @@ import { Dtype } from "../mlx/ffi";
 import * as ops from "../mlx/ops";
 import { Registry } from "../registry";
 import { Weights } from "../weights";
-import { loadModelConfig, type ModelConfig } from "../config";
+import { loadModelConfig, type ModelConfig, type TurboQuantScheme } from "../config";
 import { createModel, type RuntimeModel } from "../model/factory";
 import { loadTokenizer, type LoadedTokenizer } from "../tokenizer";
 import { maybeQuantizeKv } from "../generate";
@@ -223,10 +223,14 @@ function klScalar(p: Float32Array, q: Float32Array): number {
 }
 
 /** Prefill (bf16) → quantize → teacher-forced decode; return per-step logit
- *  distributions for the last `tokens.length - prefillLen` positions. */
+ *  distributions for the last `tokens.length - prefillLen` positions.
+ *  `kvOverride` replaces the default kvConfig-from-model-config scheme when
+ *  given — the seam evaluateKlKvArm uses to run a TurboQuant (or bf16-null)
+ *  arm instead of the env-flag self-flip. */
 function decodeArm(
   model: RuntimeModel, config: ModelConfig, tokens: number[], prefillLen: number,
   flag: string, value: string,
+  kvOverride?: Parameters<typeof maybeQuantizeKv>[1],
 ): Float32Array[] {
   const prev = process.env[flag];
   process.env[flag] = value;
@@ -237,7 +241,10 @@ function decodeArm(
       lp.dispose();
       // quantize the populated caches (mixed 4/8 per kv_config) — exactly
       // what generate() does between prefill and decode.
-      maybeQuantizeKv(cache, { kvConfig: config.kvQuant ?? undefined, quantizedKvStart: 0 });
+      maybeQuantizeKv(
+        cache,
+        kvOverride ?? { kvConfig: config.kvQuant ?? undefined, quantizedKvStart: 0 },
+      );
       const out: Float32Array[] = [];
       for (let i = prefillLen; i < tokens.length; i++) {
         const logits = model.forward([tokens[i]!], cache); // [1,1,V] quantized decode
@@ -292,3 +299,52 @@ export async function evaluateKlServingDecode(opts: {
     refLabel: `serving-decode self:${opts.flag}=${opts.refValue}→${opts.candValue} (${decodeSteps} steps)`,
   };
 }
+
+/** KV-scheme A/B on the teacher-forced serving-decode path: bf16 baseline
+ *  (no kvOverride) vs an explicit KV scheme (e.g. TurboQuant), same model /
+ *  same prompts / same weight load — the quality-vs-bpw curve gate
+ *  (scripts/eval-turboquant-curve.ts). Unlike evaluateKlServingDecode (env-flag
+ *  self-flip), the two arms differ by an actual GenerateOptions kv scheme
+ *  object, so this is the seam for schemes that aren't env-lever-shaped. */
+export async function evaluateKlKvArm(opts: {
+  candidate: string;
+  /** Undefined ⇒ bf16 (no quantization). */
+  candidateScheme?: { turboQuant?: TurboQuantScheme; kvBits?: number; kvConfig?: ModelConfig["kvQuant"] };
+  prompts: string[];
+  nPrompts?: number;
+  seqLen?: number;
+  decodeSteps?: number;
+}): Promise<KLResult> {
+  const t0 = Date.now();
+  const n = opts.nPrompts ?? 64;
+  const seqLen = opts.seqLen ?? 256;
+  const decodeSteps = opts.decodeSteps ?? 32;
+  const { model, tokenizer, config } = await loadRunnable(opts.candidate);
+  const tokenized = prepPrompts(opts.prompts, tokenizer, seqLen, n);
+  const noopFlag = "MLX_BUN_EVAL_KV_ARM_NOOP"; // decodeArm always flips a flag; this one is inert
+  const refKv = {}; // bf16: maybeQuantizeKv no-ops with no kvBits/kvConfig/turboQuant
+  const candKv = opts.candidateScheme
+    ? { ...opts.candidateScheme, quantizedKvStart: 0, kvConfig: opts.candidateScheme.kvConfig ?? undefined }
+    : {};
+
+  const perPrompt: Float32Array[] = [];
+  for (const tokens of tokenized) {
+    if (tokens.length < 2) continue;
+    const prefillLen = Math.max(1, tokens.length - decodeSteps);
+    const refSteps = decodeArm(model, config, tokens, prefillLen, noopFlag, "0", refKv);
+    const candSteps = decodeArm(model, config, tokens, prefillLen, noopFlag, "0", candKv);
+    const m = Math.min(refSteps.length, candSteps.length);
+    const kls = new Float32Array(m);
+    for (let s = 0; s < m; s++) kls[s] = klScalar(refSteps[s]!, candSteps[s]!);
+    perPrompt.push(kls);
+  }
+
+  const agg = aggregate(perPrompt);
+  return {
+    nPrompts: perPrompt.length, seqLen, meanKl: agg.mean, medianKl: agg.median, p95Kl: agg.p95,
+    elapsedSec: (Date.now() - t0) / 1000,
+    refLabel: opts.candidateScheme?.turboQuant
+      ? `bf16 vs turbo k${opts.candidateScheme.turboQuant.kBits}v${opts.candidateScheme.turboQuant.vBits} (${decodeSteps} steps)`
+      : `bf16 vs kv-scheme (${decodeSteps} steps)`,
+  };
+}

package/src/eval/quality-db.ts

@@ -5,7 +5,7 @@
 //
 // One row = one (model, config) measured on one task (or the aggregate
 // "capability" / "kl" rows). `config_json` records the perf levers that
-// were active (compile / fused-decode / perf-kernel) so a row is
+// were active (compile / fused-sdpa / kv-quant) so a row is
 // self-describing: which arm of the head-to-head produced this number.
 
 import { Database } from "bun:sqlite";
@@ -19,7 +19,7 @@ CREATE TABLE IF NOT EXISTS quality_runs (
   model_path TEXT NOT NULL,
   commit_sha TEXT,
   task TEXT NOT NULL,            -- 'kl' | 'gsm8k' | 'mmlu' | ... | 'capability'
-  config_json TEXT NOT NULL,     -- {compile,fusedDecode,perfKernel,...} active levers
+  config_json TEXT NOT NULL,     -- {compile,fusedSdpa,kvQuant,...} active levers
   n_samples INTEGER NOT NULL,
   -- task accuracy (percent 0-100); null for kl-only rows
   pct REAL,

package/src/eval/runner.ts

@@ -9,8 +9,12 @@ import { generate, type GenerateOptions } from "../generate";
 import * as ops from "../mlx/ops";
 import { ChatTemplate } from "../chat-template";
 import { loadModelConfig, type ModelConfig } from "../config";
-import { Weights } from "../weights";
-import { createModel, type RuntimeModel } from "../model/factory";
+import {
+  openModel,
+  type Glm52RuntimeOpenOptions,
+  type RuntimeModel,
+} from "../model/factory";
+import { isGlm52Config } from "../model/support";
 import { DiffusionGemmaModel } from "../model/diffusion-gemma";
 import { loadTokenizer, type LoadedTokenizer } from "../tokenizer";
 import { resolveModelDir } from "./kl";
@@ -65,11 +69,16 @@ export interface TaskModel {
   activeAdapters?: string[];
 }
 
-export async function loadTaskModel(query: string, adapterDir?: string): Promise<TaskModel> {
+export async function loadTaskModel(
+  query: string,
+  adapterDir?: string,
+  runtimeOptions: Glm52RuntimeOpenOptions = {},
+): Promise<TaskModel> {
   const dir = resolveModelDir(query);
   const config = await loadModelConfig(dir);
-  const weights = await Weights.open(dir);
-  const model = createModel(weights, config);
+  if (adapterDir && isGlm52Config(config))
+    throw new Error("GLM-5.2 does not support LoRA adapters");
+  const model = await openModel(dir, runtimeOptions);
   const activeAdapters: string[] = [];
   if (adapterDir) {
     // Mount a trained LoRA adapter so the eval measures base+adapter (e.g. the
@@ -99,11 +108,11 @@ export interface GenOpts {
    *  (its capability numbers are non-thinking; e.g. MiniCPM5 IFEval 64.7 is
    *  non-thinking). Set MLX_BUN_EVAL_THINK=1 or pass true to evaluate thinking mode. */
   enableThinking?: boolean;
-  /** Explicit KV-quant scheme (cli `generate --l2/--l3/--kv-quant`, resolved with
+  /** Explicit KV-quant scheme (cli `generate --l2/--kv-quant`, resolved with
    *  serve's semantics). Overrides the MLX_BUN_EVAL_KV_QUANT env default; an empty
    *  object forces bf16. When it names quantized KV, generation runs the product
    *  generate() path (same tokens as `serve` with the same scheme). */
-  kvScheme?: Pick<GenerateOptions, "kvBits" | "kvConfig" | "quantizedKvStart">;
+  kvScheme?: Pick<GenerateOptions, "kvBits" | "kvConfig" | "quantizedKvStart" | "turboQuant">;
 }
 
 /** Bit-exact greedy decode: raw `model.forward` + argmax loop, which matches mlx-lm
@@ -159,7 +168,7 @@ export async function generateText(tm: TaskModel, body: string, opts: GenOpts =
     (process.env.MLX_BUN_EVAL_KV_QUANT === "1" && tm.config.kvQuant?.length)
       ? { kvConfig: tm.config.kvQuant, quantizedKvStart: 0 } : undefined;
   const kvScheme = opts.kvScheme ?? envKv;
-  const kvActive = !!(kvScheme && (kvScheme.kvBits || kvScheme.kvConfig?.length));
+  const kvActive = !!(kvScheme && (kvScheme.kvBits || kvScheme.kvConfig?.length || kvScheme.turboQuant));
 
   // Parity default — greedy + full-precision KV + no sampler arm: decode bit-exactly
   // via the raw forward (matches mlx-lm token-for-token). Sampler arms / kv-quant fall

package/src/eval/tasks/ifeval.ts

@@ -5,13 +5,14 @@
 // 'banana'", "respond in JSON"). Each example carries one or more constraint
 // instructions; we generate a response and run the corresponding verifier.
 //
-// We report prompt-level pass rates (matches the official methodology):
-//   * strict — fraction of examples where ALL of the prompt's instructions
-//     pass on the raw response.
-//   * loose  — same but with response-cleaning preprocessing (strip leading
-//     "Sure, here is…" boilerplate and outer code fences).
-// The primary metric (`accuracy`) is prompt-level strict, headlined by
-// ifeval.py's IFEvalResult.__str__.
+// We report strict and loose metrics at both official granularities:
+//   * prompt — fraction of examples where ALL instructions pass;
+//   * instruction — fraction of supported individual checks that pass.
+// Loose mode first strips leading "Sure, here is…" boilerplate and outer code
+// fences. The primary metric (`accuracy`) remains prompt-level strict, as
+// headlined by ifeval.py's IFEvalResult.__str__. Unknown instruction IDs retain
+// optiq's prompt-pass behavior but are excluded from instruction accuracy and
+// reported through the coverage object.
 //
 // Faithful port note: several verifiers below read kwarg keys that the dataset
 // rows never populate (e.g. _check_capital_words_count / _check_letter_frequency
@@ -21,18 +22,56 @@
 
 import { generateText, loadJsonl, sampleIndices, type TaskModel } from "../runner";
 
-interface IfevalRow {
-  key: number;
+export interface IfevalInstance {
   prompt: string;
   instruction_id_list: string[];
   kwargs: Record<string, unknown>[];
 }
 
+interface IfevalRow extends IfevalInstance {
+  key: number;
+}
+
+export interface IfevalCoverage {
+  /** Prompts for which every instruction ID has a registered verifier. */
+  fullySupportedPrompts: number;
+  promptCoverage: number;
+  /** Registered instruction checks; unknown IDs are excluded from accuracy. */
+  supportedInstructions: number;
+  totalInstructions: number;
+  instructionCoverage: number;
+  unhandledInstructionCounts: Record<string, number>;
+}
+
 export interface IfevalResult {
   nTotal: number;
   strictAcc: number; // 0..1 — prompt-level strict pass rate
   looseAcc: number; // 0..1 — prompt-level loose pass rate
   accuracy: number; // 0..1 — primary metric == strictAcc (ifeval.py headline)
+  strictInstructionAcc: number;
+  looseInstructionAcc: number;
+  coverage: IfevalCoverage;
+}
+
+export interface IfevalVerification {
+  /**
+   * One result per requested instruction. null means the instruction ID is
+   * unhandled and therefore excluded from instruction-level accuracy.
+   */
+  instructionPasses: Array<boolean | null>;
+  /** OptiQ parity: unhandled IDs do not make the prompt fail. */
+  pass: boolean;
+  unhandled: string[];
+}
+
+export interface IfevalInstanceScore {
+  strict: IfevalVerification;
+  loose: IfevalVerification;
+}
+
+export interface IfevalPair {
+  instance: IfevalInstance;
+  response: string;
 }
 
 // ---------------------------------------------------------------------------
@@ -278,20 +317,24 @@ const VERIFIERS = new Map<string, Verifier>([
   ["detectable_content:postscript", alwaysPass],
 ]);
 
+export const SUPPORTED_INSTRUCTIONS = new Set(VERIFIERS.keys());
+
 /**
  * Verify one response against its instruction list.
- * Returns { pass, unhandled }. An instruction_id with no registered verifier is
- * treated as a PASS (does not affect `pass`) and only recorded in `unhandled` for
- * visibility — matching ifeval.py's `continue` policy EXACTLY, so our strict_acc
- * reproduces optiq's. (Our VERIFIERS map mirrors his `_VERIFIERS` 1:1, so unhandled
- * is empty for the standard IFEval set anyway; this only matters for parity safety.)
+ * `instructionPasses` is aligned with the requested IDs and uses null for
+ * unhandled IDs. An ID with no registered verifier does not affect prompt
+ * `pass` and is recorded in `unhandled` — matching ifeval.py's `continue`
+ * policy exactly, so strict prompt accuracy reproduces OptiQ's. The canonical
+ * aggregate excludes those nulls from instruction accuracy and reports them
+ * through coverage.
  */
 export function verifyResponse(
   response: string,
   instructionIds: string[],
   kwargsList: Kw[],
-): { pass: boolean; unhandled: string[] } {
+): IfevalVerification {
   const unhandled: string[] = [];
+  const instructionPasses: Array<boolean | null> = [];
   let pass = true;
   for (let i = 0; i < instructionIds.length; i++) {
     const iid = instructionIds[i]!;
@@ -299,15 +342,19 @@ export function verifyResponse(
     const verifier = VERIFIERS.get(iid);
     if (verifier === undefined) {
       unhandled.push(iid);
+      instructionPasses.push(null);
       continue; // ifeval.py: treat an unhandled instruction as PASS (don't touch `pass`)
     }
     try {
-      if (!verifier(response, kw)) pass = false;
+      const followed = verifier(response, kw);
+      instructionPasses.push(followed);
+      if (!followed) pass = false;
     } catch {
+      instructionPasses.push(false);
       pass = false;
     }
   }
-  return { pass, unhandled };
+  return { instructionPasses, pass, unhandled };
 }
 
 /** Loose-mode preprocessing — mirrors ifeval.py `_loose_clean`. */
@@ -328,6 +375,92 @@ export function stripThinking(response: string): string {
   return idx === -1 ? response : response.slice(idx + "</think>".length);
 }
 
+/** Score one generated response with the canonical strict + loose contract. */
+export function scoreIfevalInstance(
+  instance: IfevalInstance,
+  rawResponse: string,
+): IfevalInstanceScore {
+  const response = stripThinking(rawResponse);
+  return {
+    strict: verifyResponse(
+      response,
+      instance.instruction_id_list,
+      instance.kwargs,
+    ),
+    loose: verifyResponse(
+      looseClean(response),
+      instance.instruction_id_list,
+      instance.kwargs,
+    ),
+  };
+}
+
+/**
+ * Canonical IFEval aggregation.
+ *
+ * Prompt accuracy follows the OptiQ reference behavior: an unknown instruction
+ * is recorded but does not fail its prompt. Instruction accuracy has a stricter
+ * denominator contract: unknown instructions are excluded and their coverage
+ * is reported explicitly, so an incomplete verifier registry cannot silently
+ * inflate that metric.
+ */
+export function scoreIfevalPairs(pairs: IfevalPair[]): IfevalResult {
+  let strictPrompts = 0;
+  let loosePrompts = 0;
+  let fullySupportedPrompts = 0;
+  let strictInstructions = 0;
+  let looseInstructions = 0;
+  let supportedInstructions = 0;
+  let totalInstructions = 0;
+  const unhandledCounts = new Map<string, number>();
+
+  for (const { instance, response } of pairs) {
+    const score = scoreIfevalInstance(instance, response);
+    if (score.strict.pass) strictPrompts++;
+    if (score.loose.pass) loosePrompts++;
+    if (score.strict.unhandled.length === 0) fullySupportedPrompts++;
+    totalInstructions += instance.instruction_id_list.length;
+
+    for (let i = 0; i < score.strict.instructionPasses.length; i++) {
+      const strict = score.strict.instructionPasses[i];
+      if (strict === null) continue;
+      supportedInstructions++;
+      if (strict) strictInstructions++;
+      if (score.loose.instructionPasses[i]) looseInstructions++;
+    }
+    for (const id of score.strict.unhandled)
+      unhandledCounts.set(id, (unhandledCounts.get(id) ?? 0) + 1);
+  }
+
+  const nTotal = pairs.length;
+  const strictAcc = nTotal ? strictPrompts / nTotal : 0;
+  const looseAcc = nTotal ? loosePrompts / nTotal : 0;
+  return {
+    nTotal,
+    strictAcc,
+    looseAcc,
+    accuracy: strictAcc,
+    strictInstructionAcc: supportedInstructions
+      ? strictInstructions / supportedInstructions
+      : 0,
+    looseInstructionAcc: supportedInstructions
+      ? looseInstructions / supportedInstructions
+      : 0,
+    coverage: {
+      fullySupportedPrompts,
+      promptCoverage: nTotal ? fullySupportedPrompts / nTotal : 0,
+      supportedInstructions,
+      totalInstructions,
+      instructionCoverage: totalInstructions
+        ? supportedInstructions / totalInstructions
+        : 0,
+      unhandledInstructionCounts: Object.fromEntries(
+        [...unhandledCounts.entries()].sort(([a], [b]) => a.localeCompare(b)),
+      ),
+    },
+  };
+}
+
 export async function evaluateIfeval(
   tm: TaskModel,
   opts: { nSamples?: number; maxTokens?: number; seed?: number; frozen?: boolean } = {},
@@ -348,7 +481,7 @@ export async function evaluateIfeval(
 
   let nStrict = 0;
   let nLoose = 0;
-  const unhandledCounts = new Map<string, number>();
+  const pairs: IfevalPair[] = [];
 
   for (let k = 0; k < idx.length; k++) {
     const item = rows[idx[k]!]!;
@@ -356,14 +489,16 @@ export async function evaluateIfeval(
     const kwList = item.kwargs ?? iids.map(() => ({}));
 
     const raw = await generateText(tm, item.prompt, { maxTokens, useChat: true });
-    const response = stripThinking(raw);
-
-    const strict = verifyResponse(response, iids, kwList);
-    const loose = verifyResponse(looseClean(response), iids, kwList);
-    for (const u of strict.unhandled) unhandledCounts.set(u, (unhandledCounts.get(u) ?? 0) + 1);
+    const instance: IfevalInstance = {
+      prompt: item.prompt,
+      instruction_id_list: iids,
+      kwargs: kwList,
+    };
+    const score = scoreIfevalInstance(instance, raw);
+    pairs.push({ instance, response: raw });
 
-    if (strict.pass) nStrict++;
-    if (loose.pass) nLoose++;
+    if (score.strict.pass) nStrict++;
+    if (score.loose.pass) nLoose++;
 
     if ((k + 1) % 10 === 0 || k + 1 === idx.length)
       process.stderr.write(
@@ -372,8 +507,12 @@ export async function evaluateIfeval(
   }
   process.stderr.write("\n");
 
-  if (unhandledCounts.size) {
-    const top = [...unhandledCounts.entries()].sort((a, b) => b[1] - a[1]);
+  const result = scoreIfevalPairs(pairs);
+  const unhandledCounts = Object.entries(
+    result.coverage.unhandledInstructionCounts,
+  );
+  if (unhandledCounts.length) {
+    const top = unhandledCounts.sort((a, b) => b[1] - a[1]);
     process.stderr.write(
       "  ifeval unported instruction ids (treated as PASS, per optiq): " +
         top.map(([k, v]) => `${k}(${v})`).join(", ") +
@@ -381,8 +520,5 @@ export async function evaluateIfeval(
     );
   }
 
-  const nTotal = idx.length;
-  const strictAcc = nTotal ? nStrict / nTotal : 0;
-  const looseAcc = nTotal ? nLoose / nTotal : 0;
-  return { nTotal, strictAcc, looseAcc, accuracy: strictAcc };
+  return result;
 }