mlx-bun 0.0.5 → 0.0.6

package/src/eval/runner.ts

@@ -6,6 +6,7 @@
 import { homedir } from "node:os";
 import { readFileSync } from "node:fs";
 import { generate, type GenerateOptions } from "../generate";
+import * as ops from "../mlx/ops";
 import { ChatTemplate } from "../chat-template";
 import { loadModelConfig, type ModelConfig } from "../config";
 import { Weights } from "../weights";
@@ -56,17 +57,34 @@ export interface TaskModel {
    *  task's own (greedy) default — used to run the whole suite under one sampler
    *  arm (e.g. the v2 curve vs the default chat recipe) for the degradation gate. */
   samplerOverride?: SamplerArm;
+  /** Mounted-adapter ids to ACTIVATE on every forward/generate. Mounting an adapter
+   *  only stores it on the linears; `loraState.active` is what actually applies it,
+   *  and generation resets it to [] after each call. So generateText re-passes this,
+   *  and direct-forward tasks (MMLU) re-assert it — else the eval silently runs base. */
+  activeAdapters?: string[];
 }
 
-export async function loadTaskModel(query: string): Promise<TaskModel> {
+export async function loadTaskModel(query: string, adapterDir?: string): Promise<TaskModel> {
   const dir = resolveModelDir(query);
   const config = await loadModelConfig(dir);
   const weights = await Weights.open(dir);
   const model = createModel(weights, config);
+  const activeAdapters: string[] = [];
+  if (adapterDir) {
+    // Mount a trained LoRA adapter so the eval measures base+adapter (e.g. the
+    // ORPO-fine-tuned "after" model). NOTE: mount() only STORES the adapter on each
+    // linear — it does NOT activate it. Activation is `loraState.active`; without
+    // setting it the forward runs BASE (bit-identical to no-adapter). So activate it
+    // here for direct-forward tasks; generateText re-passes `activeAdapters` too.
+    const { AdapterManager } = await import("../lora");
+    await new AdapterManager(model).mount("eval-adapter", adapterDir);
+    activeAdapters.push("eval-adapter");
+    model.loraState.active = [...activeAdapters];
+  }
   const tokenizer = await loadTokenizer(dir);
   let template: ChatTemplate | null = null;
   try { template = await ChatTemplate.load(dir); } catch { template = null; }
-  return { model, tokenizer, template, config, dir };
+  return { model, tokenizer, template, config, dir, activeAdapters: activeAdapters.length ? activeAdapters : undefined };
 }
 
 export interface GenOpts {
@@ -76,21 +94,73 @@ export interface GenOpts {
   /** Sampling overrides (default greedy). Lets the HLG/diversity evals vary the
    *  sampler — temperature, top-p/k, seed, hlg — while sharing this prompt path. */
   sampler?: Partial<Pick<GenerateOptions, "temperature" | "topP" | "topK" | "seed" | "hlg" | "curve">>;
+  /** Chat-template `enable_thinking`. Defaults to OFF for eval parity with optiq
+   *  (its capability numbers are non-thinking; e.g. MiniCPM5 IFEval 64.7 is
+   *  non-thinking). Set MLX_BUN_EVAL_THINK=1 or pass true to evaluate thinking mode. */
+  enableThinking?: boolean;
+}
+
+/** Bit-exact greedy decode: raw `model.forward` + argmax loop, which matches mlx-lm
+ *  token-for-token (gen8k: 8000/8000). The product `generate()` decode wrapper
+ *  (forwardHidden/logitsFromHidden + pipelined sampler) diverges from the raw forward
+ *  on near-ties past ~32 tokens, so the EVAL decodes bit-exactly itself rather than
+ *  alter the serving path. Greedy + full-precision KV only. */
+function greedyDecodeBitExact(tm: TaskModel, ids: number[], maxTokens: number): string {
+  if (tm.activeAdapters) tm.model.loraState.active = tm.activeAdapters; // apply the mounted adapter
+  const cache = tm.model.makeCache();
+  try {
+    if (ids.length > 1) tm.model.forward(ids.slice(0, -1), cache).dispose(); // prefill
+    let last = ids[ids.length - 1]!;
+    const eos = new Set(tm.config.eosTokenIds);
+    const out: number[] = [];
+    for (let i = 0; i < maxTokens; i++) {
+      const lg = tm.model.forward([last], cache);
+      const am = ops.argmaxAxis(lg, -1); // GPU argmax (mlx-lm greedy: lowest-index tie-break)
+      lg.dispose();
+      const best = ops.itemUint32(am);
+      am.dispose();
+      if (eos.has(best)) break; // mlx-lm halts on EOS and excludes it from the output
+      out.push(best);
+      last = best;
+    }
+    return tm.tokenizer.decode(out, true);
+  } finally {
+    for (const c of cache) c.dispose();
+  }
 }
 
 /** Complete `body` via the model's real quantized-KV path. Greedy by default;
  *  pass `opts.sampler` to drive temperature/HLG/etc. */
 export async function generateText(tm: TaskModel, body: string, opts: GenOpts = {}): Promise<string> {
   const maxTokens = opts.maxTokens ?? 256;
-  const text = opts.useChat !== false && tm.template
-    ? tm.template.render([{ role: "user", content: body }], { addGenerationPrompt: true })
+  const enableThinking = opts.enableThinking ?? process.env.MLX_BUN_EVAL_THINK === "1";
+  const templated = opts.useChat !== false && tm.template !== null;
+  const text = templated
+    ? tm.template!.render([{ role: "user", content: body }], { addGenerationPrompt: true, enableThinking })
     : body;
-  const ids = tm.tokenizer.encode(text);
+  // The chat template ALREADY emits the BOS; encoding it with add_special_tokens on
+  // would prepend a SECOND BOS (<s><s>…), corrupting generation. Raw bodies (no
+  // template) get the BOS added as normal.
+  const ids = tm.tokenizer.encode(text, /* addSpecialTokens */ !templated);
+
+  // optiq's PUBLISHED eval generates with a FULL-PRECISION KV cache — its eval calls
+  // plain mlx_lm.generate (kv-quant lives in the serving runtime, not the eval). So
+  // for like-for-like parity we default the eval to UNQUANTIZED KV; MMLU's argmax
+  // path was already unquantized, which is why it matched while generation didn't.
+  // MLX_BUN_EVAL_KV_QUANT=1 generates through the model's quantized KV (serving config).
+  const kv = (process.env.MLX_BUN_EVAL_KV_QUANT === "1" && tm.config.kvQuant?.length)
+    ? tm.config.kvQuant : undefined;
 
-  const kv = tm.config.kvQuant?.length ? tm.config.kvQuant : undefined;
+  // Parity default — greedy + full-precision KV + no sampler arm: decode bit-exactly
+  // via the raw forward (matches mlx-lm token-for-token). Sampler arms / kv-quant fall
+  // through to the product generate().
+  if (!kv && !opts.sampler && !tm.samplerOverride) {
+    return greedyDecodeBitExact(tm, ids, maxTokens);
+  }
   const gen = generate(tm.model, ids, {
     maxTokens,
     temperature: 0, // greedy default — deterministic head-to-head arms
+    adapters: tm.activeAdapters ?? [], // per-request activation of the mounted eval-adapter
     ...(kv ? { kvConfig: kv, quantizedKvStart: 0 } : {}),
     ...(opts.sampler ?? {}), // overrides temperature when provided
     ...(tm.samplerOverride ?? {}), // arm override wins over the task's own sampler

package/src/eval/tasks/bfcl.ts

@@ -15,6 +15,7 @@
 // of the task — is ported exactly. (See summary for this one note.)
 
 import { generateText, loadJsonl, sampleIndices, type TaskModel } from "../runner";
+import { type ToolDefinition } from "../chat-template";
 
 // ---------------------------------------------------------------------------
 // Data shapes (the merged jsonl: each line = {query, answer}).
@@ -446,29 +447,62 @@ function gtName(gt: Record<string, unknown>): string {
 // Eval loop.
 // ---------------------------------------------------------------------------
 
+// optiq's frozen rows are FLAT (`{**question, ground_truth}`) — the question dict
+// merged with its ground truth — unlike our nested {query, answer} export.
+interface BfclFrozenRow {
+  id: string;
+  question: BfclQuery["question"];
+  function: ToolDef[] | string;
+  ground_truth: BfclAnswer["ground_truth"];
+}
+type BfclItem = { question: BfclQuery["question"]; function: ToolDef[] | string; ground_truth: BfclAnswer["ground_truth"] };
+
 export async function evaluateBfcl(
   tm: TaskModel,
-  opts: { nSamples?: number; maxTokens?: number; seed?: number } = {},
+  opts: { nSamples?: number; maxTokens?: number; seed?: number; frozen?: boolean } = {},
 ): Promise<BfclResult> {
   const maxTokens = opts.maxTokens ?? 512;
-  const rows = loadJsonl<BfclRow>("bfcl");
-  const idx = sampleIndices(rows.length, opts.nSamples ?? 200, opts.seed ?? 42);
+
+  // Optiq-parity mode (DEFAULT): optiq's EXACT 200 BFCL-simple questions, scored
+  // through OUR pipeline. MLX_BUN_BFCL_FROZEN=0 reverts to our copy + sampling.
+  const useFrozen = opts.frozen ?? (process.env.MLX_BUN_BFCL_FROZEN !== "0");
+  let items: BfclItem[];
+  if (useFrozen) {
+    items = loadJsonl<BfclFrozenRow>("bfcl_optiq_frozen").map((r) => ({
+      question: r.question, function: r.function, ground_truth: r.ground_truth,
+    }));
+  } else {
+    const rows = loadJsonl<BfclRow>("bfcl");
+    items = sampleIndices(rows.length, opts.nSamples ?? 200, opts.seed ?? 42).map((i) => ({
+      question: rows[i]!.query.question, function: rows[i]!.query.function, ground_truth: rows[i]!.answer.ground_truth,
+    }));
+  }
 
   let nCorrect = 0;
   let nNoCall = 0;
   let nWrongName = 0;
   let nWrongArgs = 0;
 
-  for (let k = 0; k < idx.length; k++) {
-    const row = rows[idx[k]!]!;
-    const q = row.query;
-    const text = userText(q.question);
-    const tools = wrapToolsForChat(q.function);
-    const prompt = buildPrompt(text, tools);
-
-    const out = await generateText(tm, prompt, { maxTokens, useChat: true });
+  for (let k = 0; k < items.length; k++) {
+    const item = items[k]!;
+    const text = userText(item.question);
+    const tools = wrapToolsForChat(item.function);
+
+    // optiq's primary path: tools go THROUGH the chat template (the MiniCPM5
+    // template renders them as a `# Tools` system turn). Only fall back to the
+    // textual prompt when there's no chat template — mirrors bfcl.py's try/except.
+    let out: string;
+    if (tm.template) {
+      const rendered = tm.template.render(
+        [{ role: "user", content: text }],
+        { addGenerationPrompt: true, enableThinking: false, tools: tools as ToolDefinition[] },
+      );
+      out = await generateText(tm, rendered, { maxTokens, useChat: false });
+    } else {
+      out = await generateText(tm, buildPrompt(text, tools), { maxTokens, useChat: true });
+    }
     const predicted = extractToolCall(out);
-    const gt = parseGroundTruth(row.answer.ground_truth);
+    const gt = parseGroundTruth(item.ground_truth);
 
     if (predicted === null) {
       nNoCall++;
@@ -480,15 +514,15 @@ export async function evaluateBfcl(
       nCorrect++;
     }
 
-    if ((k + 1) % 10 === 0 || k + 1 === idx.length) {
+    if ((k + 1) % 10 === 0 || k + 1 === items.length) {
       process.stderr.write(
-        `\r  bfcl ${k + 1}/${idx.length}  acc=${((nCorrect / (k + 1)) * 100).toFixed(1)}%`,
+        `\r  bfcl ${k + 1}/${items.length}  acc=${((nCorrect / (k + 1)) * 100).toFixed(1)}%`,
       );
     }
   }
   process.stderr.write("\n");
 
-  const nTotal = idx.length;
+  const nTotal = items.length;
   return {
     nCorrect,
     nTotal,

package/src/eval/tasks/gsm8k.ts

@@ -38,7 +38,8 @@ const NUM = String.raw`-?[\d,]+\.?\d*`;
 /** Model output → numeric answer string (####, then \boxed{}, then last number). */
 export function extractAnswer(textIn: string): string | null {
   let text = textIn;
-  if (text.includes("</think>")) text = text.split("</think>")[1] ?? text;
+  // Match Python's `split("</think>", 1)[1]`: everything after the FIRST tag.
+  if (text.includes("</think>")) text = text.split("</think>").slice(1).join("</think>");
 
   const hash = text.match(new RegExp(String.raw`####\s*(${NUM})`));
   if (hash) return hash[1]!.replace(/,/g, "").trim();
@@ -63,23 +64,34 @@ function toNum(s: string | null): number | null {
 
 export async function evaluateGsm8k(
   tm: TaskModel,
-  opts: { nSamples?: number; nShots?: number; maxTokens?: number; seed?: number } = {},
+  opts: { nSamples?: number; nShots?: number; maxTokens?: number; seed?: number; frozen?: boolean } = {},
 ): Promise<Gsm8kResult> {
   const nShots = opts.nShots ?? 3;
   const maxTokens = opts.maxTokens ?? 256;
-  const rows = loadJsonl<Gsm8kRow>("gsm8k");
-  const idx = sampleIndices(rows.length, opts.nSamples ?? 200, opts.seed ?? 42);
+
+  // Optiq-parity mode (DEFAULT): score optiq's EXACT 1000 questions (the same data
+  // his published GSM8K used) but through OUR OWN pipeline — our few-shot builder,
+  // our chat template, our scorer. If our number reproduces his, our runtime is
+  // faithful. MLX_BUN_GSM8K_FROZEN=0 falls back to our own sampling of the full set.
+  const useFrozen = opts.frozen ?? (process.env.MLX_BUN_GSM8K_FROZEN !== "0");
+  let rows: Gsm8kRow[];
+  if (useFrozen) {
+    rows = loadJsonl<Gsm8kRow>("gsm8k_optiq_frozen"); // optiq's exact draw; we use {question, answer}
+  } else {
+    const all = loadJsonl<Gsm8kRow>("gsm8k");
+    rows = sampleIndices(all.length, opts.nSamples ?? 1000, opts.seed ?? 42).map((i) => all[i]!);
+  }
 
   let nCorrect = 0;
-  for (let k = 0; k < idx.length; k++) {
-    const item = rows[idx[k]!]!;
+  for (let k = 0; k < rows.length; k++) {
+    const item = rows[k]!;
     const gt = toNum(groundTruth(item.answer));
     const out = await generateText(tm, buildPrompt(item.question, nShots), { maxTokens, useChat: true });
     const pred = toNum(extractAnswer(out));
     if (gt !== null && pred !== null && Math.abs(gt - pred) < 1e-3) nCorrect++;
-    if ((k + 1) % 10 === 0 || k + 1 === idx.length)
-      process.stderr.write(`\r  gsm8k ${k + 1}/${idx.length}  acc=${(nCorrect / (k + 1) * 100).toFixed(1)}%`);
+    if ((k + 1) % 10 === 0 || k + 1 === rows.length)
+      process.stderr.write(`\r  gsm8k ${k + 1}/${rows.length}  acc=${(nCorrect / (k + 1) * 100).toFixed(1)}%`);
   }
   process.stderr.write("\n");
-  return { nCorrect, nTotal: idx.length, accuracy: idx.length ? nCorrect / idx.length : 0 };
+  return { nCorrect, nTotal: rows.length, accuracy: rows.length ? nCorrect / rows.length : 0 };
 }

package/src/eval/tasks/hashhop.ts

@@ -9,7 +9,7 @@
 // softmax / mixed-4/8-bit-KV error compounds across hops, so it's where the
 // fused-attention kernel (opportunity A) has the most to prove.
 
-import { generateText, type TaskModel } from "../runner";
+import { generateText, loadJsonl, type TaskModel } from "../runner";
 
 const INSTR =
   "You will be given a list of hash assignments and a starting hash. " +
@@ -80,16 +80,45 @@ export interface HashhopResult {
   byHops: Record<number, number>;
 }
 
+interface HashhopFrozen { raw_problem: string; query: string; expected: string }
+
 export async function evaluateHashhop(
   tm: TaskModel,
-  opts: { nPerHop?: number; hopsList?: number[]; chars?: number; maxTokens?: number; seed?: number } = {},
+  opts: { nPerHop?: number; hopsList?: number[]; chars?: number; maxTokens?: number; seed?: number; frozen?: boolean } = {},
 ): Promise<HashhopResult> {
   const nPerHop = opts.nPerHop ?? 25;
   const hopsList = opts.hopsList ?? [1, 2, 3, 4];
-  const chars = opts.chars ?? 12000;
   const maxTokens = opts.maxTokens ?? 32;
-  const rand = mulberry32(opts.seed ?? 42);
 
+  // Optiq-parity mode (DEFAULT): use optiq's EXACT generated problems (his
+  // MultiHopEval samples — seed 42, 25/hop × hops 1-4 = 100, emitted in hop order),
+  // fed through OUR chat template + extractor. Our generator below is a
+  // RECONSTRUCTION producing DIFFERENT problems, so it cannot reproduce optiq's
+  // number — the frozen problems are the only way to match. MLX_BUN_HASHHOP_FROZEN=0
+  // uses our generator.
+  const useFrozen = opts.frozen ?? (process.env.MLX_BUN_HASHHOP_FROZEN !== "0");
+  if (useFrozen) {
+    const rows = loadJsonl<HashhopFrozen>("hashhop_optiq_frozen");
+    const hopCorrect: Record<number, number> = {};
+    let total = 0;
+    let correct = 0;
+    for (let k = 0; k < rows.length; k++) {
+      const item = rows[k]!;
+      const hops = hopsList[Math.floor(k / nPerHop)] ?? 0; // optiq emits in hop order
+      const out = await generateText(tm, item.raw_problem, { maxTokens, useChat: true });
+      if (extractPred(out) === item.expected) { correct++; hopCorrect[hops] = (hopCorrect[hops] ?? 0) + 1; }
+      total++;
+      process.stderr.write(`\r  hashhop ${k + 1}/${rows.length}  acc=${(correct / (k + 1) * 100).toFixed(0)}%`);
+    }
+    process.stderr.write("\n");
+    const byHops: Record<number, number> = {};
+    for (const h of hopsList) byHops[h] = (hopCorrect[h] ?? 0) / nPerHop;
+    return { nTotal: total, nCorrect: correct, accuracy: total ? correct / total : 0, byHops };
+  }
+
+  // Fallback: our own reconstructed generator (produces DIFFERENT problems).
+  const chars = opts.chars ?? 12000;
+  const rand = mulberry32(opts.seed ?? 42);
   const byHops: Record<number, number> = {};
   let total = 0;
   let correct = 0;

package/src/eval/tasks/humaneval.ts

@@ -18,7 +18,9 @@ import { join } from "node:path";
 import { generateText, loadJsonl, sampleIndices, type TaskModel } from "../runner";
 
 // Oracle venv python — the same interpreter the Python reference uses.
-const ORACLE_PYTHON = "/Users/joshrossi/Code/mlx-lm/.venv/bin/python";
+// Machine-specific: overridable via MLX_BUN_ORACLE_PYTHON (the committed default is
+// the reference box; other laptops set the env, never re-commit the path).
+const ORACLE_PYTHON = process.env.MLX_BUN_ORACLE_PYTHON ?? "/Users/joshrossi/Code/mlx-lm/.venv/bin/python";
 
 interface HumanevalRow {
   task_id: string;
@@ -144,7 +146,7 @@ const HAS_SANDBOX_EXEC = (() => {
   }
 })();
 
-interface RunResult { ok: boolean; timedOut: boolean }
+interface RunResult { ok: boolean; timedOut: boolean; stderr?: string }
 
 /**
  * Run `program` as a python script in a fresh temp dir, sandboxed if
@@ -183,7 +185,8 @@ function runProgram(program: string, timeoutSec: number, memoryMb: number): RunR
       env,
       timeout: timeoutSec * 1000, // hard wall-clock kill
       killSignal: "SIGKILL",
-      stdio: ["ignore", "ignore", "ignore"],
+      stdio: ["ignore", "ignore", process.env.MLX_BUN_EVAL_DEBUG === "1" ? "pipe" : "ignore"],
+      encoding: "utf8",
     });
 
     // spawnSync sets signal === "SIGKILL" (and status === null) when the
@@ -192,7 +195,7 @@ function runProgram(program: string, timeoutSec: number, memoryMb: number): RunR
       res.signal === "SIGKILL" ||
       (res.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT";
     const ok = !timedOut && res.status === 0;
-    return { ok, timedOut };
+    return { ok, timedOut, stderr: typeof res.stderr === "string" ? res.stderr : undefined };
   } finally {
     rmSync(sbRoot, { recursive: true, force: true });
   }
@@ -206,13 +209,21 @@ export async function evaluateHumaneval(
     seed?: number;
     timeoutSec?: number;
     memoryMb?: number;
+    frozen?: boolean;
   } = {},
 ): Promise<HumanevalResult> {
   const maxTokens = opts.maxTokens ?? 512;
   const timeoutSec = opts.timeoutSec ?? 10;
   const memoryMb = opts.memoryMb ?? 512;
-  const rows = loadJsonl<HumanevalRow>("humaneval");
-  const idx = sampleIndices(rows.length, opts.nSamples ?? rows.length, opts.seed ?? 42);
+
+  // Optiq-parity mode (DEFAULT): optiq's EXACT 164-problem set (he runs the full
+  // HumanEval test split), scored through OUR pipeline. MLX_BUN_HUMANEVAL_FROZEN=0
+  // reverts to our own copy + optional sampling.
+  const useFrozen = opts.frozen ?? (process.env.MLX_BUN_HUMANEVAL_FROZEN !== "0");
+  const rows = loadJsonl<HumanevalRow>(useFrozen ? "humaneval_optiq_frozen" : "humaneval");
+  const idx = useFrozen
+    ? Array.from({ length: rows.length }, (_, i) => i)
+    : sampleIndices(rows.length, opts.nSamples ?? rows.length, opts.seed ?? 42);
 
   // When a chat template exists, ask the model (instruct-style) for the
   // complete function inside a ```python block — humaneval.py's use_chat
@@ -239,8 +250,19 @@ export async function evaluateHumaneval(
     const completion = truncateCompletion(raw, ex.entry_point);
     const program = buildProgram(ex.prompt, completion, ex.test, ex.entry_point);
 
-    const { ok } = runProgram(program, timeoutSec, memoryMb);
-    if (ok) nPass++;
+    const rr = runProgram(program, timeoutSec, memoryMb);
+    if (rr.ok) nPass++;
+    else if (process.env.MLX_BUN_EVAL_DEBUG === "1") {
+      const err = rr.stderr ?? "";
+      const reason = rr.timedOut ? "TIMEOUT"
+        : completion.trim() === "" ? "EMPTY"
+        : /SyntaxError|IndentationError/.test(err) ? "SYNTAX"
+        : /AssertionError/.test(err) ? "WRONG-ANSWER"
+        : /Error|Exception|Traceback/.test(err) ? "RUNTIME-ERR"
+        : "OTHER";
+      const errSnip = err.trim().split("\n").slice(-1)[0]?.slice(0, 120) ?? "";
+      process.stderr.write(`\n  [FAIL ${ex.entry_point}] ${reason} complen=${completion.length} :: ${errSnip}\n`);
+    }
 
     if ((k + 1) % 5 === 0 || k + 1 === idx.length)
       process.stderr.write(

package/src/eval/tasks/ifeval.ts

@@ -67,7 +67,12 @@ function reEscape(s: string): string {
 // \b\w+\b — JS \w is ASCII-ish like Python's default re (no re.UNICODE here).
 function checkLengthWords(response: string, kw: Kw): boolean {
   const n = (response.match(/\b\w+\b/g) ?? []).length;
-  const rel = kwStr(kw, "relation", "at least");
+  // Python `kw.get("relation", "at least")`: the default applies ONLY when the key is
+  // ABSENT — a present-but-None `relation` (as the frozen kwargs supply for letter/
+  // paragraph/capital checks) returns None, and optiq's `"least" in None` RAISES →
+  // _verify_response returns False. Replicate exactly: rel is null → `.includes` throws
+  // below → verifyResponse catches → FAIL. (kwStr's None→default was the leniency.)
+  const rel = ("relation" in kw ? kw["relation"] : "at least") as string;
   const target = kwInt(kw, "num_words", 0);
   return rel.includes("least") ? n >= target : n <= target;
 }
@@ -75,14 +80,24 @@ function checkLengthWords(response: string, kw: Kw): boolean {
 function checkLengthSentences(response: string, kw: Kw): boolean {
   // Python: re.split(r"[.!?]+\s+", response.strip())
   const n = response.trim().split(/[.!?]+\s+/).length;
-  const rel = kwStr(kw, "relation", "at least");
+  // Python `kw.get("relation", "at least")`: the default applies ONLY when the key is
+  // ABSENT — a present-but-None `relation` (as the frozen kwargs supply for letter/
+  // paragraph/capital checks) returns None, and optiq's `"least" in None` RAISES →
+  // _verify_response returns False. Replicate exactly: rel is null → `.includes` throws
+  // below → verifyResponse catches → FAIL. (kwStr's None→default was the leniency.)
+  const rel = ("relation" in kw ? kw["relation"] : "at least") as string;
   const target = kwInt(kw, "num_sentences", 0);
   return rel.includes("least") ? n >= target : n <= target;
 }
 
 function checkLengthParagraphs(response: string, kw: Kw): boolean {
   const n = response.split("\n\n").filter((p) => p.trim()).length;
-  const rel = kwStr(kw, "relation", "at least");
+  // Python `kw.get("relation", "at least")`: the default applies ONLY when the key is
+  // ABSENT — a present-but-None `relation` (as the frozen kwargs supply for letter/
+  // paragraph/capital checks) returns None, and optiq's `"least" in None` RAISES →
+  // _verify_response returns False. Replicate exactly: rel is null → `.includes` throws
+  // below → verifyResponse catches → FAIL. (kwStr's None→default was the leniency.)
+  const rel = ("relation" in kw ? kw["relation"] : "at least") as string;
   const target = kwInt(kw, "num_paragraphs", 0);
   return rel.includes("least") ? n >= target : n <= target;
 }
@@ -101,7 +116,12 @@ function checkKeywordsForbidden(response: string, kw: Kw): boolean {
 
 function checkKeywordFrequency(response: string, kw: Kw): boolean {
   const kwWord = kwStr(kw, "keyword", "").toLowerCase();
-  const rel = kwStr(kw, "relation", "at least");
+  // Python `kw.get("relation", "at least")`: the default applies ONLY when the key is
+  // ABSENT — a present-but-None `relation` (as the frozen kwargs supply for letter/
+  // paragraph/capital checks) returns None, and optiq's `"least" in None` RAISES →
+  // _verify_response returns False. Replicate exactly: rel is null → `.includes` throws
+  // below → verifyResponse catches → FAIL. (kwStr's None→default was the leniency.)
+  const rel = ("relation" in kw ? kw["relation"] : "at least") as string;
   const target = kwInt(kw, "frequency", 0);
   const n = (response.toLowerCase().match(new RegExp(`\\b${reEscape(kwWord)}\\b`, "g")) ?? []).length;
   return rel.includes("least") ? n >= target : n <= target;
@@ -109,7 +129,12 @@ function checkKeywordFrequency(response: string, kw: Kw): boolean {
 
 function checkLetterFrequency(response: string, kw: Kw): boolean {
   const letter = kwStr(kw, "letter", "").toLowerCase();
-  const rel = kwStr(kw, "relation", "at least");
+  // Python `kw.get("relation", "at least")`: the default applies ONLY when the key is
+  // ABSENT — a present-but-None `relation` (as the frozen kwargs supply for letter/
+  // paragraph/capital checks) returns None, and optiq's `"least" in None` RAISES →
+  // _verify_response returns False. Replicate exactly: rel is null → `.includes` throws
+  // below → verifyResponse catches → FAIL. (kwStr's None→default was the leniency.)
+  const rel = ("relation" in kw ? kw["relation"] : "at least") as string;
   // Python: int(kw.get("let_frequency", kw.get("frequency", 0)))
   const target = kw["let_frequency"] !== undefined && kw["let_frequency"] !== null
     ? kwInt(kw, "let_frequency", 0)
@@ -124,7 +149,12 @@ function checkCapitalWordsCount(response: string, kw: Kw): boolean {
   // Python: w.isupper() and len(w) > 1 — str.isupper() requires ≥1 cased char,
   // all cased chars uppercase. Approximate: has a letter, no lowercase letter.
   const n = words.filter((w) => w.length > 1 && /[A-Za-z]/.test(w) && w === w.toUpperCase()).length;
-  const rel = kwStr(kw, "relation", "at least");
+  // Python `kw.get("relation", "at least")`: the default applies ONLY when the key is
+  // ABSENT — a present-but-None `relation` (as the frozen kwargs supply for letter/
+  // paragraph/capital checks) returns None, and optiq's `"least" in None` RAISES →
+  // _verify_response returns False. Replicate exactly: rel is null → `.includes` throws
+  // below → verifyResponse catches → FAIL. (kwStr's None→default was the leniency.)
+  const rel = ("relation" in kw ? kw["relation"] : "at least") as string;
   // Python: int(kw.get("capital_frequency", kw.get("frequency", 0)))
   const target = kw["capital_frequency"] !== undefined && kw["capital_frequency"] !== null
     ? kwInt(kw, "capital_frequency", 0)
@@ -251,11 +281,12 @@ const VERIFIERS = new Map<string, Verifier>([
 /**
  * Verify one response against its instruction list.
  * Returns { pass, unhandled }. An instruction_id with no registered verifier is
- * counted as a FAIL (NOT silently skipped) and recorded in `unhandled`. This is
- * stricter than ifeval.py (which treats unhandled as pass); per the mlx-bun
- * harness rule, anything we cannot port must fail rather than bias high.
+ * treated as a PASS (does not affect `pass`) and only recorded in `unhandled` for
+ * visibility — matching ifeval.py's `continue` policy EXACTLY, so our strict_acc
+ * reproduces optiq's. (Our VERIFIERS map mirrors his `_VERIFIERS` 1:1, so unhandled
+ * is empty for the standard IFEval set anyway; this only matters for parity safety.)
  */
-function verifyResponse(
+export function verifyResponse(
   response: string,
   instructionIds: string[],
   kwargsList: Kw[],
@@ -268,8 +299,7 @@ function verifyResponse(
     const verifier = VERIFIERS.get(iid);
     if (verifier === undefined) {
       unhandled.push(iid);
-      pass = false; // count unported instruction as a failure
-      continue;
+      continue; // ifeval.py: treat an unhandled instruction as PASS (don't touch `pass`)
     }
     try {
       if (!verifier(response, kw)) pass = false;
@@ -281,7 +311,7 @@ function verifyResponse(
 }
 
 /** Loose-mode preprocessing — mirrors ifeval.py `_loose_clean`. */
-function looseClean(response: string): string {
+export function looseClean(response: string): string {
   let s = response.trim();
   // Python: re.sub(r"^(Sure|Here|Of course)[,!.]\s*[^\n]*\n+", "", s, count=1)
   s = s.replace(/^(Sure|Here|Of course)[,!.]\s*[^\n]*\n+/, "");
@@ -293,22 +323,26 @@ function looseClean(response: string): string {
 }
 
 /** Strip a leading thinking block, matching ifeval.py's split on </think>. */
-function stripThinking(response: string): string {
+export function stripThinking(response: string): string {
   const idx = response.indexOf("</think>");
   return idx === -1 ? response : response.slice(idx + "</think>".length);
 }
 
 export async function evaluateIfeval(
   tm: TaskModel,
-  opts: { nSamples?: number; maxTokens?: number; seed?: number } = {},
+  opts: { nSamples?: number; maxTokens?: number; seed?: number; frozen?: boolean } = {},
 ): Promise<IfevalResult> {
   // ifeval.py: max_tokens default 512.
   const maxTokens = opts.maxTokens ?? 512;
   const seed = opts.seed ?? 42;
 
-  const rows = loadJsonl<IfevalRow>("ifeval");
+  // Optiq-parity mode (DEFAULT): score optiq's EXACT prompt set (he runs the full
+  // IFEval split — 541 prompts) through OUR verifiers + OUR chat template.
+  // MLX_BUN_IFEVAL_FROZEN=0 reverts to our own copy + optional sampling.
+  const useFrozen = opts.frozen ?? (process.env.MLX_BUN_IFEVAL_FROZEN !== "0");
+  const rows = loadJsonl<IfevalRow>(useFrozen ? "ifeval_optiq_frozen" : "ifeval");
   const nSamples = opts.nSamples ?? rows.length;
-  const idx = nSamples < rows.length
+  const idx = (!useFrozen && nSamples < rows.length)
     ? sampleIndices(rows.length, nSamples, seed)
     : Array.from({ length: rows.length }, (_, i) => i);
 
@@ -341,7 +375,7 @@ export async function evaluateIfeval(
   if (unhandledCounts.size) {
     const top = [...unhandledCounts.entries()].sort((a, b) => b[1] - a[1]);
     process.stderr.write(
-      "  ifeval unported instruction ids (counted as FAIL): " +
+      "  ifeval unported instruction ids (treated as PASS, per optiq): " +
         top.map(([k, v]) => `${k}(${v})`).join(", ") +
         "\n",
     );