mlx-bun 0.0.4 → 0.0.6

package/src/eval/ifeval.ts

@@ -0,0 +1,138 @@
+// IFEval — programmatically-verifiable instruction-following scoring (Zhou et
+// al. 2023, "Instruction-Following Evaluation for Large Language Models"). Each
+// instance pairs a prompt with one or more *verifiable* instructions; a response
+// is scored by deterministic checks (no LLM judge), so before/after ORPO
+// comparisons are exact and reproducible.
+//
+// This implements the high-frequency instruction families (a working subset of
+// the ~25 in the reference suite, extensible via INSTRUCTIONS). Strict scoring:
+// an instance passes (strict) iff every instruction passes; we also report the
+// per-instruction pass rate. The official metrics are prompt-level and
+// instruction-level accuracy — both derived from `scoreInstance`.
+
+export interface IFEvalInstance {
+  /** The prompt text (used by instructions that reference it, e.g. "repeat the request"). */
+  prompt: string;
+  /** Instruction ids, e.g. "keywords:existence". */
+  instruction_id_list: string[];
+  /** Per-instruction kwargs, aligned with instruction_id_list. */
+  kwargs: Array<Record<string, unknown>>;
+}
+
+export interface InstanceResult {
+  /** One pass/fail per instruction (aligned with instruction_id_list). */
+  perInstruction: boolean[];
+  /** Strict: all instructions passed. */
+  followedAll: boolean;
+}
+
+type Checker = (response: string, kw: Record<string, unknown>, inst: IFEvalInstance) => boolean;
+
+const words = (s: string): string[] => s.trim().split(/\s+/).filter(Boolean);
+const sentences = (s: string): string[] =>
+  s.split(/(?<=[.!?])\s+/).map((x) => x.trim()).filter(Boolean);
+const paragraphs = (s: string): string[] =>
+  s.split(/\n\s*\n/).map((x) => x.trim()).filter(Boolean);
+const norm = (s: string): string => s.toLowerCase();
+
+const cmp = (n: number, rel: string | undefined, target: number): boolean => {
+  switch (rel) {
+    case "at least": return n >= target;
+    case "at most": return n <= target;
+    case "less than": return n < target;
+    case "exactly": return n === target;
+    default: return n >= target; // IFEval default for "around"/unspecified is lenient ≥
+  }
+};
+
+/** The verifiable-instruction registry. Keyed by IFEval instruction_id. */
+export const INSTRUCTIONS: Record<string, Checker> = {
+  // ---- keywords ----
+  "keywords:existence": (r, kw) =>
+    (kw.keywords as string[]).every((k) => norm(r).includes(norm(k))),
+  "keywords:frequency": (r, kw) => {
+    const k = norm(kw.keyword as string);
+    const count = norm(r).split(k).length - 1;
+    return cmp(count, kw.relation as string, kw.frequency as number);
+  },
+  "keywords:forbidden_words": (r, kw) =>
+    !(kw.forbidden_words as string[]).some((k) => norm(r).includes(norm(k))),
+
+  // ---- length constraints ----
+  "length_constraints:number_words": (r, kw) =>
+    cmp(words(r).length, kw.relation as string, kw.num_words as number),
+  "length_constraints:number_sentences": (r, kw) =>
+    cmp(sentences(r).length, kw.relation as string, kw.num_sentences as number),
+  "length_constraints:number_paragraphs": (r, kw) =>
+    cmp(paragraphs(r).length, kw.relation as string, kw.num_paragraphs as number),
+
+  // ---- detectable format ----
+  "detectable_format:number_bullet_lists": (r, kw) => {
+    const bullets = r.split("\n").filter((l) => /^\s*[*\-]\s+/.test(l)).length;
+    return cmp(bullets, kw.relation as string, kw.num_bullets as number);
+  },
+  "detectable_format:number_highlighted_sections": (r, kw) => {
+    const hi = (r.match(/\*[^*\n]+\*/g) ?? []).length;
+    return cmp(hi, kw.relation as string, kw.num_highlights as number);
+  },
+  "detectable_format:title": (r) => /<<[^>\n]+>>/.test(r),
+  "detectable_format:json_format": (r) => {
+    const t = r.trim().replace(/^```(?:json)?\s*|\s*```$/g, "").trim();
+    try { JSON.parse(t); return true; } catch { return false; }
+  },
+
+  // ---- case ----
+  "change_case:english_lowercase": (r) => r === r.toLowerCase(),
+  "change_case:english_capital": (r) => r === r.toUpperCase(),
+
+  // ---- start / end ----
+  "startend:end_checker": (r, kw) =>
+    norm(r.trimEnd()).endsWith(norm((kw.end_phrase as string).trim())),
+  "startend:quotation": (r) => {
+    const t = r.trim();
+    return t.startsWith('"') && t.endsWith('"') && t.length >= 2;
+  },
+
+  // ---- punctuation ----
+  "punctuation:no_comma": (r) => !r.includes(","),
+};
+
+/** Score one IFEval instance (strict). Unknown instruction ids count as failed
+ *  (so coverage gaps never inflate the score) — filter the dataset to the
+ *  supported ids, or extend INSTRUCTIONS, to avoid that. */
+export function scoreInstance(instance: IFEvalInstance, response: string): InstanceResult {
+  const perInstruction = instance.instruction_id_list.map((id, i) => {
+    const check = INSTRUCTIONS[id];
+    if (!check) return false;
+    try { return check(response, instance.kwargs[i] ?? {}, instance); } catch { return false; }
+  });
+  return { perInstruction, followedAll: perInstruction.every(Boolean) };
+}
+
+export interface IFEvalReport {
+  n: number;
+  /** Fraction of instances where ALL instructions were followed. */
+  promptAccuracy: number;
+  /** Fraction of individual instructions followed. */
+  instructionAccuracy: number;
+}
+
+/** Aggregate the official prompt-level + instruction-level accuracies over a set
+ *  of (instance, response) pairs. */
+export function aggregate(pairs: Array<{ instance: IFEvalInstance; response: string }>): IFEvalReport {
+  let prompts = 0, instrFollowed = 0, instrTotal = 0;
+  for (const { instance, response } of pairs) {
+    const r = scoreInstance(instance, response);
+    if (r.followedAll) prompts++;
+    instrFollowed += r.perInstruction.filter(Boolean).length;
+    instrTotal += r.perInstruction.length;
+  }
+  return {
+    n: pairs.length,
+    promptAccuracy: pairs.length ? prompts / pairs.length : 0,
+    instructionAccuracy: instrTotal ? instrFollowed / instrTotal : 0,
+  };
+}
+
+/** Instruction ids this implementation supports (filter datasets to these). */
+export const SUPPORTED_INSTRUCTIONS = new Set(Object.keys(INSTRUCTIONS));

package/src/eval/runner.ts

@@ -6,6 +6,7 @@
 import { homedir } from "node:os";
 import { readFileSync } from "node:fs";
 import { generate, type GenerateOptions } from "../generate";
+import * as ops from "../mlx/ops";
 import { ChatTemplate } from "../chat-template";
 import { loadModelConfig, type ModelConfig } from "../config";
 import { Weights } from "../weights";
@@ -56,17 +57,34 @@ export interface TaskModel {
    *  task's own (greedy) default — used to run the whole suite under one sampler
    *  arm (e.g. the v2 curve vs the default chat recipe) for the degradation gate. */
   samplerOverride?: SamplerArm;
+  /** Mounted-adapter ids to ACTIVATE on every forward/generate. Mounting an adapter
+   *  only stores it on the linears; `loraState.active` is what actually applies it,
+   *  and generation resets it to [] after each call. So generateText re-passes this,
+   *  and direct-forward tasks (MMLU) re-assert it — else the eval silently runs base. */
+  activeAdapters?: string[];
 }
 
-export async function loadTaskModel(query: string): Promise<TaskModel> {
+export async function loadTaskModel(query: string, adapterDir?: string): Promise<TaskModel> {
   const dir = resolveModelDir(query);
   const config = await loadModelConfig(dir);
   const weights = await Weights.open(dir);
   const model = createModel(weights, config);
+  const activeAdapters: string[] = [];
+  if (adapterDir) {
+    // Mount a trained LoRA adapter so the eval measures base+adapter (e.g. the
+    // ORPO-fine-tuned "after" model). NOTE: mount() only STORES the adapter on each
+    // linear — it does NOT activate it. Activation is `loraState.active`; without
+    // setting it the forward runs BASE (bit-identical to no-adapter). So activate it
+    // here for direct-forward tasks; generateText re-passes `activeAdapters` too.
+    const { AdapterManager } = await import("../lora");
+    await new AdapterManager(model).mount("eval-adapter", adapterDir);
+    activeAdapters.push("eval-adapter");
+    model.loraState.active = [...activeAdapters];
+  }
   const tokenizer = await loadTokenizer(dir);
   let template: ChatTemplate | null = null;
   try { template = await ChatTemplate.load(dir); } catch { template = null; }
-  return { model, tokenizer, template, config, dir };
+  return { model, tokenizer, template, config, dir, activeAdapters: activeAdapters.length ? activeAdapters : undefined };
 }
 
 export interface GenOpts {
@@ -76,21 +94,73 @@ export interface GenOpts {
   /** Sampling overrides (default greedy). Lets the HLG/diversity evals vary the
    *  sampler — temperature, top-p/k, seed, hlg — while sharing this prompt path. */
   sampler?: Partial<Pick<GenerateOptions, "temperature" | "topP" | "topK" | "seed" | "hlg" | "curve">>;
+  /** Chat-template `enable_thinking`. Defaults to OFF for eval parity with optiq
+   *  (its capability numbers are non-thinking; e.g. MiniCPM5 IFEval 64.7 is
+   *  non-thinking). Set MLX_BUN_EVAL_THINK=1 or pass true to evaluate thinking mode. */
+  enableThinking?: boolean;
+}
+
+/** Bit-exact greedy decode: raw `model.forward` + argmax loop, which matches mlx-lm
+ *  token-for-token (gen8k: 8000/8000). The product `generate()` decode wrapper
+ *  (forwardHidden/logitsFromHidden + pipelined sampler) diverges from the raw forward
+ *  on near-ties past ~32 tokens, so the EVAL decodes bit-exactly itself rather than
+ *  alter the serving path. Greedy + full-precision KV only. */
+function greedyDecodeBitExact(tm: TaskModel, ids: number[], maxTokens: number): string {
+  if (tm.activeAdapters) tm.model.loraState.active = tm.activeAdapters; // apply the mounted adapter
+  const cache = tm.model.makeCache();
+  try {
+    if (ids.length > 1) tm.model.forward(ids.slice(0, -1), cache).dispose(); // prefill
+    let last = ids[ids.length - 1]!;
+    const eos = new Set(tm.config.eosTokenIds);
+    const out: number[] = [];
+    for (let i = 0; i < maxTokens; i++) {
+      const lg = tm.model.forward([last], cache);
+      const am = ops.argmaxAxis(lg, -1); // GPU argmax (mlx-lm greedy: lowest-index tie-break)
+      lg.dispose();
+      const best = ops.itemUint32(am);
+      am.dispose();
+      if (eos.has(best)) break; // mlx-lm halts on EOS and excludes it from the output
+      out.push(best);
+      last = best;
+    }
+    return tm.tokenizer.decode(out, true);
+  } finally {
+    for (const c of cache) c.dispose();
+  }
 }
 
 /** Complete `body` via the model's real quantized-KV path. Greedy by default;
  *  pass `opts.sampler` to drive temperature/HLG/etc. */
 export async function generateText(tm: TaskModel, body: string, opts: GenOpts = {}): Promise<string> {
   const maxTokens = opts.maxTokens ?? 256;
-  const text = opts.useChat !== false && tm.template
-    ? tm.template.render([{ role: "user", content: body }], { addGenerationPrompt: true })
+  const enableThinking = opts.enableThinking ?? process.env.MLX_BUN_EVAL_THINK === "1";
+  const templated = opts.useChat !== false && tm.template !== null;
+  const text = templated
+    ? tm.template!.render([{ role: "user", content: body }], { addGenerationPrompt: true, enableThinking })
     : body;
-  const ids = tm.tokenizer.encode(text);
+  // The chat template ALREADY emits the BOS; encoding it with add_special_tokens on
+  // would prepend a SECOND BOS (<s><s>…), corrupting generation. Raw bodies (no
+  // template) get the BOS added as normal.
+  const ids = tm.tokenizer.encode(text, /* addSpecialTokens */ !templated);
+
+  // optiq's PUBLISHED eval generates with a FULL-PRECISION KV cache — its eval calls
+  // plain mlx_lm.generate (kv-quant lives in the serving runtime, not the eval). So
+  // for like-for-like parity we default the eval to UNQUANTIZED KV; MMLU's argmax
+  // path was already unquantized, which is why it matched while generation didn't.
+  // MLX_BUN_EVAL_KV_QUANT=1 generates through the model's quantized KV (serving config).
+  const kv = (process.env.MLX_BUN_EVAL_KV_QUANT === "1" && tm.config.kvQuant?.length)
+    ? tm.config.kvQuant : undefined;
 
-  const kv = tm.config.kvQuant?.length ? tm.config.kvQuant : undefined;
+  // Parity default — greedy + full-precision KV + no sampler arm: decode bit-exactly
+  // via the raw forward (matches mlx-lm token-for-token). Sampler arms / kv-quant fall
+  // through to the product generate().
+  if (!kv && !opts.sampler && !tm.samplerOverride) {
+    return greedyDecodeBitExact(tm, ids, maxTokens);
+  }
   const gen = generate(tm.model, ids, {
     maxTokens,
     temperature: 0, // greedy default — deterministic head-to-head arms
+    adapters: tm.activeAdapters ?? [], // per-request activation of the mounted eval-adapter
     ...(kv ? { kvConfig: kv, quantizedKvStart: 0 } : {}),
     ...(opts.sampler ?? {}), // overrides temperature when provided
     ...(tm.samplerOverride ?? {}), // arm override wins over the task's own sampler

package/src/eval/tasks/bfcl.ts

@@ -15,6 +15,7 @@
 // of the task — is ported exactly. (See summary for this one note.)
 
 import { generateText, loadJsonl, sampleIndices, type TaskModel } from "../runner";
+import { type ToolDefinition } from "../chat-template";
 
 // ---------------------------------------------------------------------------
 // Data shapes (the merged jsonl: each line = {query, answer}).
@@ -446,29 +447,62 @@ function gtName(gt: Record<string, unknown>): string {
 // Eval loop.
 // ---------------------------------------------------------------------------
 
+// optiq's frozen rows are FLAT (`{**question, ground_truth}`) — the question dict
+// merged with its ground truth — unlike our nested {query, answer} export.
+interface BfclFrozenRow {
+  id: string;
+  question: BfclQuery["question"];
+  function: ToolDef[] | string;
+  ground_truth: BfclAnswer["ground_truth"];
+}
+type BfclItem = { question: BfclQuery["question"]; function: ToolDef[] | string; ground_truth: BfclAnswer["ground_truth"] };
+
 export async function evaluateBfcl(
   tm: TaskModel,
-  opts: { nSamples?: number; maxTokens?: number; seed?: number } = {},
+  opts: { nSamples?: number; maxTokens?: number; seed?: number; frozen?: boolean } = {},
 ): Promise<BfclResult> {
   const maxTokens = opts.maxTokens ?? 512;
-  const rows = loadJsonl<BfclRow>("bfcl");
-  const idx = sampleIndices(rows.length, opts.nSamples ?? 200, opts.seed ?? 42);
+
+  // Optiq-parity mode (DEFAULT): optiq's EXACT 200 BFCL-simple questions, scored
+  // through OUR pipeline. MLX_BUN_BFCL_FROZEN=0 reverts to our copy + sampling.
+  const useFrozen = opts.frozen ?? (process.env.MLX_BUN_BFCL_FROZEN !== "0");
+  let items: BfclItem[];
+  if (useFrozen) {
+    items = loadJsonl<BfclFrozenRow>("bfcl_optiq_frozen").map((r) => ({
+      question: r.question, function: r.function, ground_truth: r.ground_truth,
+    }));
+  } else {
+    const rows = loadJsonl<BfclRow>("bfcl");
+    items = sampleIndices(rows.length, opts.nSamples ?? 200, opts.seed ?? 42).map((i) => ({
+      question: rows[i]!.query.question, function: rows[i]!.query.function, ground_truth: rows[i]!.answer.ground_truth,
+    }));
+  }
 
   let nCorrect = 0;
   let nNoCall = 0;
   let nWrongName = 0;
   let nWrongArgs = 0;
 
-  for (let k = 0; k < idx.length; k++) {
-    const row = rows[idx[k]!]!;
-    const q = row.query;
-    const text = userText(q.question);
-    const tools = wrapToolsForChat(q.function);
-    const prompt = buildPrompt(text, tools);
-
-    const out = await generateText(tm, prompt, { maxTokens, useChat: true });
+  for (let k = 0; k < items.length; k++) {
+    const item = items[k]!;
+    const text = userText(item.question);
+    const tools = wrapToolsForChat(item.function);
+
+    // optiq's primary path: tools go THROUGH the chat template (the MiniCPM5
+    // template renders them as a `# Tools` system turn). Only fall back to the
+    // textual prompt when there's no chat template — mirrors bfcl.py's try/except.
+    let out: string;
+    if (tm.template) {
+      const rendered = tm.template.render(
+        [{ role: "user", content: text }],
+        { addGenerationPrompt: true, enableThinking: false, tools: tools as ToolDefinition[] },
+      );
+      out = await generateText(tm, rendered, { maxTokens, useChat: false });
+    } else {
+      out = await generateText(tm, buildPrompt(text, tools), { maxTokens, useChat: true });
+    }
     const predicted = extractToolCall(out);
-    const gt = parseGroundTruth(row.answer.ground_truth);
+    const gt = parseGroundTruth(item.ground_truth);
 
     if (predicted === null) {
       nNoCall++;
@@ -480,15 +514,15 @@ export async function evaluateBfcl(
       nCorrect++;
     }
 
-    if ((k + 1) % 10 === 0 || k + 1 === idx.length) {
+    if ((k + 1) % 10 === 0 || k + 1 === items.length) {
       process.stderr.write(
-        `\r  bfcl ${k + 1}/${idx.length}  acc=${((nCorrect / (k + 1)) * 100).toFixed(1)}%`,
+        `\r  bfcl ${k + 1}/${items.length}  acc=${((nCorrect / (k + 1)) * 100).toFixed(1)}%`,
       );
     }
   }
   process.stderr.write("\n");
 
-  const nTotal = idx.length;
+  const nTotal = items.length;
   return {
     nCorrect,
     nTotal,

package/src/eval/tasks/gsm8k.ts

@@ -38,7 +38,8 @@ const NUM = String.raw`-?[\d,]+\.?\d*`;
 /** Model output → numeric answer string (####, then \boxed{}, then last number). */
 export function extractAnswer(textIn: string): string | null {
   let text = textIn;
-  if (text.includes("</think>")) text = text.split("</think>")[1] ?? text;
+  // Match Python's `split("</think>", 1)[1]`: everything after the FIRST tag.
+  if (text.includes("</think>")) text = text.split("</think>").slice(1).join("</think>");
 
   const hash = text.match(new RegExp(String.raw`####\s*(${NUM})`));
   if (hash) return hash[1]!.replace(/,/g, "").trim();
@@ -63,23 +64,34 @@ function toNum(s: string | null): number | null {
 
 export async function evaluateGsm8k(
   tm: TaskModel,
-  opts: { nSamples?: number; nShots?: number; maxTokens?: number; seed?: number } = {},
+  opts: { nSamples?: number; nShots?: number; maxTokens?: number; seed?: number; frozen?: boolean } = {},
 ): Promise<Gsm8kResult> {
   const nShots = opts.nShots ?? 3;
   const maxTokens = opts.maxTokens ?? 256;
-  const rows = loadJsonl<Gsm8kRow>("gsm8k");
-  const idx = sampleIndices(rows.length, opts.nSamples ?? 200, opts.seed ?? 42);
+
+  // Optiq-parity mode (DEFAULT): score optiq's EXACT 1000 questions (the same data
+  // his published GSM8K used) but through OUR OWN pipeline — our few-shot builder,
+  // our chat template, our scorer. If our number reproduces his, our runtime is
+  // faithful. MLX_BUN_GSM8K_FROZEN=0 falls back to our own sampling of the full set.
+  const useFrozen = opts.frozen ?? (process.env.MLX_BUN_GSM8K_FROZEN !== "0");
+  let rows: Gsm8kRow[];
+  if (useFrozen) {
+    rows = loadJsonl<Gsm8kRow>("gsm8k_optiq_frozen"); // optiq's exact draw; we use {question, answer}
+  } else {
+    const all = loadJsonl<Gsm8kRow>("gsm8k");
+    rows = sampleIndices(all.length, opts.nSamples ?? 1000, opts.seed ?? 42).map((i) => all[i]!);
+  }
 
   let nCorrect = 0;
-  for (let k = 0; k < idx.length; k++) {
-    const item = rows[idx[k]!]!;
+  for (let k = 0; k < rows.length; k++) {
+    const item = rows[k]!;
     const gt = toNum(groundTruth(item.answer));
     const out = await generateText(tm, buildPrompt(item.question, nShots), { maxTokens, useChat: true });
     const pred = toNum(extractAnswer(out));
     if (gt !== null && pred !== null && Math.abs(gt - pred) < 1e-3) nCorrect++;
-    if ((k + 1) % 10 === 0 || k + 1 === idx.length)
-      process.stderr.write(`\r  gsm8k ${k + 1}/${idx.length}  acc=${(nCorrect / (k + 1) * 100).toFixed(1)}%`);
+    if ((k + 1) % 10 === 0 || k + 1 === rows.length)
+      process.stderr.write(`\r  gsm8k ${k + 1}/${rows.length}  acc=${(nCorrect / (k + 1) * 100).toFixed(1)}%`);
   }
   process.stderr.write("\n");
-  return { nCorrect, nTotal: idx.length, accuracy: idx.length ? nCorrect / idx.length : 0 };
+  return { nCorrect, nTotal: rows.length, accuracy: rows.length ? nCorrect / rows.length : 0 };
 }

package/src/eval/tasks/hashhop.ts

@@ -9,7 +9,7 @@
 // softmax / mixed-4/8-bit-KV error compounds across hops, so it's where the
 // fused-attention kernel (opportunity A) has the most to prove.
 
-import { generateText, type TaskModel } from "../runner";
+import { generateText, loadJsonl, type TaskModel } from "../runner";
 
 const INSTR =
   "You will be given a list of hash assignments and a starting hash. " +
@@ -80,16 +80,45 @@ export interface HashhopResult {
   byHops: Record<number, number>;
 }
 
+interface HashhopFrozen { raw_problem: string; query: string; expected: string }
+
 export async function evaluateHashhop(
   tm: TaskModel,
-  opts: { nPerHop?: number; hopsList?: number[]; chars?: number; maxTokens?: number; seed?: number } = {},
+  opts: { nPerHop?: number; hopsList?: number[]; chars?: number; maxTokens?: number; seed?: number; frozen?: boolean } = {},
 ): Promise<HashhopResult> {
   const nPerHop = opts.nPerHop ?? 25;
   const hopsList = opts.hopsList ?? [1, 2, 3, 4];
-  const chars = opts.chars ?? 12000;
   const maxTokens = opts.maxTokens ?? 32;
-  const rand = mulberry32(opts.seed ?? 42);
 
+  // Optiq-parity mode (DEFAULT): use optiq's EXACT generated problems (his
+  // MultiHopEval samples — seed 42, 25/hop × hops 1-4 = 100, emitted in hop order),
+  // fed through OUR chat template + extractor. Our generator below is a
+  // RECONSTRUCTION producing DIFFERENT problems, so it cannot reproduce optiq's
+  // number — the frozen problems are the only way to match. MLX_BUN_HASHHOP_FROZEN=0
+  // uses our generator.
+  const useFrozen = opts.frozen ?? (process.env.MLX_BUN_HASHHOP_FROZEN !== "0");
+  if (useFrozen) {
+    const rows = loadJsonl<HashhopFrozen>("hashhop_optiq_frozen");
+    const hopCorrect: Record<number, number> = {};
+    let total = 0;
+    let correct = 0;
+    for (let k = 0; k < rows.length; k++) {
+      const item = rows[k]!;
+      const hops = hopsList[Math.floor(k / nPerHop)] ?? 0; // optiq emits in hop order
+      const out = await generateText(tm, item.raw_problem, { maxTokens, useChat: true });
+      if (extractPred(out) === item.expected) { correct++; hopCorrect[hops] = (hopCorrect[hops] ?? 0) + 1; }
+      total++;
+      process.stderr.write(`\r  hashhop ${k + 1}/${rows.length}  acc=${(correct / (k + 1) * 100).toFixed(0)}%`);
+    }
+    process.stderr.write("\n");
+    const byHops: Record<number, number> = {};
+    for (const h of hopsList) byHops[h] = (hopCorrect[h] ?? 0) / nPerHop;
+    return { nTotal: total, nCorrect: correct, accuracy: total ? correct / total : 0, byHops };
+  }
+
+  // Fallback: our own reconstructed generator (produces DIFFERENT problems).
+  const chars = opts.chars ?? 12000;
+  const rand = mulberry32(opts.seed ?? 42);
   const byHops: Record<number, number> = {};
   let total = 0;
   let correct = 0;

package/src/eval/tasks/humaneval.ts

@@ -18,7 +18,9 @@ import { join } from "node:path";
 import { generateText, loadJsonl, sampleIndices, type TaskModel } from "../runner";
 
 // Oracle venv python — the same interpreter the Python reference uses.
-const ORACLE_PYTHON = "/Users/joshrossi/Code/mlx-lm/.venv/bin/python";
+// Machine-specific: overridable via MLX_BUN_ORACLE_PYTHON (the committed default is
+// the reference box; other laptops set the env, never re-commit the path).
+const ORACLE_PYTHON = process.env.MLX_BUN_ORACLE_PYTHON ?? "/Users/joshrossi/Code/mlx-lm/.venv/bin/python";
 
 interface HumanevalRow {
   task_id: string;
@@ -144,7 +146,7 @@ const HAS_SANDBOX_EXEC = (() => {
   }
 })();
 
-interface RunResult { ok: boolean; timedOut: boolean }
+interface RunResult { ok: boolean; timedOut: boolean; stderr?: string }
 
 /**
  * Run `program` as a python script in a fresh temp dir, sandboxed if
@@ -183,7 +185,8 @@ function runProgram(program: string, timeoutSec: number, memoryMb: number): RunR
       env,
       timeout: timeoutSec * 1000, // hard wall-clock kill
       killSignal: "SIGKILL",
-      stdio: ["ignore", "ignore", "ignore"],
+      stdio: ["ignore", "ignore", process.env.MLX_BUN_EVAL_DEBUG === "1" ? "pipe" : "ignore"],
+      encoding: "utf8",
     });
 
     // spawnSync sets signal === "SIGKILL" (and status === null) when the
@@ -192,7 +195,7 @@ function runProgram(program: string, timeoutSec: number, memoryMb: number): RunR
       res.signal === "SIGKILL" ||
       (res.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT";
     const ok = !timedOut && res.status === 0;
-    return { ok, timedOut };
+    return { ok, timedOut, stderr: typeof res.stderr === "string" ? res.stderr : undefined };
   } finally {
     rmSync(sbRoot, { recursive: true, force: true });
   }
@@ -206,13 +209,21 @@ export async function evaluateHumaneval(
     seed?: number;
     timeoutSec?: number;
     memoryMb?: number;
+    frozen?: boolean;
   } = {},
 ): Promise<HumanevalResult> {
   const maxTokens = opts.maxTokens ?? 512;
   const timeoutSec = opts.timeoutSec ?? 10;
   const memoryMb = opts.memoryMb ?? 512;
-  const rows = loadJsonl<HumanevalRow>("humaneval");
-  const idx = sampleIndices(rows.length, opts.nSamples ?? rows.length, opts.seed ?? 42);
+
+  // Optiq-parity mode (DEFAULT): optiq's EXACT 164-problem set (he runs the full
+  // HumanEval test split), scored through OUR pipeline. MLX_BUN_HUMANEVAL_FROZEN=0
+  // reverts to our own copy + optional sampling.
+  const useFrozen = opts.frozen ?? (process.env.MLX_BUN_HUMANEVAL_FROZEN !== "0");
+  const rows = loadJsonl<HumanevalRow>(useFrozen ? "humaneval_optiq_frozen" : "humaneval");
+  const idx = useFrozen
+    ? Array.from({ length: rows.length }, (_, i) => i)
+    : sampleIndices(rows.length, opts.nSamples ?? rows.length, opts.seed ?? 42);
 
   // When a chat template exists, ask the model (instruct-style) for the
   // complete function inside a ```python block — humaneval.py's use_chat
@@ -239,8 +250,19 @@ export async function evaluateHumaneval(
     const completion = truncateCompletion(raw, ex.entry_point);
     const program = buildProgram(ex.prompt, completion, ex.test, ex.entry_point);
 
-    const { ok } = runProgram(program, timeoutSec, memoryMb);
-    if (ok) nPass++;
+    const rr = runProgram(program, timeoutSec, memoryMb);
+    if (rr.ok) nPass++;
+    else if (process.env.MLX_BUN_EVAL_DEBUG === "1") {
+      const err = rr.stderr ?? "";
+      const reason = rr.timedOut ? "TIMEOUT"
+        : completion.trim() === "" ? "EMPTY"
+        : /SyntaxError|IndentationError/.test(err) ? "SYNTAX"
+        : /AssertionError/.test(err) ? "WRONG-ANSWER"
+        : /Error|Exception|Traceback/.test(err) ? "RUNTIME-ERR"
+        : "OTHER";
+      const errSnip = err.trim().split("\n").slice(-1)[0]?.slice(0, 120) ?? "";
+      process.stderr.write(`\n  [FAIL ${ex.entry_point}] ${reason} complen=${completion.length} :: ${errSnip}\n`);
+    }
 
     if ((k + 1) % 5 === 0 || k + 1 === idx.length)
       process.stderr.write(