mlx-bun 0.0.11 → 0.0.13

package/src/config.ts

@@ -2,6 +2,11 @@
 // Parses HF config.json (Gemma 4 unified layout: text_config nested),
 // the OptiQ per-layer quantization map, and kv_config.json when present.
 
+import {
+  parseGlm52Config,
+  type Glm52Config,
+} from "./model/glm52-config";
+
 export interface RopeParams {
   ropeTheta: number;
   ropeType: string;
@@ -223,6 +228,18 @@ export async function loadModelConfig(modelDir: string): Promise<ModelConfig> {
   const raw = (await Bun.file(`${modelDir}/config.json`).json()) as Record<string, any>;
   const modelType = raw.model_type;
   const isDiffusion = modelType === "diffusion_gemma";
+  const isGlm52 = modelType === "glm_moe_dsa";
+  // The converted Colibri artifact keeps stop metadata split across the two
+  // HF files. Preserve the old config-only behavior for every other family;
+  // GLM-5.2 explicitly unions both lists through its dedicated parser.
+  let glm52: Glm52Config | null = null;
+  if (isGlm52) {
+    const generationFile = Bun.file(`${modelDir}/generation_config.json`);
+    const generation = await generationFile.exists()
+      ? await generationFile.json() as Record<string, any>
+      : null;
+    glm52 = parseGlm52Config(modelDir, raw, generation);
+  }
   // Gemma 4 unified nests the LM config; plain text models keep it at top level.
   // DiffusionGemma keeps it flat too, but omits all dims — backfill from defaults.
   const baseT = (raw.text_config ?? raw) as Record<string, any>;
@@ -236,34 +253,56 @@ export async function loadModelConfig(modelDir: string): Promise<ModelConfig> {
   // partial_rotary_factor}), not the gemma per-attention-type map — and
   // type "default" means plain partial nn.RoPE (mrope_section ignored for text).
   const qwenRope = (t.rope_parameters ?? {}) as Record<string, any>;
+  // Qwen3.8 adds output_gate_type ("swish"), which every implementation
+  // (transformers ground truth, mlx-lm pinned + main) currently ignores —
+  // the attention output gate is hardcoded o_proj(out·σ(gate)). Only accept
+  // values verified to mean that; an unknown value must fail at load, not
+  // silently mis-gate a future checkpoint where the field starts mattering.
+  if (isQwen35 && t.output_gate_type != null && t.output_gate_type !== "swish")
+    throw new Error(
+      `qwen3_5: unverified output_gate_type "${t.output_gate_type}" (only "swish" ` +
+        `is verified to mean the standard sigmoid output gate)`,
+    );
 
   const text: TextConfig = {
-    hiddenSize: t.hidden_size,
-    numHiddenLayers: t.num_hidden_layers,
-    numAttentionHeads: t.num_attention_heads,
-    numKeyValueHeads: t.num_key_value_heads,
-    headDim: t.head_dim ?? Math.floor(t.hidden_size / t.num_attention_heads),
-    numGlobalKeyValueHeads: t.num_global_key_value_heads ?? t.num_key_value_heads,
-    globalHeadDim: t.global_head_dim ?? t.head_dim ?? Math.floor(t.hidden_size / t.num_attention_heads),
+    hiddenSize: glm52?.hiddenSize ?? t.hidden_size,
+    numHiddenLayers: glm52?.numHiddenLayers ?? t.num_hidden_layers,
+    numAttentionHeads: glm52?.numAttentionHeads ?? t.num_attention_heads,
+    numKeyValueHeads: glm52?.numKeyValueHeads ?? t.num_key_value_heads,
+    headDim: glm52?.qkHeadDim ?? t.head_dim ?? Math.floor(t.hidden_size / t.num_attention_heads),
+    numGlobalKeyValueHeads:
+      glm52?.numKeyValueHeads ?? t.num_global_key_value_heads ?? t.num_key_value_heads,
+    globalHeadDim:
+      glm52?.qkHeadDim ??
+      t.global_head_dim ??
+      t.head_dim ??
+      Math.floor(t.hidden_size / t.num_attention_heads),
     attentionKEqV: t.attention_k_eq_v ?? false,
-    intermediateSize: t.intermediate_size,
+    intermediateSize: glm52?.intermediateSize ?? t.intermediate_size,
     hiddenActivation: t.hidden_activation ?? t.hidden_act ?? "gelu_pytorch_tanh",
-    rmsNormEps: t.rms_norm_eps,
-    vocabSize: t.vocab_size,
-    maxPositionEmbeddings: t.max_position_embeddings,
+    rmsNormEps: glm52?.rmsNormEps ?? t.rms_norm_eps,
+    vocabSize: glm52?.vocabSize ?? t.vocab_size,
+    maxPositionEmbeddings:
+      glm52?.maxPositionEmbeddings ?? t.max_position_embeddings,
     slidingWindow: t.sliding_window ?? 0,
-    layerTypes: t.layer_types ?? (isLlama ? Array(t.num_hidden_layers).fill("full_attention") : []),
+    layerTypes: glm52
+      ? Array(glm52.numHiddenLayers).fill("full_attention")
+      : t.layer_types ?? (isLlama ? Array(t.num_hidden_layers).fill("full_attention") : []),
     hiddenSizePerLayerInput: t.hidden_size_per_layer_input ?? 0,
-    vocabSizePerLayerInput: t.vocab_size_per_layer_input ?? t.vocab_size,
+    vocabSizePerLayerInput:
+      t.vocab_size_per_layer_input ?? glm52?.vocabSize ?? t.vocab_size,
     numKvSharedLayers: t.num_kv_shared_layers ?? 0,
-    enableMoeBlock: t.enable_moe_block ?? false,
-    numExperts: t.num_experts ?? 0,
+    enableMoeBlock: glm52 ? true : t.enable_moe_block ?? false,
+    numExperts: glm52?.numRoutedExperts ?? t.num_experts ?? 0,
     // gemma4 uses `top_k_experts`; qwen3_moe uses `num_experts_per_tok`.
-    topKExperts: t.top_k_experts ?? t.num_experts_per_tok ?? 0,
-    moeIntermediateSize: t.moe_intermediate_size ?? 0,
-    decoderSparseStep: t.decoder_sparse_step ?? 1,
-    mlpOnlyLayers: t.mlp_only_layers ?? [],
-    normTopkProb: t.norm_topk_prob ?? false,
+    topKExperts:
+      glm52?.numExpertsPerToken ?? t.top_k_experts ?? t.num_experts_per_tok ?? 0,
+    moeIntermediateSize: glm52?.moeIntermediateSize ?? t.moe_intermediate_size ?? 0,
+    decoderSparseStep: glm52 ? 1 : t.decoder_sparse_step ?? 1,
+    mlpOnlyLayers: glm52
+      ? Array.from({ length: glm52.firstKDenseReplace }, (_, layer) => layer)
+      : t.mlp_only_layers ?? [],
+    normTopkProb: glm52?.normTopkProb ?? t.norm_topk_prob ?? false,
     linearNumValueHeads: t.linear_num_value_heads ?? 0,
     linearNumKeyHeads: t.linear_num_key_heads ?? 0,
     linearKeyHeadDim: t.linear_key_head_dim ?? 0,
@@ -271,9 +310,19 @@ export async function loadModelConfig(modelDir: string): Promise<ModelConfig> {
     linearConvKernelDim: t.linear_conv_kernel_dim ?? 0,
     fullAttentionInterval: t.full_attention_interval ?? 0,
     attnOutputGate: t.attn_output_gate ?? false,
-    partialRotaryFactor:
-      qwenRope.partial_rotary_factor ?? t.partial_rotary_factor ?? 1.0,
-    ropeParameters: isQwen35
+    partialRotaryFactor: glm52
+      ? glm52.qkRopeHeadDim / glm52.qkHeadDim
+      : qwenRope.partial_rotary_factor ?? t.partial_rotary_factor ?? 1.0,
+    ropeParameters: glm52
+      ? {
+          full_attention: {
+            ropeTheta: glm52.ropeTheta,
+            ropeType: "default",
+            partialRotaryFactor: glm52.qkRopeHeadDim / glm52.qkHeadDim,
+            factor: 1.0,
+          },
+        }
+      : isQwen35
       ? {
           full_attention: {
             ropeTheta: qwenRope.rope_theta ?? t.rope_theta ?? 10000,
@@ -297,7 +346,7 @@ export async function loadModelConfig(modelDir: string): Promise<ModelConfig> {
     finalLogitSoftcapping: t.final_logit_softcapping ?? null,
     tieWordEmbeddings: t.tie_word_embeddings ?? raw.tie_word_embeddings ?? false,
     bosTokenId: t.bos_token_id ?? raw.bos_token_id,
-    eosTokenId: t.eos_token_id ?? raw.eos_token_id,
+    eosTokenId: glm52?.eosTokenIds ?? t.eos_token_id ?? raw.eos_token_id,
     canvasLength: raw.canvas_length ?? t.canvas_length,
   };
 
@@ -311,14 +360,20 @@ export async function loadModelConfig(modelDir: string): Promise<ModelConfig> {
     }));
   }
 
-  const eos = raw.eos_token_id ?? text.eosTokenId;
+  const eos = glm52?.eosTokenIds ?? raw.eos_token_id ?? text.eosTokenId;
   return {
     modelDir,
     modelType,
     architectures: raw.architectures ?? [],
     dtype: raw.dtype ?? "bfloat16",
     text,
-    quantization: parseQuantization(raw.quantization ?? raw.quantization_config),
+    // Colibri's config retains the SOURCE FP8 quantization_config, while the
+    // serving shards are converted U8 + `.qs` tensors whose int4/int8/group
+    // geometry is inferred and validated by Glm52Container. Advertising the
+    // source FP8 block as mlx affine quantization would be actively wrong.
+    quantization: glm52
+      ? null
+      : parseQuantization(raw.quantization ?? raw.quantization_config),
     kvQuant,
     hasVisionSidecar: await Bun.file(`${modelDir}/optiq_vision.safetensors`).exists(),
     eosTokenIds: Array.isArray(eos) ? eos : [eos],

package/src/diffusion/diffusion-generate.ts

@@ -245,9 +245,7 @@ export function diffusionGenerate(
 
       let currentCanvas = ops.randint(0, vocab, [1, canvasLength], I32); // RNG: init
       let draftReveal = ops.zeros([1, canvasLength], Dtype.bool);
-      let draftCanvas: MlxArray = ops.logicalNot(ops.logicalNot(draftReveal)); // placeholder, replaced below
-      draftCanvas.dispose();
-      draftCanvas = ops.reshape(currentCanvas, [1, canvasLength]); // alias copy
+      let draftCanvas = ops.reshape(currentCanvas, [1, canvasLength]); // alias copy
       let argmaxCanvas = ops.reshape(currentCanvas, [1, canvasLength]);
       let scEmbeddings: MlxArray | null = null;
       const history: MlxArray[] = [];
@@ -266,7 +264,9 @@ export function diffusionGenerate(
         logits.dispose();
 
         argmaxCanvas.dispose();
-        argmaxCanvas = ops.argmaxAxis(processed, -1).astype(I32); // [1, L]
+        const argmax = ops.argmaxAxis(processed, -1);
+        argmaxCanvas = argmax.astype(I32); // [1, L]
+        argmax.dispose();
 
         if (curStep === 1) {
           processed.dispose();
@@ -296,7 +296,9 @@ export function diffusionGenerate(
           currentCanvas.dispose();
           currentCanvas = newCurrent;
           draftReveal.dispose();
-          draftReveal = ops.logicalNot(ops.logicalNot(acceptance)); // = acceptance
+          const notAcceptance = ops.logicalNot(acceptance);
+          draftReveal = ops.logicalNot(notAcceptance); // = acceptance
+          notAcceptance.dispose();
           draftCanvas.dispose();
           draftCanvas = ops.reshape(argmaxCanvas, [1, canvasLength]);
           accepted.dispose();
@@ -341,6 +343,10 @@ export function diffusionGenerate(
           stableStop &&
           stableAndConfident(argmaxCanvas, processed, history, stabilityThreshold, stopConfidence)
         ) {
+          // Entropy sampling precomputes the next step's self-conditioning
+          // tensor before evaluating this stop. The break means ownership is
+          // never transferred to scEmbeddings, so release the pending value.
+          nextSc?.dispose();
           processed.dispose();
           break;
         }
@@ -361,9 +367,9 @@ export function diffusionGenerate(
       draftCanvas.dispose();
 
       // Emit from the final argmax canvas.
-      const blockTokens = [...argmaxCanvas.astype(Dtype.float32).toFloat32()].map((x) =>
-        Math.round(x),
-      );
+      const argmaxFloat = argmaxCanvas.astype(Dtype.float32);
+      const blockTokens = [...argmaxFloat.toFloat32()].map((x) => Math.round(x));
+      argmaxFloat.dispose();
       argmaxCanvas.dispose();
       blocks.push(blockTokens);
       totalSteps += stepsThisCanvas;

package/src/download.ts

@@ -4,7 +4,7 @@
 // lore"): Xet stalls (we speak plain HTTPS resolve/CDN), no resume on
 // flaky links (we Range-resume partial blobs), and silent corruption
 // (every blob is verified — sha256 for LFS files against the API's
-// oid, git-blob sha1 for small files against blobId).
+// normalized LFS digest, git-blob sha1 for small files against blobId).
 //
 // Writes the exact huggingface_hub cache layout so the registry and
 // loaders find the result with zero changes:
@@ -18,6 +18,7 @@
 import {
   mkdirSync, existsSync, statSync, createWriteStream, renameSync, rmSync,
   symlinkSync, readFileSync, openSync, closeSync, readSync, writeSync,
+  statfsSync,
 } from "node:fs";
 import { dirname, join } from "node:path";
 import { DEFAULT_HUB } from "./registry";
@@ -29,8 +30,8 @@ export interface RepoFile {
   size: number;
   /** git blob sha1 (non-LFS identity, and the blob filename for non-LFS). */
   blobId: string;
-  /** LFS sha256 + size; present for large files. oid is the blob filename. */
-  lfs?: { oid: string; size: number };
+  /** LFS sha256 + size; present for large files. sha256 is the blob filename. */
+  lfs?: { sha256: string; size: number };
 }
 
 export interface RepoListing {
@@ -47,6 +48,56 @@ export interface DownloadOptions {
   onProgress?: (file: string, received: number, total: number) => void;
 }
 
+export interface DownloadSpacePlan {
+  /** Bytes still absent after crediting verified blobs and resumable prefixes. */
+  remainingBytes: number;
+  /** Files that still need at least one byte. */
+  remainingFiles: number;
+  /** Files already complete in the shared HF blob store. */
+  completeFiles: number;
+  /** Existing `.incomplete` bytes that will be rehashed and resumed. */
+  resumableBytes: number;
+}
+
+/** One GiB for filesystem metadata, the final refs write, and normal host
+ * operation. The artifact bytes themselves are exact; this is deliberately a
+ * small fixed guard rather than a percentage that would add tens of GiB to a
+ * Colibri download. */
+export const DOWNLOAD_DISK_RESERVE_BYTES = 2 ** 30;
+
+/** Compute the exact remaining payload without opening or hashing any blob.
+ * A valid-size final blob is already complete; an in-range `.incomplete`
+ * prefix is credited because downloadOneLocked resumes it in place. */
+export function planDownloadSpace(
+  files: readonly RepoFile[],
+  blobsDir: string,
+): DownloadSpacePlan {
+  let remainingBytes = 0;
+  let remainingFiles = 0;
+  let completeFiles = 0;
+  let resumableBytes = 0;
+  for (const file of files) {
+    const blobId = file.lfs?.sha256 ?? file.blobId;
+    if (!blobId) throw new Error(`no blob id for ${file.rfilename}`);
+    const blobPath = join(blobsDir, blobId);
+    if (existsSync(blobPath) && statSync(blobPath).size === file.size) {
+      completeFiles++;
+      continue;
+    }
+    const partPath = `${blobPath}.incomplete`;
+    const partBytes = existsSync(partPath) ? statSync(partPath).size : 0;
+    const credited = partBytes >= 0 && partBytes <= file.size ? partBytes : 0;
+    resumableBytes += credited;
+    remainingBytes += file.size - credited;
+    remainingFiles++;
+  }
+  return { remainingBytes, remainingFiles, completeFiles, resumableBytes };
+}
+
+function formatGib(bytes: number): string {
+  return `${(bytes / 2 ** 30).toFixed(2)} GiB`;
+}
+
 /** HF auth: explicit > env > the file `hf auth login` writes. */
 export function hfToken(): string | null {
   if (process.env.HF_TOKEN) return process.env.HF_TOKEN;
@@ -161,16 +212,34 @@ export async function listRepoFiles(
     );
   const body = (await res.json()) as {
     sha: string;
-    siblings: { rfilename: string; size?: number; blobId?: string; lfs?: { oid: string; size: number } }[];
+    siblings: {
+      rfilename: string;
+      size?: number;
+      blobId?: string;
+      // The current HF API calls this field `sha256`. Accept the older `oid`
+      // spelling too, but normalize it before the download path sees it.
+      lfs?: { sha256?: string; oid?: string; size: number };
+    }[];
   };
   return {
     sha: body.sha,
-    files: body.siblings.map((s) => ({
-      rfilename: s.rfilename,
-      size: s.lfs?.size ?? s.size ?? 0,
-      blobId: s.blobId ?? "",
-      lfs: s.lfs,
-    })),
+    files: body.siblings.map((s) => {
+      let lfs: RepoFile["lfs"];
+      if (s.lfs) {
+        const sha256 = s.lfs.sha256 ?? s.lfs.oid;
+        if (!sha256 || !/^[0-9a-f]{64}$/i.test(sha256))
+          throw new Error(`HF API returned an invalid LFS sha256 for ${s.rfilename}`);
+        lfs = { sha256: sha256.toLowerCase(), size: s.lfs.size };
+      }
+      if (!lfs && (!s.blobId || !/^[0-9a-f]{40}$/.test(s.blobId)))
+        throw new Error(`HF API returned an invalid git blob SHA-1 for ${s.rfilename}`);
+      return {
+        rfilename: s.rfilename,
+        size: lfs?.size ?? s.size ?? 0,
+        blobId: s.blobId ?? "",
+        lfs,
+      };
+    }),
   };
 }
 
@@ -349,6 +418,28 @@ export async function downloadModel(
   mkdirSync(blobsDir, { recursive: true });
   mkdirSync(join(repoDir, "refs"), { recursive: true });
 
+  // Preflight before the first network payload. This matters for direct
+  // Colibri artifacts (hundreds of GiB): discovering a full volume after the
+  // 140th shard is not a usable failure mode. Existing verified blobs and
+  // partial prefixes are credited exactly, so rerunning after interruption is
+  // both resumable and progressively cheaper.
+  const space = planDownloadSpace(listing.files, blobsDir);
+  if (space.remainingBytes > 0) {
+    const fs = statfsSync(blobsDir);
+    const availableBytes = fs.bavail * fs.bsize;
+    const requiredBytes = space.remainingBytes + DOWNLOAD_DISK_RESERVE_BYTES;
+    if (requiredBytes > availableBytes) {
+      throw new Error(
+        `insufficient disk space for ${repoId}: ${formatGib(space.remainingBytes)} ` +
+        `remains across ${space.remainingFiles} files, but only ` +
+        `${formatGib(availableBytes)} is available in ${hub} ` +
+        `(${formatGib(DOWNLOAD_DISK_RESERVE_BYTES)} safety reserve). ` +
+        `Free space or choose another cache directory, then rerun; ` +
+        `${formatGib(space.resumableBytes)} of partial data will resume.`,
+      );
+    }
+  }
+
   const status: DownloadStatus = {
     repoId, state: "active", currentFile: null,
     receivedBytes: 0, totalBytes: listing.files.reduce((a, f) => a + f.size, 0),
@@ -369,7 +460,7 @@ export async function downloadModel(
 
   try {
     for (const f of listing.files) {
-      const blobId = f.lfs?.oid ?? f.blobId;
+      const blobId = f.lfs?.sha256 ?? f.blobId;
       if (!blobId) throw new Error(`no blob id for ${f.rfilename} (API response missing ?blobs=true data)`);
       const blobPath = join(blobsDir, blobId);
 
@@ -378,7 +469,7 @@ export async function downloadModel(
         status.currentFile = f.rfilename;
         await downloadOne(url, token, blobPath, {
           size: f.size, name: f.rfilename,
-          ...(f.lfs ? { sha256: f.lfs.oid } : { sha1: f.blobId }),
+          ...(f.lfs ? { sha256: f.lfs.sha256 } : { sha1: f.blobId }),
         }, (file, received, total) => {
           status.receivedBytes = doneBytes + received;
           sampleRate(status.receivedBytes);

package/src/eval/ifeval.ts

@@ -1,138 +1,81 @@
-// IFEval — programmatically-verifiable instruction-following scoring (Zhou et
-// al. 2023, "Instruction-Following Evaluation for Large Language Models"). Each
-// instance pairs a prompt with one or more *verifiable* instructions; a response
-// is scored by deterministic checks (no LLM judge), so before/after ORPO
-// comparisons are exact and reproducible.
+// Compatibility facade for the canonical IFEval scorer.
 //
-// This implements the high-frequency instruction families (a working subset of
-// the ~25 in the reference suite, extensible via INSTRUCTIONS). Strict scoring:
-// an instance passes (strict) iff every instruction passes; we also report the
-// per-instruction pass rate. The official metrics are prompt-level and
-// instruction-level accuracy — both derived from `scoreInstance`.
+// The complete verifier registry and strict/loose aggregation contract live in
+// `tasks/ifeval.ts`, which is also used by `scripts/eval.ts`. Keep this module
+// as the stable import surface used by `scripts/run-ifeval.ts` and older
+// callers; it must not grow a second verifier implementation.
 
-export interface IFEvalInstance {
-  /** The prompt text (used by instructions that reference it, e.g. "repeat the request"). */
-  prompt: string;
-  /** Instruction ids, e.g. "keywords:existence". */
-  instruction_id_list: string[];
-  /** Per-instruction kwargs, aligned with instruction_id_list. */
-  kwargs: Array<Record<string, unknown>>;
-}
+import {
+  scoreIfevalInstance,
+  scoreIfevalPairs,
+  type IfevalInstance,
+  type IfevalResult,
+} from "./tasks/ifeval";
+
+export {
+  SUPPORTED_INSTRUCTIONS,
+  looseClean,
+  scoreIfevalInstance,
+  scoreIfevalPairs,
+  stripThinking,
+  verifyResponse,
+  type IfevalCoverage,
+  type IfevalInstance,
+  type IfevalInstanceScore,
+  type IfevalPair,
+  type IfevalResult,
+  type IfevalVerification,
+} from "./tasks/ifeval";
+
+/** Historical spelling retained for callers of this module. */
+export type IFEvalInstance = IfevalInstance;
 
 export interface InstanceResult {
-  /** One pass/fail per instruction (aligned with instruction_id_list). */
+  /** Unknown instructions are true here per the canonical prompt contract. */
   perInstruction: boolean[];
-  /** Strict: all instructions passed. */
   followedAll: boolean;
+  /** Consult this field rather than interpreting unknown IDs as verified. */
+  unhandled: string[];
 }
 
-type Checker = (response: string, kw: Record<string, unknown>, inst: IFEvalInstance) => boolean;
-
-const words = (s: string): string[] => s.trim().split(/\s+/).filter(Boolean);
-const sentences = (s: string): string[] =>
-  s.split(/(?<=[.!?])\s+/).map((x) => x.trim()).filter(Boolean);
-const paragraphs = (s: string): string[] =>
-  s.split(/\n\s*\n/).map((x) => x.trim()).filter(Boolean);
-const norm = (s: string): string => s.toLowerCase();
-
-const cmp = (n: number, rel: string | undefined, target: number): boolean => {
-  switch (rel) {
-    case "at least": return n >= target;
-    case "at most": return n <= target;
-    case "less than": return n < target;
-    case "exactly": return n === target;
-    default: return n >= target; // IFEval default for "around"/unspecified is lenient ≥
-  }
-};
-
-/** The verifiable-instruction registry. Keyed by IFEval instruction_id. */
-export const INSTRUCTIONS: Record<string, Checker> = {
-  // ---- keywords ----
-  "keywords:existence": (r, kw) =>
-    (kw.keywords as string[]).every((k) => norm(r).includes(norm(k))),
-  "keywords:frequency": (r, kw) => {
-    const k = norm(kw.keyword as string);
-    const count = norm(r).split(k).length - 1;
-    return cmp(count, kw.relation as string, kw.frequency as number);
-  },
-  "keywords:forbidden_words": (r, kw) =>
-    !(kw.forbidden_words as string[]).some((k) => norm(r).includes(norm(k))),
-
-  // ---- length constraints ----
-  "length_constraints:number_words": (r, kw) =>
-    cmp(words(r).length, kw.relation as string, kw.num_words as number),
-  "length_constraints:number_sentences": (r, kw) =>
-    cmp(sentences(r).length, kw.relation as string, kw.num_sentences as number),
-  "length_constraints:number_paragraphs": (r, kw) =>
-    cmp(paragraphs(r).length, kw.relation as string, kw.num_paragraphs as number),
-
-  // ---- detectable format ----
-  "detectable_format:number_bullet_lists": (r, kw) => {
-    const bullets = r.split("\n").filter((l) => /^\s*[*\-]\s+/.test(l)).length;
-    return cmp(bullets, kw.relation as string, kw.num_bullets as number);
-  },
-  "detectable_format:number_highlighted_sections": (r, kw) => {
-    const hi = (r.match(/\*[^*\n]+\*/g) ?? []).length;
-    return cmp(hi, kw.relation as string, kw.num_highlights as number);
-  },
-  "detectable_format:title": (r) => /<<[^>\n]+>>/.test(r),
-  "detectable_format:json_format": (r) => {
-    const t = r.trim().replace(/^```(?:json)?\s*|\s*```$/g, "").trim();
-    try { JSON.parse(t); return true; } catch { return false; }
-  },
-
-  // ---- case ----
-  "change_case:english_lowercase": (r) => r === r.toLowerCase(),
-  "change_case:english_capital": (r) => r === r.toUpperCase(),
-
-  // ---- start / end ----
-  "startend:end_checker": (r, kw) =>
-    norm(r.trimEnd()).endsWith(norm((kw.end_phrase as string).trim())),
-  "startend:quotation": (r) => {
-    const t = r.trim();
-    return t.startsWith('"') && t.endsWith('"') && t.length >= 2;
-  },
-
-  // ---- punctuation ----
-  "punctuation:no_comma": (r) => !r.includes(","),
-};
-
-/** Score one IFEval instance (strict). Unknown instruction ids count as failed
- *  (so coverage gaps never inflate the score) — filter the dataset to the
- *  supported ids, or extend INSTRUCTIONS, to avoid that. */
-export function scoreInstance(instance: IFEvalInstance, response: string): InstanceResult {
-  const perInstruction = instance.instruction_id_list.map((id, i) => {
-    const check = INSTRUCTIONS[id];
-    if (!check) return false;
-    try { return check(response, instance.kwargs[i] ?? {}, instance); } catch { return false; }
-  });
-  return { perInstruction, followedAll: perInstruction.every(Boolean) };
+/**
+ * Historical strict-only helper, now delegated to the canonical scorer.
+ * New code should prefer `scoreIfevalInstance` for both strict and loose modes.
+ */
+export function scoreInstance(
+  instance: IFEvalInstance,
+  response: string,
+): InstanceResult {
+  const { strict } = scoreIfevalInstance(instance, response);
+  return {
+    perInstruction: strict.instructionPasses.map((result) => result ?? true),
+    followedAll: strict.pass,
+    unhandled: strict.unhandled,
+  };
 }
 
-export interface IFEvalReport {
+export interface IFEvalReport extends IfevalResult {
+  /** Historical alias for nTotal. */
   n: number;
-  /** Fraction of instances where ALL instructions were followed. */
+  /** Historical alias for strictAcc (prompt-level strict). */
   promptAccuracy: number;
-  /** Fraction of individual instructions followed. */
+  /** Historical alias for strictInstructionAcc. */
   instructionAccuracy: number;
 }
 
-/** Aggregate the official prompt-level + instruction-level accuracies over a set
- *  of (instance, response) pairs. */
-export function aggregate(pairs: Array<{ instance: IFEvalInstance; response: string }>): IFEvalReport {
-  let prompts = 0, instrFollowed = 0, instrTotal = 0;
-  for (const { instance, response } of pairs) {
-    const r = scoreInstance(instance, response);
-    if (r.followedAll) prompts++;
-    instrFollowed += r.perInstruction.filter(Boolean).length;
-    instrTotal += r.perInstruction.length;
-  }
+/**
+ * Historical aggregation helper with unambiguous canonical fields plus aliases.
+ * Unknown instructions do not fail prompt accuracy, are excluded from
+ * instruction accuracy, and remain visible in `coverage`.
+ */
+export function aggregate(
+  pairs: Array<{ instance: IFEvalInstance; response: string }>,
+): IFEvalReport {
+  const result = scoreIfevalPairs(pairs);
   return {
-    n: pairs.length,
-    promptAccuracy: pairs.length ? prompts / pairs.length : 0,
-    instructionAccuracy: instrTotal ? instrFollowed / instrTotal : 0,
+    ...result,
+    n: result.nTotal,
+    promptAccuracy: result.strictAcc,
+    instructionAccuracy: result.strictInstructionAcc,
   };
 }
-
-/** Instruction ids this implementation supports (filter datasets to these). */
-export const SUPPORTED_INSTRUCTIONS = new Set(Object.keys(INSTRUCTIONS));

package/src/eval/runner.ts

@@ -9,8 +9,12 @@ import { generate, type GenerateOptions } from "../generate";
 import * as ops from "../mlx/ops";
 import { ChatTemplate } from "../chat-template";
 import { loadModelConfig, type ModelConfig } from "../config";
-import { Weights } from "../weights";
-import { createModel, type RuntimeModel } from "../model/factory";
+import {
+  openModel,
+  type Glm52RuntimeOpenOptions,
+  type RuntimeModel,
+} from "../model/factory";
+import { isGlm52Config } from "../model/support";
 import { DiffusionGemmaModel } from "../model/diffusion-gemma";
 import { loadTokenizer, type LoadedTokenizer } from "../tokenizer";
 import { resolveModelDir } from "./kl";
@@ -65,11 +69,16 @@ export interface TaskModel {
   activeAdapters?: string[];
 }
 
-export async function loadTaskModel(query: string, adapterDir?: string): Promise<TaskModel> {
+export async function loadTaskModel(
+  query: string,
+  adapterDir?: string,
+  runtimeOptions: Glm52RuntimeOpenOptions = {},
+): Promise<TaskModel> {
   const dir = resolveModelDir(query);
   const config = await loadModelConfig(dir);
-  const weights = await Weights.open(dir);
-  const model = createModel(weights, config);
+  if (adapterDir && isGlm52Config(config))
+    throw new Error("GLM-5.2 does not support LoRA adapters");
+  const model = await openModel(dir, runtimeOptions);
   const activeAdapters: string[] = [];
   if (adapterDir) {
     // Mount a trained LoRA adapter so the eval measures base+adapter (e.g. the