mlx-bun 0.0.11 → 0.0.12

package/src/download.ts

@@ -4,7 +4,7 @@
 // lore"): Xet stalls (we speak plain HTTPS resolve/CDN), no resume on
 // flaky links (we Range-resume partial blobs), and silent corruption
 // (every blob is verified — sha256 for LFS files against the API's
-// oid, git-blob sha1 for small files against blobId).
+// normalized LFS digest, git-blob sha1 for small files against blobId).
 //
 // Writes the exact huggingface_hub cache layout so the registry and
 // loaders find the result with zero changes:
@@ -18,6 +18,7 @@
 import {
   mkdirSync, existsSync, statSync, createWriteStream, renameSync, rmSync,
   symlinkSync, readFileSync, openSync, closeSync, readSync, writeSync,
+  statfsSync,
 } from "node:fs";
 import { dirname, join } from "node:path";
 import { DEFAULT_HUB } from "./registry";
@@ -29,8 +30,8 @@ export interface RepoFile {
   size: number;
   /** git blob sha1 (non-LFS identity, and the blob filename for non-LFS). */
   blobId: string;
-  /** LFS sha256 + size; present for large files. oid is the blob filename. */
-  lfs?: { oid: string; size: number };
+  /** LFS sha256 + size; present for large files. sha256 is the blob filename. */
+  lfs?: { sha256: string; size: number };
 }
 
 export interface RepoListing {
@@ -47,6 +48,56 @@ export interface DownloadOptions {
   onProgress?: (file: string, received: number, total: number) => void;
 }
 
+export interface DownloadSpacePlan {
+  /** Bytes still absent after crediting verified blobs and resumable prefixes. */
+  remainingBytes: number;
+  /** Files that still need at least one byte. */
+  remainingFiles: number;
+  /** Files already complete in the shared HF blob store. */
+  completeFiles: number;
+  /** Existing `.incomplete` bytes that will be rehashed and resumed. */
+  resumableBytes: number;
+}
+
+/** One GiB for filesystem metadata, the final refs write, and normal host
+ * operation. The artifact bytes themselves are exact; this is deliberately a
+ * small fixed guard rather than a percentage that would add tens of GiB to a
+ * Colibri download. */
+export const DOWNLOAD_DISK_RESERVE_BYTES = 2 ** 30;
+
+/** Compute the exact remaining payload without opening or hashing any blob.
+ * A valid-size final blob is already complete; an in-range `.incomplete`
+ * prefix is credited because downloadOneLocked resumes it in place. */
+export function planDownloadSpace(
+  files: readonly RepoFile[],
+  blobsDir: string,
+): DownloadSpacePlan {
+  let remainingBytes = 0;
+  let remainingFiles = 0;
+  let completeFiles = 0;
+  let resumableBytes = 0;
+  for (const file of files) {
+    const blobId = file.lfs?.sha256 ?? file.blobId;
+    if (!blobId) throw new Error(`no blob id for ${file.rfilename}`);
+    const blobPath = join(blobsDir, blobId);
+    if (existsSync(blobPath) && statSync(blobPath).size === file.size) {
+      completeFiles++;
+      continue;
+    }
+    const partPath = `${blobPath}.incomplete`;
+    const partBytes = existsSync(partPath) ? statSync(partPath).size : 0;
+    const credited = partBytes >= 0 && partBytes <= file.size ? partBytes : 0;
+    resumableBytes += credited;
+    remainingBytes += file.size - credited;
+    remainingFiles++;
+  }
+  return { remainingBytes, remainingFiles, completeFiles, resumableBytes };
+}
+
+function formatGib(bytes: number): string {
+  return `${(bytes / 2 ** 30).toFixed(2)} GiB`;
+}
+
 /** HF auth: explicit > env > the file `hf auth login` writes. */
 export function hfToken(): string | null {
   if (process.env.HF_TOKEN) return process.env.HF_TOKEN;
@@ -161,16 +212,34 @@ export async function listRepoFiles(
     );
   const body = (await res.json()) as {
     sha: string;
-    siblings: { rfilename: string; size?: number; blobId?: string; lfs?: { oid: string; size: number } }[];
+    siblings: {
+      rfilename: string;
+      size?: number;
+      blobId?: string;
+      // The current HF API calls this field `sha256`. Accept the older `oid`
+      // spelling too, but normalize it before the download path sees it.
+      lfs?: { sha256?: string; oid?: string; size: number };
+    }[];
   };
   return {
     sha: body.sha,
-    files: body.siblings.map((s) => ({
-      rfilename: s.rfilename,
-      size: s.lfs?.size ?? s.size ?? 0,
-      blobId: s.blobId ?? "",
-      lfs: s.lfs,
-    })),
+    files: body.siblings.map((s) => {
+      let lfs: RepoFile["lfs"];
+      if (s.lfs) {
+        const sha256 = s.lfs.sha256 ?? s.lfs.oid;
+        if (!sha256 || !/^[0-9a-f]{64}$/i.test(sha256))
+          throw new Error(`HF API returned an invalid LFS sha256 for ${s.rfilename}`);
+        lfs = { sha256: sha256.toLowerCase(), size: s.lfs.size };
+      }
+      if (!lfs && (!s.blobId || !/^[0-9a-f]{40}$/.test(s.blobId)))
+        throw new Error(`HF API returned an invalid git blob SHA-1 for ${s.rfilename}`);
+      return {
+        rfilename: s.rfilename,
+        size: lfs?.size ?? s.size ?? 0,
+        blobId: s.blobId ?? "",
+        lfs,
+      };
+    }),
   };
 }
 
@@ -349,6 +418,28 @@ export async function downloadModel(
   mkdirSync(blobsDir, { recursive: true });
   mkdirSync(join(repoDir, "refs"), { recursive: true });
 
+  // Preflight before the first network payload. This matters for direct
+  // Colibri artifacts (hundreds of GiB): discovering a full volume after the
+  // 140th shard is not a usable failure mode. Existing verified blobs and
+  // partial prefixes are credited exactly, so rerunning after interruption is
+  // both resumable and progressively cheaper.
+  const space = planDownloadSpace(listing.files, blobsDir);
+  if (space.remainingBytes > 0) {
+    const fs = statfsSync(blobsDir);
+    const availableBytes = fs.bavail * fs.bsize;
+    const requiredBytes = space.remainingBytes + DOWNLOAD_DISK_RESERVE_BYTES;
+    if (requiredBytes > availableBytes) {
+      throw new Error(
+        `insufficient disk space for ${repoId}: ${formatGib(space.remainingBytes)} ` +
+        `remains across ${space.remainingFiles} files, but only ` +
+        `${formatGib(availableBytes)} is available in ${hub} ` +
+        `(${formatGib(DOWNLOAD_DISK_RESERVE_BYTES)} safety reserve). ` +
+        `Free space or choose another cache directory, then rerun; ` +
+        `${formatGib(space.resumableBytes)} of partial data will resume.`,
+      );
+    }
+  }
+
   const status: DownloadStatus = {
     repoId, state: "active", currentFile: null,
     receivedBytes: 0, totalBytes: listing.files.reduce((a, f) => a + f.size, 0),
@@ -369,7 +460,7 @@ export async function downloadModel(
 
   try {
     for (const f of listing.files) {
-      const blobId = f.lfs?.oid ?? f.blobId;
+      const blobId = f.lfs?.sha256 ?? f.blobId;
       if (!blobId) throw new Error(`no blob id for ${f.rfilename} (API response missing ?blobs=true data)`);
       const blobPath = join(blobsDir, blobId);
 
@@ -378,7 +469,7 @@ export async function downloadModel(
         status.currentFile = f.rfilename;
         await downloadOne(url, token, blobPath, {
           size: f.size, name: f.rfilename,
-          ...(f.lfs ? { sha256: f.lfs.oid } : { sha1: f.blobId }),
+          ...(f.lfs ? { sha256: f.lfs.sha256 } : { sha1: f.blobId }),
         }, (file, received, total) => {
           status.receivedBytes = doneBytes + received;
           sampleRate(status.receivedBytes);

package/src/eval/ifeval.ts

@@ -1,138 +1,81 @@
-// IFEval — programmatically-verifiable instruction-following scoring (Zhou et
-// al. 2023, "Instruction-Following Evaluation for Large Language Models"). Each
-// instance pairs a prompt with one or more *verifiable* instructions; a response
-// is scored by deterministic checks (no LLM judge), so before/after ORPO
-// comparisons are exact and reproducible.
+// Compatibility facade for the canonical IFEval scorer.
 //
-// This implements the high-frequency instruction families (a working subset of
-// the ~25 in the reference suite, extensible via INSTRUCTIONS). Strict scoring:
-// an instance passes (strict) iff every instruction passes; we also report the
-// per-instruction pass rate. The official metrics are prompt-level and
-// instruction-level accuracy — both derived from `scoreInstance`.
+// The complete verifier registry and strict/loose aggregation contract live in
+// `tasks/ifeval.ts`, which is also used by `scripts/eval.ts`. Keep this module
+// as the stable import surface used by `scripts/run-ifeval.ts` and older
+// callers; it must not grow a second verifier implementation.
 
-export interface IFEvalInstance {
-  /** The prompt text (used by instructions that reference it, e.g. "repeat the request"). */
-  prompt: string;
-  /** Instruction ids, e.g. "keywords:existence". */
-  instruction_id_list: string[];
-  /** Per-instruction kwargs, aligned with instruction_id_list. */
-  kwargs: Array<Record<string, unknown>>;
-}
+import {
+  scoreIfevalInstance,
+  scoreIfevalPairs,
+  type IfevalInstance,
+  type IfevalResult,
+} from "./tasks/ifeval";
+
+export {
+  SUPPORTED_INSTRUCTIONS,
+  looseClean,
+  scoreIfevalInstance,
+  scoreIfevalPairs,
+  stripThinking,
+  verifyResponse,
+  type IfevalCoverage,
+  type IfevalInstance,
+  type IfevalInstanceScore,
+  type IfevalPair,
+  type IfevalResult,
+  type IfevalVerification,
+} from "./tasks/ifeval";
+
+/** Historical spelling retained for callers of this module. */
+export type IFEvalInstance = IfevalInstance;
 
 export interface InstanceResult {
-  /** One pass/fail per instruction (aligned with instruction_id_list). */
+  /** Unknown instructions are true here per the canonical prompt contract. */
   perInstruction: boolean[];
-  /** Strict: all instructions passed. */
   followedAll: boolean;
+  /** Consult this field rather than interpreting unknown IDs as verified. */
+  unhandled: string[];
 }
 
-type Checker = (response: string, kw: Record<string, unknown>, inst: IFEvalInstance) => boolean;
-
-const words = (s: string): string[] => s.trim().split(/\s+/).filter(Boolean);
-const sentences = (s: string): string[] =>
-  s.split(/(?<=[.!?])\s+/).map((x) => x.trim()).filter(Boolean);
-const paragraphs = (s: string): string[] =>
-  s.split(/\n\s*\n/).map((x) => x.trim()).filter(Boolean);
-const norm = (s: string): string => s.toLowerCase();
-
-const cmp = (n: number, rel: string | undefined, target: number): boolean => {
-  switch (rel) {
-    case "at least": return n >= target;
-    case "at most": return n <= target;
-    case "less than": return n < target;
-    case "exactly": return n === target;
-    default: return n >= target; // IFEval default for "around"/unspecified is lenient ≥
-  }
-};
-
-/** The verifiable-instruction registry. Keyed by IFEval instruction_id. */
-export const INSTRUCTIONS: Record<string, Checker> = {
-  // ---- keywords ----
-  "keywords:existence": (r, kw) =>
-    (kw.keywords as string[]).every((k) => norm(r).includes(norm(k))),
-  "keywords:frequency": (r, kw) => {
-    const k = norm(kw.keyword as string);
-    const count = norm(r).split(k).length - 1;
-    return cmp(count, kw.relation as string, kw.frequency as number);
-  },
-  "keywords:forbidden_words": (r, kw) =>
-    !(kw.forbidden_words as string[]).some((k) => norm(r).includes(norm(k))),
-
-  // ---- length constraints ----
-  "length_constraints:number_words": (r, kw) =>
-    cmp(words(r).length, kw.relation as string, kw.num_words as number),
-  "length_constraints:number_sentences": (r, kw) =>
-    cmp(sentences(r).length, kw.relation as string, kw.num_sentences as number),
-  "length_constraints:number_paragraphs": (r, kw) =>
-    cmp(paragraphs(r).length, kw.relation as string, kw.num_paragraphs as number),
-
-  // ---- detectable format ----
-  "detectable_format:number_bullet_lists": (r, kw) => {
-    const bullets = r.split("\n").filter((l) => /^\s*[*\-]\s+/.test(l)).length;
-    return cmp(bullets, kw.relation as string, kw.num_bullets as number);
-  },
-  "detectable_format:number_highlighted_sections": (r, kw) => {
-    const hi = (r.match(/\*[^*\n]+\*/g) ?? []).length;
-    return cmp(hi, kw.relation as string, kw.num_highlights as number);
-  },
-  "detectable_format:title": (r) => /<<[^>\n]+>>/.test(r),
-  "detectable_format:json_format": (r) => {
-    const t = r.trim().replace(/^```(?:json)?\s*|\s*```$/g, "").trim();
-    try { JSON.parse(t); return true; } catch { return false; }
-  },
-
-  // ---- case ----
-  "change_case:english_lowercase": (r) => r === r.toLowerCase(),
-  "change_case:english_capital": (r) => r === r.toUpperCase(),
-
-  // ---- start / end ----
-  "startend:end_checker": (r, kw) =>
-    norm(r.trimEnd()).endsWith(norm((kw.end_phrase as string).trim())),
-  "startend:quotation": (r) => {
-    const t = r.trim();
-    return t.startsWith('"') && t.endsWith('"') && t.length >= 2;
-  },
-
-  // ---- punctuation ----
-  "punctuation:no_comma": (r) => !r.includes(","),
-};
-
-/** Score one IFEval instance (strict). Unknown instruction ids count as failed
- *  (so coverage gaps never inflate the score) — filter the dataset to the
- *  supported ids, or extend INSTRUCTIONS, to avoid that. */
-export function scoreInstance(instance: IFEvalInstance, response: string): InstanceResult {
-  const perInstruction = instance.instruction_id_list.map((id, i) => {
-    const check = INSTRUCTIONS[id];
-    if (!check) return false;
-    try { return check(response, instance.kwargs[i] ?? {}, instance); } catch { return false; }
-  });
-  return { perInstruction, followedAll: perInstruction.every(Boolean) };
+/**
+ * Historical strict-only helper, now delegated to the canonical scorer.
+ * New code should prefer `scoreIfevalInstance` for both strict and loose modes.
+ */
+export function scoreInstance(
+  instance: IFEvalInstance,
+  response: string,
+): InstanceResult {
+  const { strict } = scoreIfevalInstance(instance, response);
+  return {
+    perInstruction: strict.instructionPasses.map((result) => result ?? true),
+    followedAll: strict.pass,
+    unhandled: strict.unhandled,
+  };
 }
 
-export interface IFEvalReport {
+export interface IFEvalReport extends IfevalResult {
+  /** Historical alias for nTotal. */
   n: number;
-  /** Fraction of instances where ALL instructions were followed. */
+  /** Historical alias for strictAcc (prompt-level strict). */
   promptAccuracy: number;
-  /** Fraction of individual instructions followed. */
+  /** Historical alias for strictInstructionAcc. */
   instructionAccuracy: number;
 }
 
-/** Aggregate the official prompt-level + instruction-level accuracies over a set
- *  of (instance, response) pairs. */
-export function aggregate(pairs: Array<{ instance: IFEvalInstance; response: string }>): IFEvalReport {
-  let prompts = 0, instrFollowed = 0, instrTotal = 0;
-  for (const { instance, response } of pairs) {
-    const r = scoreInstance(instance, response);
-    if (r.followedAll) prompts++;
-    instrFollowed += r.perInstruction.filter(Boolean).length;
-    instrTotal += r.perInstruction.length;
-  }
+/**
+ * Historical aggregation helper with unambiguous canonical fields plus aliases.
+ * Unknown instructions do not fail prompt accuracy, are excluded from
+ * instruction accuracy, and remain visible in `coverage`.
+ */
+export function aggregate(
+  pairs: Array<{ instance: IFEvalInstance; response: string }>,
+): IFEvalReport {
+  const result = scoreIfevalPairs(pairs);
   return {
-    n: pairs.length,
-    promptAccuracy: pairs.length ? prompts / pairs.length : 0,
-    instructionAccuracy: instrTotal ? instrFollowed / instrTotal : 0,
+    ...result,
+    n: result.nTotal,
+    promptAccuracy: result.strictAcc,
+    instructionAccuracy: result.strictInstructionAcc,
   };
 }
-
-/** Instruction ids this implementation supports (filter datasets to these). */
-export const SUPPORTED_INSTRUCTIONS = new Set(Object.keys(INSTRUCTIONS));

package/src/eval/runner.ts

@@ -9,8 +9,12 @@ import { generate, type GenerateOptions } from "../generate";
 import * as ops from "../mlx/ops";
 import { ChatTemplate } from "../chat-template";
 import { loadModelConfig, type ModelConfig } from "../config";
-import { Weights } from "../weights";
-import { createModel, type RuntimeModel } from "../model/factory";
+import {
+  openModel,
+  type Glm52RuntimeOpenOptions,
+  type RuntimeModel,
+} from "../model/factory";
+import { isGlm52Config } from "../model/support";
 import { DiffusionGemmaModel } from "../model/diffusion-gemma";
 import { loadTokenizer, type LoadedTokenizer } from "../tokenizer";
 import { resolveModelDir } from "./kl";
@@ -65,11 +69,16 @@ export interface TaskModel {
   activeAdapters?: string[];
 }
 
-export async function loadTaskModel(query: string, adapterDir?: string): Promise<TaskModel> {
+export async function loadTaskModel(
+  query: string,
+  adapterDir?: string,
+  runtimeOptions: Glm52RuntimeOpenOptions = {},
+): Promise<TaskModel> {
   const dir = resolveModelDir(query);
   const config = await loadModelConfig(dir);
-  const weights = await Weights.open(dir);
-  const model = createModel(weights, config);
+  if (adapterDir && isGlm52Config(config))
+    throw new Error("GLM-5.2 does not support LoRA adapters");
+  const model = await openModel(dir, runtimeOptions);
   const activeAdapters: string[] = [];
   if (adapterDir) {
     // Mount a trained LoRA adapter so the eval measures base+adapter (e.g. the