mlx-bun 0.5.0 → 1.0.0

package/src/eval/perplexity.ts

@@ -1,164 +0,0 @@
-// Perplexity over a local text/JSONL dataset — mlx_lm.perplexity methodology.
-//
-// The reference (mlx_lm/perplexity.py, mlx-lm 0.31.3) does:
-//   1. tokenize dataset samples, visit them in a seeded random permutation,
-//   2. concatenate tokens into one stream and cut it into NON-OVERLAPPING
-//      rows of `sequence_length` (no sliding-window stride, no BOS re-seeding
-//      per row), keeping at most `num_samples` rows,
-//   3. per batch: logits = model(batch[:, :-1]) in f32, per-token
-//      cross-entropy vs batch[:, 1:] — EVERY position counts,
-//   4. ppl = exp(mean loss); SE via the delta method:
-//      se_ppl = ppl · std(losses, ddof=1) / √N.
-//
-// We reproduce 2-4 exactly. Step 1 differs only in the data source: the
-// reference pulls a Hugging Face dataset (network); we take a LOCAL .txt or
-// .jsonl file — never a download. The permutation RNG is a seeded
-// deterministic shuffle (not NumPy's), so row selection is reproducible per
-// seed but not bit-matched to the Python tool's sampling.
-//
-// The forward reuses the existing full-sequence machinery
-// (src/train/forward.ts trainForward → model.forwardHidden + logitsFromHidden).
-
-import { clearCache, Dtype } from "../mlx/ffi";
-import * as ops from "../mlx/ops";
-import { trainForward } from "../train/forward";
-import type { RuntimeModel } from "../model/factory";
-
-export interface PplDataOptions {
-  sequenceLength: number;
-  /** Max rows to keep (-1 / 0 = all available). */
-  numSamples: number;
-  /** Seed for the sample-order shuffle (reference default: 123). */
-  seed: number;
-}
-
-export interface PplResult {
-  ppl: number;
-  /** Delta-method standard error of the perplexity. */
-  standardError: number;
-  meanLoss: number;
-  tokens: number;
-  rows: number;
-}
-
-/** Deterministic PRNG (mulberry32) for the seeded sample shuffle. */
-function mulberry32(seed: number): () => number {
-  let s = seed >>> 0;
-  return () => {
-    s = (s + 0x6d2b79f5) >>> 0;
-    let t = s;
-    t = Math.imul(t ^ (t >>> 15), t | 1);
-    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
-    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
-  };
-}
-
-/** Split a raw dataset file into text samples: `.jsonl` → one sample per
- *  line from the `text` field; anything else → the whole file is one sample. */
-export function parseSamples(raw: string, path: string): string[] {
-  if (path.endsWith(".jsonl") || path.endsWith(".ndjson")) {
-    const samples: string[] = [];
-    for (const [i, line] of raw.split("\n").entries()) {
-      if (!line.trim()) continue;
-      let row: unknown;
-      try {
-        row = JSON.parse(line);
-      } catch {
-        throw new Error(`${path}:${i + 1}: not valid JSON`);
-      }
-      const text = (row as { text?: unknown }).text;
-      if (typeof text !== "string")
-        throw new Error(`${path}:${i + 1}: expected a {"text": "..."} row`);
-      samples.push(text);
-    }
-    return samples;
-  }
-  return raw.length > 0 ? [raw] : [];
-}
-
-/** Tokenize samples (seeded shuffle), concatenate, and pack into
- *  non-overlapping rows of `sequenceLength` tokens (reference load_data). */
-export function packRows(
-  sampleTokens: number[][],
-  opts: PplDataOptions,
-): Int32Array[] {
-  const { sequenceLength: L, numSamples, seed } = opts;
-  const order = sampleTokens.map((_, i) => i);
-  const rnd = mulberry32(seed);
-  for (let i = order.length - 1; i > 0; i--) {
-    const j = Math.floor(rnd() * (i + 1));
-    [order[i], order[j]] = [order[j]!, order[i]!];
-  }
-
-  const wanted = numSamples > 0 ? L * numSamples : Infinity;
-  const stream: number[] = [];
-  for (const idx of order) {
-    if (stream.length >= wanted) break;
-    stream.push(...sampleTokens[idx]!);
-  }
-
-  const nRows = Math.min(
-    Math.floor(stream.length / L),
-    numSamples > 0 ? numSamples : Infinity,
-  );
-  const rows: Int32Array[] = [];
-  for (let r = 0; r < nRows; r++)
-    rows.push(Int32Array.from(stream.slice(r * L, (r + 1) * L)));
-  return rows;
-}
-
-/** Evaluate perplexity over packed rows. Per batch (all rows are the same
- *  length, so the stock unpadded forward applies): f32 logits over
- *  batch[:, :-1], per-token CE vs batch[:, 1:], every position counted. */
-export function evalPpl(
-  model: RuntimeModel,
-  rows: Int32Array[],
-  batchSize: number,
-  onBatch?: (done: number, total: number) => void,
-): PplResult {
-  const L = rows[0]!.length;
-  const allLosses: number[] = [];
-  const numBatches = Math.ceil(rows.length / batchSize);
-
-  for (let s = 0, bi = 0; s < rows.length; s += batchSize, bi++) {
-    const batch = rows.slice(s, s + batchSize);
-    const B = batch.length;
-    const inputs: number[] = [];
-    const targets: number[] = [];
-    for (const row of batch) {
-      for (let i = 0; i < L - 1; i++) inputs.push(row[i]!);
-      for (let i = 1; i < L; i++) targets.push(row[i]!);
-    }
-    const ids = ops.fromInt32(inputs, [B, L - 1]);
-    const tgt = ops.fromInt32(targets, [B, L - 1, 1]);
-    const logits = trainForward(model, ids);
-    const f32 = logits.astype(Dtype.float32);
-    logits.dispose();
-    // CE[t] = logsumexp(logits[t]) − logits[t, target]  (f32, like nn.losses)
-    const lse = ops.logsumexpAxis(f32, 2, false); // [B, L-1]
-    const picked = ops.takeAlongAxis(f32, tgt, 2); // [B, L-1, 1]
-    f32.dispose();
-    const picked2d = ops.reshape(picked, [B, L - 1]);
-    picked.dispose();
-    const losses = ops.sub(lse, picked2d);
-    lse.dispose();
-    picked2d.dispose();
-    const vals = losses.toFloat32();
-    losses.dispose();
-    ids.dispose();
-    tgt.dispose();
-    for (const v of vals) allLosses.push(v);
-    clearCache();
-    onBatch?.(bi + 1, numBatches);
-  }
-
-  const n = allLosses.length;
-  const mean = allLosses.reduce((a, b) => a + b, 0) / n;
-  const ppl = Math.exp(mean);
-  // Sample std (ddof=1) → delta-method SE of the perplexity.
-  const varSum = allLosses.reduce((a, b) => a + (b - mean) ** 2, 0);
-  const std = n > 1 ? Math.sqrt(varSum / (n - 1)) : 0;
-  const standardError = ppl * (std / Math.sqrt(n));
-
-  return { ppl, standardError, meanLoss: mean, tokens: n, rows: rows.length };
-}

package/src/eval/quality-db.ts

@@ -1,92 +0,0 @@
-// Quality-eval DB — the capability/KL counterpart to evaldb.ts (which is
-// throughput-only: prefill_tps/decode_tps/peak_bytes). Lives in the SAME
-// sqlite file (~/.cache/mlx-bun/evals.sqlite) as a separate `quality_runs`
-// table so the throughput head-to-head and the quality gate sit together.
-//
-// One row = one (model, config) measured on one task (or the aggregate
-// "capability" / "kl" rows). `config_json` records the perf levers that
-// were active (compile / fused-sdpa / kv-quant) so a row is
-// self-describing: which arm of the head-to-head produced this number.
-
-import { Database } from "bun:sqlite";
-import { mkdirSync } from "node:fs";
-import { DEFAULT_EVAL_DB } from "../evaldb";
-
-const SCHEMA = `
-CREATE TABLE IF NOT EXISTS quality_runs (
-  id INTEGER PRIMARY KEY AUTOINCREMENT,
-  ts INTEGER NOT NULL,
-  model_path TEXT NOT NULL,
-  commit_sha TEXT,
-  task TEXT NOT NULL,            -- 'kl' | 'gsm8k' | 'mmlu' | ... | 'capability'
-  config_json TEXT NOT NULL,     -- {compile,fusedSdpa,kvQuant,...} active levers
-  n_samples INTEGER NOT NULL,
-  -- task accuracy (percent 0-100); null for kl-only rows
-  pct REAL,
-  -- KL drift block (null for task rows)
-  kl_mean REAL,
-  kl_median REAL,
-  kl_p95 REAL,
-  kl_ref TEXT,                   -- 'self:<FLAG>' | model path/id of the reference
-  -- aggregate capability score (null except on 'capability' rows)
-  capability_score REAL,
-  disk_gb REAL,
-  notes TEXT,
-  machine_state TEXT
-);
-`;
-
-export interface QualityRun {
-  modelPath: string;
-  commitSha?: string | null;
-  task: string;
-  /** Active perf levers when this number was produced. */
-  config: Record<string, string | boolean | number | null>;
-  nSamples: number;
-  pct?: number | null;
-  klMean?: number | null;
-  klMedian?: number | null;
-  klP95?: number | null;
-  klRef?: string | null;
-  capabilityScore?: number | null;
-  diskGb?: number | null;
-  notes?: string | null;
-  machineState?: string | null;
-}
-
-export class QualityDB {
-  readonly db: Database;
-
-  constructor(dbPath: string = DEFAULT_EVAL_DB) {
-    if (dbPath !== ":memory:") {
-      try { mkdirSync(dbPath.slice(0, dbPath.lastIndexOf("/")), { recursive: true }); } catch {}
-    }
-    this.db = new Database(dbPath);
-    this.db.exec(SCHEMA);
-  }
-
-  record(run: QualityRun): void {
-    this.db.prepare(`
-      INSERT INTO quality_runs (ts, model_path, commit_sha, task, config_json,
-        n_samples, pct, kl_mean, kl_median, kl_p95, kl_ref,
-        capability_score, disk_gb, notes, machine_state)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    `).run(
-      Date.now(), run.modelPath, run.commitSha ?? null, run.task,
-      JSON.stringify(run.config), run.nSamples,
-      run.pct ?? null, run.klMean ?? null, run.klMedian ?? null, run.klP95 ?? null,
-      run.klRef ?? null, run.capabilityScore ?? null, run.diskGb ?? null,
-      run.notes ?? null, run.machineState ?? null,
-    );
-  }
-
-  recent(limit = 20): Record<string, unknown>[] {
-    return this.db
-      .query("SELECT * FROM quality_runs ORDER BY ts DESC LIMIT ?")
-      .all(limit) as Record<string, unknown>[];
-  }
-
-  close(): void {
-    this.db.close();
-  }
-}

package/src/eval/reference-logits.ts

@@ -1,207 +0,0 @@
-// Reader for a saved bf16 TEACHER-LOGITS dump (format_version 1).
-//
-// Why the format exists: a bf16 teacher can never be resident on a 32 GB box
-// alongside the candidate, so the teacher's distributions are computed ONCE
-// (Python side, one model at a time) and written to disk as a top-k slice
-// plus the full-vocab logsumexp. Any candidate loaded through OUR engine can
-// then be scored against it (src/eval/kl.ts `evaluateKlVsReferenceLogits`)
-// without the teacher ever being co-resident.
-//
-// On-disk contract (fixed — this module reads it, it does not negotiate it):
-//
-//   <dir>/manifest.json  { format_version: 1, model, corpus_sha256, ctx_len,
-//                          n_seqs, top_k, positions_per_seq, created, notes }
-//   <dir>/tokens.bin     int32 LE [n_seqs][ctx_len] — teacher-forced ids
-//   <dir>/seq-<i>.bin    one record per r = 0..ctx_len-2; record r is the
-//                        distribution over the token at index r+1, i.e. the
-//                        distribution predicted from tokens[0..r] — so record
-//                        r lines up with candidate logits at POSITION r:
-//                          int32[top_k]   indices, descending by logit
-//                          float16[top_k] reference logits at those indices
-//                          float32        logsumexp over the FULL vocab
-//
-// Records are fixed-width (6·top_k + 4 bytes), so a range is one pread — the
-// scorer streams a chunk of records at a time and never holds a whole
-// sequence file (top_k 2048 × ctx 4096 ≈ 50 MB) in the JS heap.
-
-import { closeSync, existsSync, openSync, readFileSync, readSync, statSync } from "node:fs";
-
-export interface ReferenceLogitsManifest {
-  format_version: number;
-  model: string;
-  corpus_sha256: string;
-  ctx_len: number;
-  n_seqs: number;
-  top_k: number;
-  positions_per_seq: number;
-  created?: string;
-  notes?: string;
-}
-
-/** A contiguous run of records, flattened: row i of `indices`/`logits` is
- *  `[i*topK, (i+1)*topK)`. Logits are decoded from float16 to float32. */
-export interface ReferenceRecords {
-  /** Absolute record index of row 0. */
-  start: number;
-  count: number;
-  topK: number;
-  indices: Int32Array;    // [count · topK]
-  logits: Float32Array;   // [count · topK]
-  logsumexp: Float32Array; // [count] — over the FULL teacher vocab
-}
-
-const SUPPORTED_FORMAT_VERSION = 1;
-
-/** float16 bits → float32 (host-side; same expansion as MlxArray.toFloat32Host,
- *  kept local so reading a dump never touches the GPU). */
-function decodeFloat16(bits: number): number {
-  const sign = bits & 0x8000;
-  const exponent = (bits >>> 10) & 0x1f;
-  const fraction = bits & 0x03ff;
-  let value: number;
-  if (exponent === 0) value = fraction === 0 ? 0 : fraction * 2 ** -24;
-  else if (exponent === 0x1f) value = fraction === 0 ? Infinity : NaN;
-  else value = (1 + fraction / 1024) * 2 ** (exponent - 15);
-  return sign ? -value : value;
-}
-
-export class ReferenceLogitsDump {
-  readonly dir: string;
-  readonly manifest: ReferenceLogitsManifest;
-  readonly topK: number;
-  readonly ctxLen: number;
-  readonly nSeqs: number;
-  /** Records per sequence file — derived from the file, cross-checked
-   *  against the manifest. */
-  readonly positionsPerSeq: number;
-  readonly recordBytes: number;
-
-  #tokens: Int32Array;
-  #fd: number | null = null;
-  #fdSeq = -1;
-
-  private constructor(dir: string, manifest: ReferenceLogitsManifest, tokens: Int32Array, positionsPerSeq: number) {
-    this.dir = dir;
-    this.manifest = manifest;
-    this.topK = manifest.top_k;
-    this.ctxLen = manifest.ctx_len;
-    this.nSeqs = manifest.n_seqs;
-    this.positionsPerSeq = positionsPerSeq;
-    this.recordBytes = manifest.top_k * 6 + 4;
-    this.#tokens = tokens;
-  }
-
-  static open(dir: string): ReferenceLogitsDump {
-    const manifestPath = `${dir}/manifest.json`;
-    if (!existsSync(manifestPath))
-      throw new Error(`reference-logits: no manifest.json in ${dir}`);
-    const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as ReferenceLogitsManifest;
-    if (manifest.format_version !== SUPPORTED_FORMAT_VERSION)
-      throw new Error(
-        `reference-logits: format_version ${manifest.format_version} unsupported ` +
-        `(this reader implements ${SUPPORTED_FORMAT_VERSION})`,
-      );
-    for (const k of ["ctx_len", "n_seqs", "top_k"] as const)
-      if (!Number.isInteger(manifest[k]) || manifest[k] <= 0)
-        throw new Error(`reference-logits: manifest.${k} must be a positive integer (got ${manifest[k]})`);
-
-    const tokensBytes = readFileSync(`${dir}/tokens.bin`);
-    const wantTokenBytes = manifest.n_seqs * manifest.ctx_len * 4;
-    if (tokensBytes.byteLength !== wantTokenBytes)
-      throw new Error(
-        `reference-logits: tokens.bin is ${tokensBytes.byteLength} B, expected ` +
-        `${wantTokenBytes} B (n_seqs ${manifest.n_seqs} × ctx_len ${manifest.ctx_len} × int32)`,
-      );
-    // Copy: the Buffer's byteOffset need not be int32-aligned for a view.
-    const tokens = new Int32Array(
-      tokensBytes.buffer.slice(tokensBytes.byteOffset, tokensBytes.byteOffset + tokensBytes.byteLength),
-    );
-
-    const recordBytes = manifest.top_k * 6 + 4;
-    const seq0 = `${dir}/seq-0.bin`;
-    if (!existsSync(seq0)) throw new Error(`reference-logits: missing ${seq0}`);
-    const size = statSync(seq0).size;
-    if (size % recordBytes !== 0)
-      throw new Error(
-        `reference-logits: seq-0.bin is ${size} B, not a multiple of the ` +
-        `${recordBytes} B record (top_k ${manifest.top_k})`,
-      );
-    const positions = size / recordBytes;
-    if (positions > manifest.ctx_len - 1)
-      throw new Error(
-        `reference-logits: seq-0.bin holds ${positions} records but ctx_len ` +
-        `${manifest.ctx_len} allows at most ${manifest.ctx_len - 1}`,
-      );
-    if (Number.isInteger(manifest.positions_per_seq) && manifest.positions_per_seq !== positions)
-      console.warn(
-        `[reference-logits] manifest.positions_per_seq=${manifest.positions_per_seq} but ` +
-        `seq-0.bin holds ${positions} records — using the file`,
-      );
-    return new ReferenceLogitsDump(dir, manifest, tokens, positions);
-  }
-
-  /** Teacher-forced token ids of sequence `seq` (length ctx_len). */
-  tokens(seq: number): number[] {
-    this.#assertSeq(seq);
-    const off = seq * this.ctxLen;
-    return Array.from(this.#tokens.subarray(off, off + this.ctxLen));
-  }
-
-  /** Records [lo, hi) of sequence `seq` — one pread, decoded to float32. */
-  readRecords(seq: number, lo: number, hi: number): ReferenceRecords {
-    this.#assertSeq(seq);
-    if (lo < 0 || hi > this.positionsPerSeq || hi <= lo)
-      throw new Error(
-        `reference-logits: record range [${lo},${hi}) out of bounds for ` +
-        `${this.positionsPerSeq} records`,
-      );
-    const count = hi - lo;
-    const K = this.topK;
-    const buf = Buffer.allocUnsafe(count * this.recordBytes);
-    const fd = this.#open(seq);
-    const read = readSync(fd, buf, 0, buf.byteLength, lo * this.recordBytes);
-    if (read !== buf.byteLength)
-      throw new Error(`reference-logits: short read on seq-${seq}.bin (${read}/${buf.byteLength} B)`);
-
-    const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
-    const indices = new Int32Array(count * K);
-    const logits = new Float32Array(count * K);
-    const logsumexp = new Float32Array(count);
-    for (let r = 0; r < count; r++) {
-      const base = r * this.recordBytes;
-      const row = r * K;
-      for (let j = 0; j < K; j++) indices[row + j] = dv.getInt32(base + j * 4, true);
-      const lbase = base + K * 4;
-      for (let j = 0; j < K; j++) logits[row + j] = decodeFloat16(dv.getUint16(lbase + j * 2, true));
-      logsumexp[r] = dv.getFloat32(lbase + K * 2, true);
-    }
-    return { start: lo, count, topK: K, indices, logits, logsumexp };
-  }
-
-  close(): void {
-    if (this.#fd !== null) closeSync(this.#fd);
-    this.#fd = null;
-    this.#fdSeq = -1;
-  }
-
-  #open(seq: number): number {
-    if (this.#fdSeq === seq && this.#fd !== null) return this.#fd;
-    if (this.#fd !== null) closeSync(this.#fd);
-    const path = `${this.dir}/seq-${seq}.bin`;
-    if (!existsSync(path)) throw new Error(`reference-logits: missing ${path}`);
-    const size = statSync(path).size;
-    if (size < this.positionsPerSeq * this.recordBytes)
-      throw new Error(
-        `reference-logits: seq-${seq}.bin is ${size} B, short of the ` +
-        `${this.positionsPerSeq * this.recordBytes} B seq-0.bin defines`,
-      );
-    this.#fd = openSync(path, "r");
-    this.#fdSeq = seq;
-    return this.#fd;
-  }
-
-  #assertSeq(seq: number): void {
-    if (!Number.isInteger(seq) || seq < 0 || seq >= this.nSeqs)
-      throw new Error(`reference-logits: sequence ${seq} out of range (n_seqs ${this.nSeqs})`);
-  }
-}

package/src/eval/runner.ts

@@ -1,124 +0,0 @@
-import type { GenerateOptions } from "../generate";
-// Task-eval runner: load a model and greedily complete prompts through the
-// REAL serving path. Unlike the M0 KL (compat forward() + plain caches),
-// this drives generate() with the model's kv_config, so generation runs the
-// mixed-4/8-bit quantized-KV path the generated class actually serves.
-
-import { homedir } from "node:os";
-import { readFileSync } from "node:fs";
-import { ChatTemplate } from "../chat-template";
-import { loadModelConfig, type ModelConfig } from "../config";
-import { runtimeValue } from "../runtime-config";
-import {
-  openModel,
-  type Glm52RuntimeOpenOptions,
-  type RuntimeModel,
-} from "../model/factory";
-import { isGlm52Config } from "../model/support";
-import { loadTokenizer, type LoadedTokenizer } from "../tokenizer";
-import { resolveModelDir } from "./kl";
-
-export const EVAL_DATA_DIR = runtimeValue("MLX_BUN_EVAL_DATA") ?? `${homedir()}/.cache/mlx-bun/eval-data`;
-
-/** Read a jsonl dataset exported by scripts/oracle/export-datasets.py. */
-export function loadJsonl<T = Record<string, unknown>>(name: string): T[] {
-  const path = `${EVAL_DATA_DIR}/${name}.jsonl`;
-  const out: T[] = [];
-  for (const line of readFileSync(path, "utf8").split("\n")) {
-    const t = line.trim();
-    if (t) out.push(JSON.parse(t) as T);
-  }
-  return out;
-}
-
-/** Deterministic seeded subsample of [0, total): mulberry32 shuffle, sorted. */
-export function sampleIndices(total: number, n: number, seed = 42): number[] {
-  let s = seed >>> 0;
-  const rand = (): number => {
-    s = (s + 0x6d2b79f5) | 0;
-    let t = Math.imul(s ^ (s >>> 15), 1 | s);
-    t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
-    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
-  };
-  const idx = Array.from({ length: total }, (_, i) => i);
-  for (let i = total - 1; i > 0; i--) {
-    const j = Math.floor(rand() * (i + 1));
-    [idx[i], idx[j]] = [idx[j]!, idx[i]!];
-  }
-  return idx.slice(0, Math.min(n, total)).sort((a, b) => a - b);
-}
-
-/** Sampler keys an eval arm may force across every task (greedy/HLG/curve). */
-type SamplerArm = Partial<Pick<GenerateOptions, "temperature" | "topP" | "topK" | "seed" | "hlg" | "curve">>;
-
-export interface TaskModel {
-  completionClient?: import("../contracts/completion").CompletionClient<{ body: string; options?: GenOpts }, string>;
-  model: RuntimeModel;
-  tokenizer: LoadedTokenizer;
-  template: ChatTemplate | null;
-  config: ModelConfig;
-  dir: string;
-  /** When set, FORCES this sampler on every generateText call, overriding the
-   *  task's own (greedy) default — used to run the whole suite under one sampler
-   *  arm (e.g. the v2 curve vs the default chat recipe) for the degradation gate. */
-  samplerOverride?: SamplerArm;
-  /** Mounted-adapter ids to ACTIVATE on every forward/generate. Mounting an adapter
-   *  only stores it on the linears; `loraState.active` is what actually applies it,
-   *  and generation resets it to [] after each call. So generateText re-passes this,
-   *  and direct-forward tasks (MMLU) re-assert it — else the eval silently runs base. */
-  activeAdapters?: string[];
-}
-
-export async function loadTaskModel(
-  query: string,
-  adapterDir?: string,
-  runtimeOptions: Glm52RuntimeOpenOptions = {},
-): Promise<TaskModel> {
-  const dir = resolveModelDir(query);
-  const config = await loadModelConfig(dir);
-  if (adapterDir && isGlm52Config(config))
-    throw new Error("GLM-5.2 does not support LoRA adapters");
-  const model = await openModel(dir, runtimeOptions);
-  const activeAdapters: string[] = [];
-  if (adapterDir) {
-    // Mount a trained LoRA adapter so the eval measures base+adapter (e.g. the
-    // ORPO-fine-tuned "after" model). NOTE: mount() only STORES the adapter on each
-    // linear — it does NOT activate it. Activation is `loraState.active`; without
-    // setting it the forward runs BASE (bit-identical to no-adapter). So activate it
-    // here for direct-forward tasks; generateText re-passes `activeAdapters` too.
-    const { AdapterManager } = await import("../lora");
-    await new AdapterManager(model).mount("eval-adapter", adapterDir);
-    activeAdapters.push("eval-adapter");
-    model.loraState.active = [...activeAdapters];
-  }
-  const tokenizer = await loadTokenizer(dir);
-  let template: ChatTemplate | null = null;
-  try { template = await ChatTemplate.load(dir); } catch { template = null; }
-  return { model, tokenizer, template, config, dir, activeAdapters: activeAdapters.length ? activeAdapters : undefined };
-}
-
-export interface GenOpts {
-  maxTokens?: number;
-  /** Wrap `body` as a single user turn in the chat template (default true). */
-  useChat?: boolean;
-  /** Sampling overrides (default greedy). Lets the HLG/diversity evals vary the
-   *  sampler — temperature, top-p/k, seed, hlg — while sharing this prompt path. */
-  sampler?: Partial<Pick<GenerateOptions, "temperature" | "topP" | "topK" | "seed" | "hlg" | "curve">>;
-  /** Chat-template `enable_thinking`. Defaults to OFF for eval parity with optiq
-   *  (its capability numbers are non-thinking; e.g. MiniCPM5 IFEval 64.7 is
-   *  non-thinking). Set MLX_BUN_EVAL_THINK=1 or pass true to evaluate thinking mode. */
-  enableThinking?: boolean;
-  /** Explicit KV-quant scheme (cli `generate --l2/--kv-quant`, resolved with
-   *  serve's semantics). Overrides the MLX_BUN_EVAL_KV_QUANT env default; an empty
-   *  object forces bf16. When it names quantized KV, generation runs the product
-   *  generate() path (same tokens as `serve` with the same scheme). */
-  kvScheme?: Pick<GenerateOptions, "kvBits" | "kvConfig" | "quantizedKvStart" | "turboQuant">;
-}
-
-export { greedyDecodeBitExact } from "../backends/mlx/evaluation";
-import { createEvalCompletionClient } from "../backends/mlx/evaluation";
-
-/** Application entry point; selection and model math belong to its client. */
-export function generateText(tm: TaskModel, body: string, opts: GenOpts = {}): Promise<string> {
-  return (tm.completionClient ?? createEvalCompletionClient(tm)).complete({ body, options: opts });
-}

package/src/eval/score.ts

@@ -1,49 +0,0 @@
-// Capability score — port of optiq/eval/score.py.
-//
-// A single scalar: the simple UNWEIGHTED mean of whichever of the six
-// benchmarks (MMLU, GSM8K, IFEval, BFCL, HumanEval, HashHop) were run.
-// Disk size is reported alongside but NEVER enters the score — no hidden
-// quality/disk tradeoff baked into the number (optiq's explicit design
-// choice). Missing components are skipped, not zero-filled.
-
-export interface CapabilityScore {
-  /** 0–100, simple mean of the provided benchmark percents. */
-  score: number;
-  /** Per-benchmark contribution to the mean (only the ones that ran). */
-  components: Record<string, number>;
-  /** Reported separately; does NOT affect `score`. */
-  diskGb: number;
-}
-
-export type TaskName = "MMLU" | "GSM8K" | "IFEval" | "BFCL" | "HumanEval" | "HashHop";
-
-/** Unweighted mean of the provided task percents (each 0–100). */
-export function computeCapabilityScore(
-  percents: Partial<Record<TaskName, number>>,
-  diskGb: number,
-): CapabilityScore {
-  const components: Record<string, number> = {};
-  for (const [name, val] of Object.entries(percents))
-    if (val !== undefined && val !== null && Number.isFinite(val))
-      components[name] = val;
-
-  const keys = Object.keys(components);
-  const score = keys.length === 0
-    ? 0
-    : keys.reduce((a, k) => a + components[k]!, 0) / keys.length;
-
-  return { score, components, diskGb };
-}
-
-/** Sum of `*.safetensors` shard sizes for a local model dir, in GB. */
-export function diskGbForDir(modelDir: string): number {
-  try {
-    const { readdirSync, statSync } = require("node:fs") as typeof import("node:fs");
-    let total = 0;
-    for (const f of readdirSync(modelDir))
-      if (f.endsWith(".safetensors")) total += statSync(`${modelDir}/${f}`).size;
-    return total / 1024 ** 3;
-  } catch {
-    return 0;
-  }
-}