anyformer 0.0.1 → 0.3.1

package/src/backends/diskformer.js

@@ -0,0 +1,124 @@
+// diskformer: MoE models larger than GPU memory. Weights stay in OPFS; the GPU holds the dense layers, the KV cache
+// and an expert cache sized by the budget. Engines load from the CDN at load() time, at the commit src/sources.js pins;
+// set diskformerBackend.source to load another copy.
+
+import { AnyformerError } from '../errors.js';
+import { SOURCES } from '../sources.js';
+
+// The GGUFs diskformer's engines are pinned to (GEMMA4_26B_A4B, QWEN36_35B_A3B in engines/*.js), with the smallest budget each
+// has passed its gate at and the measured decode speeds (diskformer README, M4 Pro, 2026-10-05).
+export const DISKFORMER_MODELS = [
+  {
+    repo: 'google/gemma-4-26B-A4B-it-qat-q4_0-gguf', file: 'gemma-4-26B_q4_0-it.gguf', size: 14439363584,
+    engine: 'gemma4_moe_ssd.js', Engine: 'Gemma4MoeSsd', minBudget: 2.5e9,
+    speeds: [[2.5e9, 12.4], [4.3e9, 14.2], [7e9, 17.9]],
+  },
+  {
+    repo: 'unsloth/Qwen3.6-35B-A3B-GGUF', file: 'Qwen3.6-35B-A3B-Q8_0.gguf', size: 36903140320,
+    engine: 'qwen35_moe_ssd.js', Engine: 'Qwen35MoeSsd', minBudget: 4e9,
+    speeds: [[4e9, 9.2]],
+  },
+  // Qwen3.6's language model with a vision encoder (mmproj, ~1.1 GB, loaded on the first image by ensureVision).
+  // From LocalMind's gate (plan/2026-10-14-minicpm-harness, 4 GiB pool, context 512): ~6.1 GB on the GPU (dense 1.59,
+  // pool 4.29), 14.1 tok/s over short greedy continuations. The smallest working budget is unmeasured; the floor is
+  // Qwen3.6's, the same language-model shape. Only this engine reads its images, and it ranks before diskformer-llama.
+  {
+    repo: 'openbmb/MiniCPM-V-4.7-35B-A3B-gguf', file: 'MiniCPM-V-4_7-35B-A3B-Q4_0.gguf', size: 19714550688,
+    engine: 'qwen35_moe_ssd.js', Engine: 'MiniCpmV47Ssd', minBudget: 4e9,
+    speeds: [[6.1e9, 14.1]],
+  },
+  // Two parts, ingested as one file by the engine's splitFetch (engines/ling_moe_ssd.js LING_FLASH, rev 29edce5).
+  // Budget and speed from LocalMind's gate (plan/2026-10-22-ling-harness/results.md): 2.51 GB of dense weights on
+  // the GPU plus a 4.29 GB expert pool, 5.53 tok/s decode; the OPFS store is 80.7 GB.
+  {
+    repo: 'inclusionAI/Ling-3.0-flash-GGUF',
+    parts: [
+      { file: 'Q4_K_M/Ling-3.0-flash-Q4_K_M-00001-of-00002.gguf', size: 49995534208 },
+      { file: 'Q4_K_M/Ling-3.0-flash-Q4_K_M-00002-of-00002.gguf', size: 27014610464 },
+    ],
+    size: 49995534208 + 27014610464, storeBytes: 80.7e9,
+    engine: 'ling_moe_ssd.js', Engine: 'LingMoeSsd', minBudget: 6.8e9,
+    speeds: [[6.8e9, 5.53]],
+  },
+];
+
+// diskformer's engines refuse a budget with a plain Error ("GPU budget 2.50 GB is too small: … need 3.04 GB, and the
+// expert cache at least 0.05 GB more"). Until they carry a code, read the two numbers to give BUDGET_TOO_SMALL a need.
+function budgetError(e) {
+  const m = /GPU budget [\d.]+ GB is too small\b.*?need ([\d.]+) GB.*?at least ([\d.]+) GB more/s.exec(e?.message || '');
+  return m && new AnyformerError('BUDGET_TOO_SMALL', `diskformer: ${e.message}`, { cause: e, need: Math.ceil((+m[1] + +m[2]) * 1e9) });
+}
+
+const has = (model, file, size) => model.files.gguf.some((f) => f.name === file && f.size === size);
+const match = (model) => DISKFORMER_MODELS.find((m) => m.repo === model.repo &&
+  (m.parts ? m.parts.every((p) => has(model, p.file, p.size)) : has(model, m.file, m.size)));
+
+// The speed measured at the largest budget not above this one.
+const speedAt = (m, budget) => m.speeds.filter(([b]) => b <= budget).at(-1)?.[1] ?? null;
+
+export { budgetError };
+
+export const diskformerBackend = {
+  id: 'diskformer',
+  // Gemma 4's repo is tagged image-text-to-text; the engines run its text turns.
+  tasks: ['text-generation', 'image-text-to-text'],
+  source: SOURCES.diskformer,
+  // Where the engine downloads its GGUF from; null is Hugging Face. A test points it at a local server.
+  url: null,
+
+  canRun(model, caps, { budget, task }) {
+    if (!model.formats.includes('gguf')) return { ok: false, code: 'format', reason: 'no GGUF in the repo' };
+    if (task && !diskformerBackend.tasks.includes(task)) return { ok: false, code: 'task', reason: `task ${task}; diskformer engines run text generation only` };
+    const m = match(model);
+    if (!m) return { ok: false, code: 'model', reason: `no diskformer engine for this GGUF; engines cover ${DISKFORMER_MODELS.map((x) => x.repo).join(', ')}` };
+    if (!caps.webgpu) return { ok: false, code: 'webgpu', reason: 'no WebGPU adapter' };
+    if (!caps.f16) return { ok: false, code: 'f16', reason: 'the GPU lacks shader-f16' };
+    if (!caps.opfs) return { ok: false, code: 'opfs', reason: 'no OPFS (navigator.storage.getDirectory)' };
+    if (budget < m.minBudget) return { ok: false, code: 'budget', need: m.minBudget, reason: `a ${(budget / 1e9).toFixed(1)} GB budget is under the ${m.minBudget / 1e9} GB this model has passed its gate at` };
+    // Storage quota is not checked: Chrome raises it as the download writes, while the disk has room.
+    return { ok: true, estimate: { gpuBytes: budget, diskBytes: m.storeBytes ?? m.size, tokPerSec: speedAt(m, budget) } };
+  },
+
+  // maxCtx sizes the KV cache, which stays on the GPU: the 2.5 GB minimum and the speeds above hold at 1024 tokens,
+  // the context diskformer's chat gate runs at. A longer context needs a larger budget.
+  // poolBytes: the expert cache size, passed to the engine; the budget then only admits the model (the engine sizes
+  //   its pool from gpuBudgetBytes when given one, so it gets one or the other).
+  // localFile: the GGUF as a File the user picked; the engine ingests it instead of downloading, and refuses a file
+  //   whose size differs from the one it is pinned to.
+  async load(model, { budget, maxCtx = 1024, poolBytes, localFile, onProgress, signal }) {
+    const m = match(model);
+    const mod = await import(`${diskformerBackend.source}/${m.engine}`);
+    let engine;
+    try {
+      engine = await mod[m.Engine].load(null, {
+        // The engines' own default root, named so the store module (src/store.js) lists the same folder.
+        ...(poolBytes ? { poolBytes } : { gpuBudgetBytes: budget }), maxCtx, signal, root: 'localmind-ssd',
+        ...(localFile && { localFile }),
+        ...(diskformerBackend.url && (m.parts ? { urls: m.parts.map((p) => diskformerBackend.url(model, { ...m, file: p.file })) } : { url: diskformerBackend.url(model, m) })),
+        onProgress: onProgress && ((e) => onProgress({ backend: 'diskformer', ...e })),
+      });
+    } catch (e) { throw budgetError(e) || e; }
+    // Where the model's reasoning starts and ends, as token ids, so a caller can split it out without parsing text.
+    const open = engine.thinkOpenTokenId ?? null;
+    return {
+      task: 'text-generation',
+      engine,
+      think: open === null ? null : {
+        openId: open, closeId: engine.thinkCloseTokenId ?? null, inPrompt: engine.thinkInPrompt ?? null,
+        preamble: engine.thinkPreamble ?? null, always: engine.alwaysThinks ?? null,
+      },
+      // thinking: false (the default, as on every back end here) answers directly; true lets a model that reasons
+      // (Gemma 4, Qwen3.6, Ling) think first. Image blocks ({ type: 'image', rgb, width, height }) pass through to
+      // an engine that reads them.
+      async *generate(messages, { maxNewTokens = 512, signal, thinking = false } = {}) {
+        for await (const r of engine.generate(messages, { maxNewTokens, signal, enableThinking: thinking === true }))
+          yield { text: r.text, token: r.token, tokens: r.tokens };
+      },
+      // Checkpoints a system prompt after load, so the first question does not wait for it (Qwen3.6, MiniCPM-V).
+      ...(typeof engine.prime === 'function' && { prime: (system) => engine.prime(system) }),
+      // Loads the vision tower, for an engine that reads images (MiniCPM-V).
+      ...(typeof engine.ensureVision === 'function' && { ensureVision: (o) => engine.ensureVision(o) }),
+      dispose: () => engine.dispose(),
+    };
+  },
+};

package/src/backends/kohra.js

@@ -0,0 +1,59 @@
+// kohra: text-diffusion LMs (MDLM, BD3LM). kohra.js runs the denoising loop over onnxruntime-web; it loads from the
+// CDN at load() time, at the commit src/sources.js pins; set kohraBackend.source to load another copy.
+
+import { graphBytes } from './transformers.js';
+import { SOURCES } from '../sources.js';
+
+const FP16 = 'onnx/model_fp16_fused.onnx';
+// The q4 graph decodes on WebGPU only with RTN packing and this ORT-web dev build (kohra README, "Known constraints").
+const Q4 = 'onnx/model_q4f16_rtn_sym.onnx';
+const Q4_ORT = '1.26.0-dev.20260416-b7804b056c';
+
+const pickGraph = (model, caps) => {
+  const have = new Set(model.files.onnx.map((f) => f.name));
+  return caps.f16 && have.has(FP16) ? FP16 : have.has(Q4) ? Q4 : null;
+};
+
+// kohra's own template covers one user turn; this is the same ChatML over the whole conversation.
+const chatml = (messages) =>
+  messages.map((m) => `<|im_start|>${m.role}\n${m.content}<|im_end|>\n`).join('') + '<|im_start|>assistant\n';
+
+export const kohraBackend = {
+  id: 'kohra',
+  tasks: ['text-generation'],
+  source: SOURCES.kohra,
+
+  canRun(model, caps, { task } = {}) {
+    if (!model.tags?.includes('text-diffusion')) return { ok: false, code: 'model', reason: 'not a text-diffusion model' };
+    if (task && task !== 'text-generation') return { ok: false, code: 'task', reason: `task ${task}; kohra runs text-generation only` };
+    if (!caps.webgpu) return { ok: false, code: 'webgpu', reason: 'no WebGPU adapter' };
+    const graph = pickGraph(model, caps);
+    if (!graph) return { ok: false, code: 'format', reason: `no kohra graph (${FP16} or ${Q4}) in the repo` };
+    const bytes = graphBytes(model, graph);
+    return { ok: true, estimate: { gpuBytes: bytes, diskBytes: bytes, tokPerSec: null } };
+  },
+
+  async load(model, { caps, onProgress, signal }) {
+    const graph = pickGraph(model, caps);
+    const { DiffusionLM } = await import(kohraBackend.source);
+    onProgress?.({ backend: 'kohra', status: 'loading', file: graph });
+    const base = `https://huggingface.co/${model.repo}/resolve/${model.revision || 'main'}`;
+    const lm = await DiffusionLM.from_pretrained({
+      model: `${base}/${graph}`, tokenizer: model.repo, ...(graph === Q4 && { ortVersion: Q4_ORT }),
+    });
+    signal?.throwIfAborted();
+    const blockCausal = model.tags.includes('block-diffusion');
+    return {
+      task: 'text-generation',
+      // Diffusion fills the whole block at once, so the reply arrives as one chunk.
+      async *generate(messages, { maxNewTokens = 128, steps, signal } = {}) {
+        signal?.throwIfAborted();
+        const out = await lm.generate(chatml(messages), {
+          maxNewTokens, steps: steps ?? maxNewTokens, blockCausal, chatTemplate: false, stripThink: true,
+        });
+        yield { text: out.text, tokens: out.tokens, tokPerSec: out.tokensPerSecond };
+      },
+      dispose: () => lm.session?.release?.(),
+    };
+  },
+};

package/src/backends/localmind.js

@@ -0,0 +1,116 @@
+// LocalMind's hand-written WebGPU engines: whole model resident on the GPU, kernels written per architecture. Each
+// engine reads its config from the model file, so it runs any checkpoint of its architecture in a format it reads,
+// not only the one LocalMind ships. They live in diskformer (engines/, since v0.6.4) and load from the CDN at load()
+// time, at the commit src/sources.js pins; set localmindBackend.source to load another copy. The id stays 'localmind',
+// the name callers and stored() already use, though no code loads from LocalMind's repo any more.
+//
+//   lfm2     src/lfm2_5.js           GGUF, general.architecture 'lfm2' (dense LFM2 / LFM2.5), Q4_0 Q8_0 Q4_K Q5_K Q6_K
+//   gemma4   src/gemma-4-e2b.js      Google's QAT-mobile safetensors export of Gemma 4 (E2B, E4B): config.json-driven
+//   qwen35   src/ternary_bonsai_2_27b.js  GGUF, 'qwen35' with Prism's ternary types (PTQ1_0, PQ2_0) only: a standard
+//            K-quant (unsloth/Qwen3.5-4B-GGUF Q4_K_M) fails with "This feature is not included in this bundle".
+// Speeds: LocalMind's quiet-machine engine bench (plan/bench-engines-quiet-2026-09-24.md, M4 Pro, 256 tokens, greedy).
+
+import { SOURCES } from '../sources.js';
+
+const ENGINES = {
+  lfm2: { file: 'lfm2_5.js', Engine: 'Lfm2Mobile' },
+  gemma4: { file: 'gemma-4-e2b.js', Engine: 'Gemma4Mobile' },
+  qwen35: { file: 'ternary_bonsai_2_27b.js', Engine: 'TernaryBonsai2' },
+};
+
+// Quant preference per engine, by the GGUF file-name suffix. Plain Q4_0 first: it is LocalMind's default.
+const LFM2_QUANTS = ['Q4_0', 'Q4_K_M', 'Q4_K_S', 'Q5_K_M', 'Q5_K_S', 'Q6_K', 'Q8_0'];
+const QWEN35_QUANTS = ['PTQ1_0', 'PQ2_0'];
+const MEASURED = {
+  'LiquidAI/LFM2.5-230M-GGUF': 1062.5,
+  'google/gemma-4-E2B-it-qat-mobile-transformers': 171.3,
+  'prism-ml/Ternary-Bonsai-2-27B-gguf': 27.2,
+};
+
+// The GGUF to load: the first quant in preference order. Projector (mmproj) files are vision towers, never the LM.
+function pickGguf(model, quants) {
+  const lms = model.files.gguf.filter((f) => !/mmproj/i.test(f.name));
+  for (const q of quants) {
+    const f = lms.find((x) => new RegExp(`[-_.]${q}\\.gguf$`, 'i').test(x.name) && !/QAD/i.test(x.name));
+    if (f) return f;
+  }
+  return null;
+}
+
+// Which engine reads this repo, and the file and bytes it would load; null when none does.
+function match(model) {
+  if (model.formats.includes('gguf') && model.architecture === 'lfm2') {
+    const f = pickGguf(model, LFM2_QUANTS);
+    return f && { engine: 'lfm2', file: f.name, bytes: f.size };
+  }
+  if (model.formats.includes('gguf') && model.architecture === 'qwen35') {
+    const f = pickGguf(model, QWEN35_QUANTS);
+    return f && { engine: 'qwen35', file: f.name, bytes: f.size };
+  }
+  // The QAT-mobile export's tensor names (weight_scale, embedding_quantized …) are what the engine reads; a plain
+  // Gemma 4 safetensors checkpoint has model_type gemma4 too but will not load, so the repo name decides.
+  if (model.architecture === 'gemma4' && /-qat-mobile-transformers$/.test(model.repo)) {
+    const f = model.files.safetensors.find((x) => x.name === 'model.safetensors');
+    return f && { engine: 'gemma4', file: f.name, bytes: f.size };
+  }
+  return null;
+}
+
+export const localmindBackend = {
+  id: 'localmind',
+  tasks: ['text-generation'],
+  source: SOURCES.diskformer,
+
+  canRun(model, caps, { budget, task }) {
+    const m = match(model);
+    if (!m) return { ok: false, code: 'model', reason: 'no LocalMind engine reads this repo (lfm2 GGUF, qwen35 Prism GGUF, Gemma 4 QAT-mobile)' };
+    if (task && task !== 'text-generation') return { ok: false, code: 'task', reason: `task ${task}; LocalMind's engines run text generation` };
+    if (!caps.webgpu) return { ok: false, code: 'webgpu', reason: 'no WebGPU adapter' };
+    if (m.engine === 'qwen35' && !(caps.f16 && caps.subgroups)) return { ok: false, code: 'f16', reason: 'the ternary kernels need shader-f16 and subgroups' };
+    // The whole model stays on the GPU, so the budget must hold the file.
+    if (budget < m.bytes) return { ok: false, code: 'budget', need: m.bytes, reason: `${m.file} stays on the GPU whole (${(m.bytes / 1e9).toFixed(2)} GB) and the budget is ${(budget / 1e9).toFixed(2)} GB` };
+    return { ok: true, estimate: { gpuBytes: m.bytes, diskBytes: m.bytes, tokPerSec: MEASURED[model.repo] ?? null } };
+  },
+
+  async load(model, { maxCtx, onProgress, signal }) {
+    const m = match(model);
+    const e = ENGINES[m.engine];
+    const mod = await import(`${localmindBackend.source}/${e.file}`);
+    const opts = {
+      revision: model.revision || 'main', signal,
+      onProgress: onProgress && ((p) => onProgress({ backend: 'localmind', engine: m.engine, ...p })),
+      ...(m.engine !== 'gemma4' && { file: m.file }),
+      ...(maxCtx && m.engine !== 'gemma4' && { maxLength: maxCtx }),
+    };
+    const engine = await mod[e.Engine].load(model.repo, opts);
+    signal?.throwIfAborted();
+    // Thinking off, as the diskformer back end does: a chat reply, not the model's reasoning. (The qwen35 engine
+    // takes template arguments as a field; LocalMind sets the same one.)
+    if ('chatTemplateArgs' in engine) engine.chatTemplateArgs = { ...engine.chatTemplateArgs, enable_thinking: false };
+    // Compile the kernels now, as LocalMind's workers do, so the first reply is not paying for it.
+    onProgress?.({ backend: 'localmind', engine: m.engine, status: 'warmup' });
+    await engine.warmup?.();
+    // Gemma's template has no system turn; LocalMind folds the system prompt into the first user turn the same way.
+    const prepare = m.engine !== 'gemma4' ? (msgs) => msgs : (msgs) => {
+      const sys = msgs.filter((x) => x.role === 'system').map((x) => x.content).join('\n\n');
+      const rest = msgs.filter((x) => x.role !== 'system');
+      const i = rest.findIndex((x) => x.role === 'user');
+      return sys && i >= 0 ? rest.map((x, k) => (k === i ? { ...x, content: `${sys}\n\n${x.content}` } : x)) : rest;
+    };
+    let tokens = 0;
+    return {
+      task: 'text-generation',
+      engine,
+      async *generate(messages, { maxNewTokens = 512, signal } = {}) {
+        tokens = 0;
+        for await (const c of engine.generate(prepare(messages), { maxNewTokens, signal })) {
+          if (c.token != null) tokens++;
+          yield { text: c.text, tokens };
+        }
+      },
+      dispose: () => engine.dispose?.(),
+    };
+  },
+};
+
+export { match as localmindMatch };

package/src/backends/transformers.js

@@ -0,0 +1,191 @@
+// transformers.js: ONNX models on WebGPU, text generation plus the other pipeline tasks below. The library loads from the CDN at load() time, so
+// importing anyformer costs nothing until this back end is chosen.
+
+
+import { SOURCES } from '../sources.js';
+
+// dtype → the ONNX file transformers.js reads for it (decoder-only repos keep one graph per dtype under onnx/).
+const FILES = {
+  q4: 'onnx/model_q4.onnx', q2: 'onnx/model_q2.onnx', fp32: 'onnx/model.onnx',
+  q4f16: 'onnx/model_q4f16.onnx', q2f16: 'onnx/model_q2f16.onnx', fp16: 'onnx/model_fp16.onnx',
+};
+
+// f16 activations come last: on Apple Metal (Chrome 154, transformers.js 4.3.1) Qwen2.5-0.5B's q4f16 graph decodes
+// to word salad and fp16 loops, while q4 (f32 activations) answers correctly (test/browser/transformers.html,
+// 2026-10-08). kohra hit the same fp16 overflow on WebGPU (kohra README, "Known constraints").
+function pickDtype(model, caps) {
+  const have = new Set(model.files.onnx.map((f) => f.name));
+  const order = caps.f16 ? ['q4', 'q2', 'fp32', 'q4f16', 'q2f16', 'fp16'] : ['q4', 'q2', 'fp32'];
+  return order.find((d) => have.has(FILES[d])) || null;
+}
+
+// Multimodal chat models: one ONNX graph per module (embed_tokens, decoder_model_merged, vision_encoder, audio_encoder)
+// per precision, loaded through the model's conditional-generation class and its processor, as LocalMind does
+// (LocalMind index.html loadMultimodal). Keyed by config.model_type. q4f16 first: LocalMind ships these at q4f16.
+const MULTIMODAL = { gemma4: 'Gemma4ForConditionalGeneration', qwen3_5: 'Qwen3_5ForConditionalGeneration' };
+const MM_SUFFIX = { fp32: '', fp16: '_fp16', q4: '_q4', q4f16: '_q4f16', q8: '_quantized' };
+const mmModules = (model) => [...new Set(model.files.onnx.map((f) => /^onnx\/([a-z_]+?)(?:_fp16|_q4f16|_q4|_quantized)?\.onnx$/.exec(f.name)?.[1]).filter(Boolean))];
+function pickMmDtype(model, caps) {
+  const have = new Set(model.files.onnx.map((f) => f.name));
+  const order = caps.f16 ? ['q4f16', 'q4', 'fp16', 'fp32'] : ['q4', 'fp32'];
+  return order.find((d) => have.has(`onnx/decoder_model_merged${MM_SUFFIX[d]}.onnx`)) || null;
+}
+const mmBytes = (model, dtype) => mmModules(model).reduce((n, m) => n + graphBytes(model, `onnx/${m}${MM_SUFFIX[dtype]}.onnx`), 0);
+const isMultimodal = (model) => model.architecture in MULTIMODAL && model.files.onnx.some((f) => f.name.startsWith('onnx/decoder_model_merged'));
+// The chat tasks a multimodal repo is tagged with on the Hub; all run here as chat.
+const MM_TASKS = ['text-generation', 'image-text-to-text', 'any-to-any'];
+
+// The graph plus its external-data files (<graph>.data, <graph>_data, <graph>_data_1 …).
+export const graphBytes = (model, file) =>
+  model.files.all.filter((f) => f.name === file || f.name.startsWith(`${file}.data`) || f.name.startsWith(`${file}_data`))
+    .reduce((n, f) => n + (f.size || 0), 0);
+
+// The non-generation tasks this back end runs, each through transformers.js's own pipeline(). They load the fp32
+// graphs: the f16 paths gave wrong output on Metal for text generation, and these models are small.
+const PIPELINES = {
+  'feature-extraction': { options: { pooling: 'mean', normalize: true }, out: (t) => t.tolist() },
+  'sentence-similarity': { task: 'feature-extraction', options: { pooling: 'mean', normalize: true }, out: (t) => t.tolist() },
+  'automatic-speech-recognition': {},
+  'object-detection': { image: true },
+  'image-classification': { image: true },
+  'depth-estimation': { image: true },
+  'image-segmentation': { image: true },
+  'token-classification': {},
+  'text-classification': {},
+};
+
+// fp32 graphs: onnx/*.onnx without a precision suffix (model.onnx, encoder_model.onnx, decoder_model_merged.onnx).
+const FP32 = /^onnx\/[a-z_]+(?<!_fp16|_q4|_q4f16|_quantized|_int8|_uint8|_bnb4|_q8)\.onnx$/;
+const fp32Bytes = (model) => model.files.onnx.filter((f) => FP32.test(f.name)).reduce((n, f) => n + graphBytes(model, f.name), 0);
+
+export const transformersBackend = {
+  id: 'transformers',
+  // The library, at the version src/sources.js pins. Set transformersBackend.source to load another version.
+  source: SOURCES.transformers,
+  tasks: ['text-generation', ...Object.keys(PIPELINES)],
+
+  canRun(model, caps, { task, budget = Infinity } = {}) {
+    const fits = (bytes, ok) => (bytes > budget
+      ? { ok: false, code: 'budget', need: bytes, reason: `the model stays on the GPU whole (${(bytes / 1e9).toFixed(2)} GB) and the budget is ${(budget / 1e9).toFixed(2)} GB` }
+      : ok);
+    if (!model.formats.includes('onnx')) return { ok: false, code: 'format', reason: 'no ONNX graph in the repo' };
+    // kohra's diffusion graphs are ONNX too, but transformers.js generate() is autoregressive only.
+    if (model.tags?.includes('text-diffusion')) return { ok: false, code: 'model', reason: 'a text-diffusion model; transformers.js generate() is autoregressive only' };
+    // A repo with no Hub task tag (e.g. onnx-community/Ternary-Bonsai-4B-ONNX) is taken as text generation.
+    if (isMultimodal(model)) {
+      if (task && !MM_TASKS.includes(task)) return { ok: false, code: 'task', reason: `task ${task}; ${model.architecture} runs here as chat` };
+      if (!caps.webgpu) return { ok: false, code: 'webgpu', reason: 'no WebGPU adapter' };
+      const dtype = pickMmDtype(model, caps);
+      if (!dtype) return { ok: false, code: 'format', reason: 'no decoder_model_merged graph in a precision this back end loads' };
+      const bytes = mmBytes(model, dtype);
+      return fits(bytes, { ok: true, estimate: { gpuBytes: bytes, diskBytes: bytes, tokPerSec: null } });
+    }
+    const want = task || model.task || 'text-generation';
+    if (!transformersBackend.tasks.includes(want)) return { ok: false, code: 'task', reason: `task ${want || 'unknown'}; this back end runs ${transformersBackend.tasks.join(', ')}` };
+    if (!caps.webgpu) return { ok: false, code: 'webgpu', reason: 'no WebGPU adapter' };
+    if (want !== 'text-generation') {
+      const bytes = fp32Bytes(model);
+      if (!bytes) return { ok: false, code: 'format', reason: 'no fp32 ONNX graph (onnx/model.onnx or encoder/decoder graphs)' };
+      return fits(bytes, { ok: true, estimate: { gpuBytes: bytes, diskBytes: bytes, tokPerSec: null } });
+    }
+    const dtype = pickDtype(model, caps);
+    if (!dtype) return { ok: false, code: 'format', reason: `no ONNX graph at ${Object.values(FILES).join(', ')}` };
+    const bytes = graphBytes(model, FILES[dtype]);
+    return fits(bytes, { ok: true, estimate: { gpuBytes: bytes, diskBytes: bytes, tokPerSec: null } });
+  },
+
+  // dtype overrides the precision pick (the repo must ship it). generate() takes repetitionPenalty; without it the
+  // model's own generation_config.json applies (Qwen2.5 ships 1.05), as in transformers.js itself.
+  async load(model, { task, caps, dtype: forced, onProgress, signal }) {
+    const want = task || model.task || 'text-generation';
+    const lib = await import(transformersBackend.source);
+    const progress_callback = onProgress && ((p) => onProgress({ backend: 'transformers', ...p }));
+    if (isMultimodal(model)) return loadMultimodal(lib, model, forced || pickMmDtype(model, caps), { progress_callback, signal });
+    if (want !== 'text-generation') return loadPipeline(lib, model, want, { progress_callback, signal });
+    const dtype = forced || pickDtype(model, caps);
+    const gen = await lib.pipeline('text-generation', model.repo, {
+      device: 'webgpu', dtype, revision: model.revision || 'main', progress_callback,
+    });
+    signal?.throwIfAborted();
+    return {
+      task: 'text-generation',
+      generate: (messages, { maxNewTokens = 256, repetitionPenalty, signal } = {}) => stream(lib, gen.tokenizer, signal,
+        (streamer, stopping_criteria) => gen(messages, { max_new_tokens: maxNewTokens, do_sample: false, ...(repetitionPenalty && { repetition_penalty: repetitionPenalty }), streamer, stopping_criteria })),
+      dispose: () => gen.dispose?.(),
+    };
+  },
+};
+
+// A non-generation task: one pipeline, run(input, opts) calls it. ImageData (from a canvas) becomes a RawImage; URLs,
+// RawImage and, for speech, 16 kHz Float32Array samples pass through as transformers.js takes them.
+async function loadPipeline(lib, model, task, { progress_callback, signal }) {
+  const spec = PIPELINES[task];
+  const pipe = await lib.pipeline(spec.task || task, model.repo, {
+    device: 'webgpu', dtype: 'fp32', revision: model.revision || 'main', progress_callback,
+  });
+  signal?.throwIfAborted();
+  const image = (x) => (typeof ImageData !== 'undefined' && x instanceof ImageData ? new lib.RawImage(x.data, x.width, x.height, 4) : x);
+  return {
+    task,
+    async run(input, opts = {}) {
+      const out = await pipe(spec.image ? image(input) : input, { ...spec.options, ...opts });
+      return spec.out ? spec.out(out) : out;
+    },
+    dispose: () => pipe.dispose?.(),
+  };
+}
+
+// transformers.js streams through a callback; this turns one generate call into the contract's async iterable of
+// { text, tokens }, with the signal wired to an InterruptableStoppingCriteria.
+async function* stream(lib, tokenizer, signal, run) {
+  const queue = [];
+  let wake = () => {}, done = false, failed = null, text = '', tokens = 0;
+  const streamer = new lib.TextStreamer(tokenizer, {
+    skip_prompt: true, skip_special_tokens: true,
+    callback_function: (t) => { text += t; queue.push({ text, tokens }); wake(); },
+    token_callback_function: () => { tokens++; },
+  });
+  const stop = new lib.InterruptableStoppingCriteria();
+  signal?.addEventListener('abort', () => stop.interrupt(), { once: true });
+  run(streamer, stop).catch((e) => { failed = e; }).finally(() => { done = true; wake(); });
+  while (true) {
+    if (queue.length) { yield queue.shift(); continue; }
+    if (done) break;
+    await new Promise((r) => { wake = r; });
+  }
+  if (failed) throw failed;
+}
+
+// A multimodal chat model. Message content is a string or an array of blocks: { type: 'text', text },
+// { type: 'image', image } (ImageData, RawImage or a URL) and, for Gemma 4, { type: 'audio', audio } (16 kHz
+// Float32Array). The template sees placeholders; the processor gets the media, in order.
+async function loadMultimodal(lib, model, dtype, { progress_callback, signal }) {
+  const opts = { revision: model.revision || 'main', progress_callback };
+  const processor = await lib.AutoProcessor.from_pretrained(model.repo, opts);
+  const net = await lib[MULTIMODAL[model.architecture]].from_pretrained(model.repo, { ...opts, dtype, device: 'webgpu' });
+  signal?.throwIfAborted();
+  // One-token warmup compiles the kernels before the first reply (LocalMind does the same).
+  await net.generate({ ...(await processor('a', null, null, { add_special_tokens: false })), max_new_tokens: 1 });
+  const toImage = async (x) => (typeof ImageData !== 'undefined' && x instanceof ImageData ? new lib.RawImage(x.data, x.width, x.height, 4)
+    : typeof x === 'string' ? lib.load_image(x) : x);
+  return {
+    task: 'text-generation',
+    generate: (messages, { maxNewTokens = 256, repetitionPenalty, signal } = {}) => stream(lib, processor.tokenizer, signal, async (streamer, stopping_criteria) => {
+      const images = [], audios = [];
+      const plain = messages.map((m) => (typeof m.content === 'string' ? m : {
+        ...m,
+        content: m.content.map((b) => {
+          if (b.type === 'image') { images.push(b.image); return { type: 'image' }; }
+          if (b.type === 'audio') { audios.push(b.audio); return { type: 'audio' }; }
+          return b;
+        }),
+      }));
+      const prompt = processor.apply_chat_template(plain, { add_generation_prompt: true, enable_thinking: false });
+      const imgs = await Promise.all(images.map(toImage));
+      const one = (a) => (a.length ? (a.length === 1 ? a[0] : a) : null);
+      const inputs = await processor(prompt, one(imgs), one(audios), { add_special_tokens: false });
+      return net.generate({ ...inputs, max_new_tokens: maxNewTokens, do_sample: false, ...(repetitionPenalty && { repetition_penalty: repetitionPenalty }), streamer, stopping_criteria });
+    }),
+    dispose: () => net.dispose?.(),
+  };
+}

package/src/backends/wllama.js

@@ -0,0 +1,122 @@
+// wllama: llama.cpp compiled to WebAssembly, for GGUF models no hand-written engine covers (any architecture its
+// llama.cpp knows, e.g. openbmb/MiniCPM5-2B-GGUF). It offloads every layer to WebGPU when the browser has it and runs
+// on the CPU otherwise, so it is anyformer's one path that needs no WebGPU. Listed last in BACKENDS: an engine written
+// for the architecture (localmind, diskformer) wins when one exists.
+
+import { SOURCES } from '../sources.js';
+import { splitToOpfs } from '../gguf-split.js';
+
+// wllama reads files of at most 2 GB (one ArrayBuffer each) into a 32-bit WebAssembly memory of 4 GB. A model over
+// 2 GB loads as shards: the repo's own split set when it has one, else shards anyformer writes to OPFS
+// (src/gguf-split.js). The whole model, shards included, must stay under TOTAL_LIMIT to leave room for the context.
+const FILE_LIMIT = 2 ** 31;
+const TOTAL_LIMIT = 3.6e9;
+const QUANTS = ['Q4_K_M', 'Q4_0', 'Q4_K_S', 'Q5_K_M', 'Q5_K_S', 'Q6_K', 'Q8_0'];
+const SPLIT = /-(\d{5})-of-(\d{5})\.gguf$/i;
+
+// What to load: { kind: 'file' | 'split' | 'local-split', name, files: [{ name, size }], bytes }, or null.
+function pickFile(model) {
+  const lms = model.files.gguf.filter((f) => !/mmproj/i.test(f.name));
+  const sets = new Map();
+  for (const f of lms.filter((x) => SPLIT.test(x.name))) {
+    const base = f.name.replace(SPLIT, '');
+    sets.set(base, [...(sets.get(base) || []), f]);
+  }
+  for (const q of QUANTS) {
+    const quant = new RegExp(`[-_.]${q}$`, 'i');
+    const single = lms.find((x) => !SPLIT.test(x.name) && quant.test(x.name.replace(/\.gguf$/i, '')));
+    if (single) return { kind: single.size > FILE_LIMIT ? 'local-split' : 'file', name: single.name, files: [single], bytes: single.size };
+    for (const [base, parts] of sets) {
+      const total = Number(SPLIT.exec(parts[0].name)[2]);
+      if (quant.test(base) && parts.length === total) {
+        const files = [...parts].sort((a, b) => a.name.localeCompare(b.name));
+        return { kind: 'split', name: files[0].name, files, bytes: files.reduce((n, f) => n + f.size, 0) };
+      }
+    }
+  }
+  return null;
+}
+
+// wllama 3.4.1's stream ends one result early: the last chunk stays queued in the wasm side and would open the next
+// request. Read the queue dry before and after every reply, as LocalMind does (its WLLAMA-DRAIN, index.html).
+async function drain(w) {
+  const proxy = w?.proxy;
+  if (!proxy?.wllamaAction) return [];
+  const out = [];
+  for (let i = 0; i < 64; i++) {
+    const r = await proxy.wllamaAction('get_result', { _name: 'gres_req' });
+    const j = r?.data_json;
+    if (!j || j === 'null') { if (!r?.has_more) break; continue; }
+    let arr; try { arr = JSON.parse(j); } catch { break; }
+    out.push(...(Array.isArray(arr) ? arr : [arr]));
+  }
+  return out;
+}
+const delta = (chunk) => chunk?.choices?.[0]?.delta?.content || '';
+
+export const wllamaBackend = {
+  id: 'wllama',
+  tasks: ['text-generation'],
+  source: SOURCES.wllama,
+
+  canRun(model, caps, { budget, task } = {}) {
+    if (!model.formats.includes('gguf')) return { ok: false, code: 'format', reason: 'no GGUF in the repo' };
+    if (task && task !== 'text-generation' && task !== 'image-text-to-text') return { ok: false, code: 'task', reason: `task ${task}; wllama runs text generation` };
+    const f = pickFile(model);
+    if (!f) return { ok: false, code: 'model', reason: `no GGUF at ${QUANTS.join(', ')}` };
+    if (f.bytes > TOTAL_LIMIT) return { ok: false, code: 'model', reason: `${f.name} is ${(f.bytes / 1e9).toFixed(2)} GB; wllama's 32-bit WebAssembly memory holds about ${TOTAL_LIMIT / 1e9} GB of model` };
+    if (f.kind === 'local-split' && !caps.opfs) return { ok: false, code: 'opfs', reason: `${f.name} is over 2 GB and is split into OPFS before loading` };
+    // On WebGPU the whole model sits on the GPU, so it must fit the budget; without WebGPU it runs on the CPU.
+    const gpu = caps.webgpu && f.bytes <= budget;
+    return { ok: true, estimate: { gpuBytes: gpu ? f.bytes : 0, diskBytes: f.bytes, tokPerSec: null } };
+  },
+
+  async load(model, { caps, budget, maxCtx = 4096, onProgress, signal }) {
+    const f = pickFile(model);
+    const { Wllama } = await import(`${wllamaBackend.source}/index.js`);
+    const w = new Wllama({ default: `${wllamaBackend.source}/wasm/wllama.wasm` }, { logger: { debug() {}, log() {}, warn() {}, error: (m) => console.error('[wllama]', m) } });
+    // isSupportWebGPU() may answer directly or with a promise, depending on the wllama version.
+    let gpu = false;
+    if (caps.webgpu && f.bytes <= budget) { try { gpu = !!(await w.isSupportWebGPU()); } catch { gpu = false; } }
+    const rev = model.revision || 'main';
+    const url = (name) => `https://huggingface.co/${model.repo}/resolve/${rev}/${name}`;
+    const cfg = {
+      n_ctx: maxCtx, n_gpu_layers: gpu ? 999 : 0,
+      progressCallback: onProgress && (({ loaded, total }) => onProgress({ backend: 'wllama', status: 'download', file: f.name, loaded, total })),
+    };
+    if (f.kind === 'local-split') {
+      const shards = await splitToOpfs({
+        url: url(f.name), size: f.bytes, repo: model.repo, file: f.name, signal,
+        key: `${model.repo}--${rev}--${f.name}`.replace(/[^A-Za-z0-9._-]+/g, '-'),
+        onProgress: onProgress && ((p) => onProgress({ backend: 'wllama', ...p })),
+      });
+      await w.loadModel(shards, cfg);
+    } else {
+      await w.loadModelFromUrl(f.kind === 'split' ? f.files.map((x) => url(x.name)) : url(f.name), cfg);
+    }
+    signal?.throwIfAborted();
+    return {
+      task: 'text-generation',
+      device: gpu ? 'webgpu' : 'cpu',
+      async *generate(messages, { maxNewTokens = 256, signal } = {}) {
+        const msgs = messages.map((m) => ({ role: m.role, content: typeof m.content === 'string' ? m.content : m.content.filter((b) => b.type === 'text').map((b) => b.text).join('\n') }));
+        await drain(w).catch(() => {});
+        const ctl = new AbortController();
+        signal?.addEventListener('abort', () => ctl.abort(), { once: true });
+        let text = '', tokens = 0;
+        try {
+          // Thinking off, as on every other back end: MiniCPM5 otherwise spends the reply in reasoning_content.
+          const stream = await w.createChatCompletion({
+            messages: msgs, stream: true, max_tokens: maxNewTokens, temperature: 0, abortSignal: ctl.signal,
+            chat_template_kwargs: { enable_thinking: false },
+          });
+          for await (const c of stream) { const d = delta(c); if (d) { text += d; tokens++; yield { text, tokens }; } }
+          for (const c of await drain(w)) { const d = delta(c); if (d) { text += d; tokens++; yield { text, tokens }; } }
+        } catch (e) {
+          if (!ctl.signal.aborted) throw e;
+        }
+      },
+      dispose: () => w.exit(),
+    };
+  },
+};