anyformer 0.0.0-stage → 0.1.0

package/src/backends/transformers.js

@@ -0,0 +1,191 @@
+// transformers.js: ONNX models on WebGPU, text generation plus the other pipeline tasks below. The library loads from the CDN at load() time, so
+// importing anyformer costs nothing until this back end is chosen.
+
+
+import { SOURCES } from '../sources.js';
+
+// dtype → the ONNX file transformers.js reads for it (decoder-only repos keep one graph per dtype under onnx/).
+const FILES = {
+  q4: 'onnx/model_q4.onnx', q2: 'onnx/model_q2.onnx', fp32: 'onnx/model.onnx',
+  q4f16: 'onnx/model_q4f16.onnx', q2f16: 'onnx/model_q2f16.onnx', fp16: 'onnx/model_fp16.onnx',
+};
+
+// f16 activations come last: on Apple Metal (Chrome 154, transformers.js 4.3.1) Qwen2.5-0.5B's q4f16 graph decodes
+// to word salad and fp16 loops, while q4 (f32 activations) answers correctly (test/browser/transformers.html,
+// 2026-10-08). kohra hit the same fp16 overflow on WebGPU (kohra README, "Known constraints").
+function pickDtype(model, caps) {
+  const have = new Set(model.files.onnx.map((f) => f.name));
+  const order = caps.f16 ? ['q4', 'q2', 'fp32', 'q4f16', 'q2f16', 'fp16'] : ['q4', 'q2', 'fp32'];
+  return order.find((d) => have.has(FILES[d])) || null;
+}
+
+// Multimodal chat models: one ONNX graph per module (embed_tokens, decoder_model_merged, vision_encoder, audio_encoder)
+// per precision, loaded through the model's conditional-generation class and its processor, as LocalMind does
+// (LocalMind index.html loadMultimodal). Keyed by config.model_type. q4f16 first: LocalMind ships these at q4f16.
+const MULTIMODAL = { gemma4: 'Gemma4ForConditionalGeneration', qwen3_5: 'Qwen3_5ForConditionalGeneration' };
+const MM_SUFFIX = { fp32: '', fp16: '_fp16', q4: '_q4', q4f16: '_q4f16', q8: '_quantized' };
+const mmModules = (model) => [...new Set(model.files.onnx.map((f) => /^onnx\/([a-z_]+?)(?:_fp16|_q4f16|_q4|_quantized)?\.onnx$/.exec(f.name)?.[1]).filter(Boolean))];
+function pickMmDtype(model, caps) {
+  const have = new Set(model.files.onnx.map((f) => f.name));
+  const order = caps.f16 ? ['q4f16', 'q4', 'fp16', 'fp32'] : ['q4', 'fp32'];
+  return order.find((d) => have.has(`onnx/decoder_model_merged${MM_SUFFIX[d]}.onnx`)) || null;
+}
+const mmBytes = (model, dtype) => mmModules(model).reduce((n, m) => n + graphBytes(model, `onnx/${m}${MM_SUFFIX[dtype]}.onnx`), 0);
+const isMultimodal = (model) => model.architecture in MULTIMODAL && model.files.onnx.some((f) => f.name.startsWith('onnx/decoder_model_merged'));
+// The chat tasks a multimodal repo is tagged with on the Hub; all run here as chat.
+const MM_TASKS = ['text-generation', 'image-text-to-text', 'any-to-any'];
+
+// The graph plus its external-data files (<graph>.data, <graph>_data, <graph>_data_1 …).
+export const graphBytes = (model, file) =>
+  model.files.all.filter((f) => f.name === file || f.name.startsWith(`${file}.data`) || f.name.startsWith(`${file}_data`))
+    .reduce((n, f) => n + (f.size || 0), 0);
+
+// The non-generation tasks this back end runs, each through transformers.js's own pipeline(). They load the fp32
+// graphs: the f16 paths gave wrong output on Metal for text generation, and these models are small.
+const PIPELINES = {
+  'feature-extraction': { options: { pooling: 'mean', normalize: true }, out: (t) => t.tolist() },
+  'sentence-similarity': { task: 'feature-extraction', options: { pooling: 'mean', normalize: true }, out: (t) => t.tolist() },
+  'automatic-speech-recognition': {},
+  'object-detection': { image: true },
+  'image-classification': { image: true },
+  'depth-estimation': { image: true },
+  'image-segmentation': { image: true },
+  'token-classification': {},
+  'text-classification': {},
+};
+
+// fp32 graphs: onnx/*.onnx without a precision suffix (model.onnx, encoder_model.onnx, decoder_model_merged.onnx).
+const FP32 = /^onnx\/[a-z_]+(?<!_fp16|_q4|_q4f16|_quantized|_int8|_uint8|_bnb4|_q8)\.onnx$/;
+const fp32Bytes = (model) => model.files.onnx.filter((f) => FP32.test(f.name)).reduce((n, f) => n + graphBytes(model, f.name), 0);
+
+export const transformersBackend = {
+  id: 'transformers',
+  // The library, at the version src/sources.js pins. Set transformersBackend.source to load another version.
+  source: SOURCES.transformers,
+  tasks: ['text-generation', ...Object.keys(PIPELINES)],
+
+  canRun(model, caps, { task, budget = Infinity } = {}) {
+    const fits = (bytes, ok) => (bytes > budget
+      ? { ok: false, code: 'budget', need: bytes, reason: `the model stays on the GPU whole (${(bytes / 1e9).toFixed(2)} GB) and the budget is ${(budget / 1e9).toFixed(2)} GB` }
+      : ok);
+    if (!model.formats.includes('onnx')) return { ok: false, code: 'format', reason: 'no ONNX graph in the repo' };
+    // kohra's diffusion graphs are ONNX too, but transformers.js generate() is autoregressive only.
+    if (model.tags?.includes('text-diffusion')) return { ok: false, code: 'model', reason: 'a text-diffusion model; transformers.js generate() is autoregressive only' };
+    // A repo with no Hub task tag (e.g. onnx-community/Ternary-Bonsai-4B-ONNX) is taken as text generation.
+    if (isMultimodal(model)) {
+      if (task && !MM_TASKS.includes(task)) return { ok: false, code: 'task', reason: `task ${task}; ${model.architecture} runs here as chat` };
+      if (!caps.webgpu) return { ok: false, code: 'webgpu', reason: 'no WebGPU adapter' };
+      const dtype = pickMmDtype(model, caps);
+      if (!dtype) return { ok: false, code: 'format', reason: 'no decoder_model_merged graph in a precision this back end loads' };
+      const bytes = mmBytes(model, dtype);
+      return fits(bytes, { ok: true, estimate: { gpuBytes: bytes, diskBytes: bytes, tokPerSec: null } });
+    }
+    const want = task || model.task || 'text-generation';
+    if (!transformersBackend.tasks.includes(want)) return { ok: false, code: 'task', reason: `task ${want || 'unknown'}; this back end runs ${transformersBackend.tasks.join(', ')}` };
+    if (!caps.webgpu) return { ok: false, code: 'webgpu', reason: 'no WebGPU adapter' };
+    if (want !== 'text-generation') {
+      const bytes = fp32Bytes(model);
+      if (!bytes) return { ok: false, code: 'format', reason: 'no fp32 ONNX graph (onnx/model.onnx or encoder/decoder graphs)' };
+      return fits(bytes, { ok: true, estimate: { gpuBytes: bytes, diskBytes: bytes, tokPerSec: null } });
+    }
+    const dtype = pickDtype(model, caps);
+    if (!dtype) return { ok: false, code: 'format', reason: `no ONNX graph at ${Object.values(FILES).join(', ')}` };
+    const bytes = graphBytes(model, FILES[dtype]);
+    return fits(bytes, { ok: true, estimate: { gpuBytes: bytes, diskBytes: bytes, tokPerSec: null } });
+  },
+
+  // dtype overrides the precision pick (the repo must ship it). generate() takes repetitionPenalty; without it the
+  // model's own generation_config.json applies (Qwen2.5 ships 1.05), as in transformers.js itself.
+  async load(model, { task, caps, dtype: forced, onProgress, signal }) {
+    const want = task || model.task || 'text-generation';
+    const lib = await import(transformersBackend.source);
+    const progress_callback = onProgress && ((p) => onProgress({ backend: 'transformers', ...p }));
+    if (isMultimodal(model)) return loadMultimodal(lib, model, forced || pickMmDtype(model, caps), { progress_callback, signal });
+    if (want !== 'text-generation') return loadPipeline(lib, model, want, { progress_callback, signal });
+    const dtype = forced || pickDtype(model, caps);
+    const gen = await lib.pipeline('text-generation', model.repo, {
+      device: 'webgpu', dtype, revision: model.revision || 'main', progress_callback,
+    });
+    signal?.throwIfAborted();
+    return {
+      task: 'text-generation',
+      generate: (messages, { maxNewTokens = 256, repetitionPenalty, signal } = {}) => stream(lib, gen.tokenizer, signal,
+        (streamer, stopping_criteria) => gen(messages, { max_new_tokens: maxNewTokens, do_sample: false, ...(repetitionPenalty && { repetition_penalty: repetitionPenalty }), streamer, stopping_criteria })),
+      dispose: () => gen.dispose?.(),
+    };
+  },
+};
+
+// A non-generation task: one pipeline, run(input, opts) calls it. ImageData (from a canvas) becomes a RawImage; URLs,
+// RawImage and, for speech, 16 kHz Float32Array samples pass through as transformers.js takes them.
+async function loadPipeline(lib, model, task, { progress_callback, signal }) {
+  const spec = PIPELINES[task];
+  const pipe = await lib.pipeline(spec.task || task, model.repo, {
+    device: 'webgpu', dtype: 'fp32', revision: model.revision || 'main', progress_callback,
+  });
+  signal?.throwIfAborted();
+  const image = (x) => (typeof ImageData !== 'undefined' && x instanceof ImageData ? new lib.RawImage(x.data, x.width, x.height, 4) : x);
+  return {
+    task,
+    async run(input, opts = {}) {
+      const out = await pipe(spec.image ? image(input) : input, { ...spec.options, ...opts });
+      return spec.out ? spec.out(out) : out;
+    },
+    dispose: () => pipe.dispose?.(),
+  };
+}
+
+// transformers.js streams through a callback; this turns one generate call into the contract's async iterable of
+// { text, tokens }, with the signal wired to an InterruptableStoppingCriteria.
+async function* stream(lib, tokenizer, signal, run) {
+  const queue = [];
+  let wake = () => {}, done = false, failed = null, text = '', tokens = 0;
+  const streamer = new lib.TextStreamer(tokenizer, {
+    skip_prompt: true, skip_special_tokens: true,
+    callback_function: (t) => { text += t; queue.push({ text, tokens }); wake(); },
+    token_callback_function: () => { tokens++; },
+  });
+  const stop = new lib.InterruptableStoppingCriteria();
+  signal?.addEventListener('abort', () => stop.interrupt(), { once: true });
+  run(streamer, stop).catch((e) => { failed = e; }).finally(() => { done = true; wake(); });
+  while (true) {
+    if (queue.length) { yield queue.shift(); continue; }
+    if (done) break;
+    await new Promise((r) => { wake = r; });
+  }
+  if (failed) throw failed;
+}
+
+// A multimodal chat model. Message content is a string or an array of blocks: { type: 'text', text },
+// { type: 'image', image } (ImageData, RawImage or a URL) and, for Gemma 4, { type: 'audio', audio } (16 kHz
+// Float32Array). The template sees placeholders; the processor gets the media, in order.
+async function loadMultimodal(lib, model, dtype, { progress_callback, signal }) {
+  const opts = { revision: model.revision || 'main', progress_callback };
+  const processor = await lib.AutoProcessor.from_pretrained(model.repo, opts);
+  const net = await lib[MULTIMODAL[model.architecture]].from_pretrained(model.repo, { ...opts, dtype, device: 'webgpu' });
+  signal?.throwIfAborted();
+  // One-token warmup compiles the kernels before the first reply (LocalMind does the same).
+  await net.generate({ ...(await processor('a', null, null, { add_special_tokens: false })), max_new_tokens: 1 });
+  const toImage = async (x) => (typeof ImageData !== 'undefined' && x instanceof ImageData ? new lib.RawImage(x.data, x.width, x.height, 4)
+    : typeof x === 'string' ? lib.load_image(x) : x);
+  return {
+    task: 'text-generation',
+    generate: (messages, { maxNewTokens = 256, repetitionPenalty, signal } = {}) => stream(lib, processor.tokenizer, signal, async (streamer, stopping_criteria) => {
+      const images = [], audios = [];
+      const plain = messages.map((m) => (typeof m.content === 'string' ? m : {
+        ...m,
+        content: m.content.map((b) => {
+          if (b.type === 'image') { images.push(b.image); return { type: 'image' }; }
+          if (b.type === 'audio') { audios.push(b.audio); return { type: 'audio' }; }
+          return b;
+        }),
+      }));
+      const prompt = processor.apply_chat_template(plain, { add_generation_prompt: true, enable_thinking: false });
+      const imgs = await Promise.all(images.map(toImage));
+      const one = (a) => (a.length ? (a.length === 1 ? a[0] : a) : null);
+      const inputs = await processor(prompt, one(imgs), one(audios), { add_special_tokens: false });
+      return net.generate({ ...inputs, max_new_tokens: maxNewTokens, do_sample: false, ...(repetitionPenalty && { repetition_penalty: repetitionPenalty }), streamer, stopping_criteria });
+    }),
+    dispose: () => net.dispose?.(),
+  };
+}

package/src/backends/wllama.js

@@ -0,0 +1,122 @@
+// wllama: llama.cpp compiled to WebAssembly, for GGUF models no hand-written engine covers (any architecture its
+// llama.cpp knows, e.g. openbmb/MiniCPM5-2B-GGUF). It offloads every layer to WebGPU when the browser has it and runs
+// on the CPU otherwise, so it is anyformer's one path that needs no WebGPU. Listed last in BACKENDS: an engine written
+// for the architecture (localmind, diskformer) wins when one exists.
+
+import { SOURCES } from '../sources.js';
+import { splitToOpfs } from '../gguf-split.js';
+
+// wllama reads files of at most 2 GB (one ArrayBuffer each) into a 32-bit WebAssembly memory of 4 GB. A model over
+// 2 GB loads as shards: the repo's own split set when it has one, else shards anyformer writes to OPFS
+// (src/gguf-split.js). The whole model, shards included, must stay under TOTAL_LIMIT to leave room for the context.
+const FILE_LIMIT = 2 ** 31;
+const TOTAL_LIMIT = 3.6e9;
+const QUANTS = ['Q4_K_M', 'Q4_0', 'Q4_K_S', 'Q5_K_M', 'Q5_K_S', 'Q6_K', 'Q8_0'];
+const SPLIT = /-(\d{5})-of-(\d{5})\.gguf$/i;
+
+// What to load: { kind: 'file' | 'split' | 'local-split', name, files: [{ name, size }], bytes }, or null.
+function pickFile(model) {
+  const lms = model.files.gguf.filter((f) => !/mmproj/i.test(f.name));
+  const sets = new Map();
+  for (const f of lms.filter((x) => SPLIT.test(x.name))) {
+    const base = f.name.replace(SPLIT, '');
+    sets.set(base, [...(sets.get(base) || []), f]);
+  }
+  for (const q of QUANTS) {
+    const quant = new RegExp(`[-_.]${q}$`, 'i');
+    const single = lms.find((x) => !SPLIT.test(x.name) && quant.test(x.name.replace(/\.gguf$/i, '')));
+    if (single) return { kind: single.size > FILE_LIMIT ? 'local-split' : 'file', name: single.name, files: [single], bytes: single.size };
+    for (const [base, parts] of sets) {
+      const total = Number(SPLIT.exec(parts[0].name)[2]);
+      if (quant.test(base) && parts.length === total) {
+        const files = [...parts].sort((a, b) => a.name.localeCompare(b.name));
+        return { kind: 'split', name: files[0].name, files, bytes: files.reduce((n, f) => n + f.size, 0) };
+      }
+    }
+  }
+  return null;
+}
+
+// wllama 3.4.1's stream ends one result early: the last chunk stays queued in the wasm side and would open the next
+// request. Read the queue dry before and after every reply, as LocalMind does (its WLLAMA-DRAIN, index.html).
+async function drain(w) {
+  const proxy = w?.proxy;
+  if (!proxy?.wllamaAction) return [];
+  const out = [];
+  for (let i = 0; i < 64; i++) {
+    const r = await proxy.wllamaAction('get_result', { _name: 'gres_req' });
+    const j = r?.data_json;
+    if (!j || j === 'null') { if (!r?.has_more) break; continue; }
+    let arr; try { arr = JSON.parse(j); } catch { break; }
+    out.push(...(Array.isArray(arr) ? arr : [arr]));
+  }
+  return out;
+}
+const delta = (chunk) => chunk?.choices?.[0]?.delta?.content || '';
+
+export const wllamaBackend = {
+  id: 'wllama',
+  tasks: ['text-generation'],
+  source: SOURCES.wllama,
+
+  canRun(model, caps, { budget, task } = {}) {
+    if (!model.formats.includes('gguf')) return { ok: false, code: 'format', reason: 'no GGUF in the repo' };
+    if (task && task !== 'text-generation' && task !== 'image-text-to-text') return { ok: false, code: 'task', reason: `task ${task}; wllama runs text generation` };
+    const f = pickFile(model);
+    if (!f) return { ok: false, code: 'model', reason: `no GGUF at ${QUANTS.join(', ')}` };
+    if (f.bytes > TOTAL_LIMIT) return { ok: false, code: 'model', reason: `${f.name} is ${(f.bytes / 1e9).toFixed(2)} GB; wllama's 32-bit WebAssembly memory holds about ${TOTAL_LIMIT / 1e9} GB of model` };
+    if (f.kind === 'local-split' && !caps.opfs) return { ok: false, code: 'opfs', reason: `${f.name} is over 2 GB and is split into OPFS before loading` };
+    // On WebGPU the whole model sits on the GPU, so it must fit the budget; without WebGPU it runs on the CPU.
+    const gpu = caps.webgpu && f.bytes <= budget;
+    return { ok: true, estimate: { gpuBytes: gpu ? f.bytes : 0, diskBytes: f.bytes, tokPerSec: null } };
+  },
+
+  async load(model, { caps, budget, maxCtx = 4096, onProgress, signal }) {
+    const f = pickFile(model);
+    const { Wllama } = await import(`${wllamaBackend.source}/index.js`);
+    const w = new Wllama({ default: `${wllamaBackend.source}/wasm/wllama.wasm` }, { logger: { debug() {}, log() {}, warn() {}, error: (m) => console.error('[wllama]', m) } });
+    // isSupportWebGPU() may answer directly or with a promise, depending on the wllama version.
+    let gpu = false;
+    if (caps.webgpu && f.bytes <= budget) { try { gpu = !!(await w.isSupportWebGPU()); } catch { gpu = false; } }
+    const rev = model.revision || 'main';
+    const url = (name) => `https://huggingface.co/${model.repo}/resolve/${rev}/${name}`;
+    const cfg = {
+      n_ctx: maxCtx, n_gpu_layers: gpu ? 999 : 0,
+      progressCallback: onProgress && (({ loaded, total }) => onProgress({ backend: 'wllama', status: 'download', file: f.name, loaded, total })),
+    };
+    if (f.kind === 'local-split') {
+      const shards = await splitToOpfs({
+        url: url(f.name), size: f.bytes, repo: model.repo, file: f.name, signal,
+        key: `${model.repo}--${rev}--${f.name}`.replace(/[^A-Za-z0-9._-]+/g, '-'),
+        onProgress: onProgress && ((p) => onProgress({ backend: 'wllama', ...p })),
+      });
+      await w.loadModel(shards, cfg);
+    } else {
+      await w.loadModelFromUrl(f.kind === 'split' ? f.files.map((x) => url(x.name)) : url(f.name), cfg);
+    }
+    signal?.throwIfAborted();
+    return {
+      task: 'text-generation',
+      device: gpu ? 'webgpu' : 'cpu',
+      async *generate(messages, { maxNewTokens = 256, signal } = {}) {
+        const msgs = messages.map((m) => ({ role: m.role, content: typeof m.content === 'string' ? m.content : m.content.filter((b) => b.type === 'text').map((b) => b.text).join('\n') }));
+        await drain(w).catch(() => {});
+        const ctl = new AbortController();
+        signal?.addEventListener('abort', () => ctl.abort(), { once: true });
+        let text = '', tokens = 0;
+        try {
+          // Thinking off, as on every other back end: MiniCPM5 otherwise spends the reply in reasoning_content.
+          const stream = await w.createChatCompletion({
+            messages: msgs, stream: true, max_tokens: maxNewTokens, temperature: 0, abortSignal: ctl.signal,
+            chat_template_kwargs: { enable_thinking: false },
+          });
+          for await (const c of stream) { const d = delta(c); if (d) { text += d; tokens++; yield { text, tokens }; } }
+          for (const c of await drain(w)) { const d = delta(c); if (d) { text += d; tokens++; yield { text, tokens }; } }
+        } catch (e) {
+          if (!ctl.signal.aborted) throw e;
+        }
+      },
+      dispose: () => w.exit(),
+    };
+  },
+};

package/src/describe.js

@@ -0,0 +1,34 @@
+// A model descriptor from a Hugging Face repo id: which weight formats the repo holds, their files and sizes, and the
+// architecture. Back ends decide from this alone; describe() never decides who runs a model.
+
+const HF = 'https://huggingface.co';
+
+export async function describe(repo, { fetch = globalThis.fetch, revision = 'main', endpoint = HF } = {}) {
+  const url = `${endpoint}/api/models/${repo}${revision === 'main' ? '' : `/revision/${revision}`}?blobs=true`;
+  const res = await fetch(url);
+  if (!res.ok) throw new Error(`describe ${repo}: ${url} returned ${res.status}`);
+  return fromHubInfo(await res.json(), { endpoint });
+}
+
+// Pure: Hub API JSON → descriptor. Exported for tests and for callers that already hold the JSON.
+export function fromHubInfo(info, { endpoint = HF } = {}) {
+  const files = (info.siblings || []).map((s) => ({ name: s.rfilename, size: s.size ?? null }));
+  const gguf = files.filter((f) => f.name.endsWith('.gguf'));
+  const onnx = files.filter((f) => f.name.endsWith('.onnx'));
+  const safetensors = files.filter((f) => f.name.endsWith('.safetensors'));
+  const formats = [gguf.length && 'gguf', onnx.length && 'onnx', safetensors.length && 'safetensors'].filter(Boolean);
+  return {
+    repo: info.id,
+    revision: info.sha || null,
+    endpoint,
+    formats,
+    // GGUF repos report the architecture in the gguf block; transformers repos in config.model_type.
+    architecture: info.gguf?.architecture || info.config?.model_type || null,
+    task: info.pipeline_tag || null,
+    // Hub tags carry what format and architecture miss, e.g. 'text-diffusion' marks a kohra model.
+    tags: info.tags || [],
+    // all: every file, for the side files a format splits off (ONNX external data, tokenizers).
+    files: { gguf, onnx, safetensors, all: files },
+    ggufTotalBytes: info.gguf?.total ?? null,
+  };
+}

package/src/errors.js

@@ -0,0 +1,36 @@
+// Every error load() and a session throw is an AnyformerError with a code from ERROR_CODES. Callers branch on the
+// code; the message is for people and can change. (Back-end refusals before loading are REFUSALS in resolve.js.)
+
+export const ERROR_CODES = {
+  NO_BACKEND: 'no back end can run the model under this budget; err.plan.candidates holds each code and reason',
+  LOAD_FAILED: 'the repo lookup, a download or the back end failed while loading; err.cause holds the original error',
+  BUDGET_TOO_SMALL: 'the back end measured the model at load time and the budget does not cover it; retry with err.need',
+  ABORTED: 'the AbortSignal fired; err.cause holds its reason',
+  EXECUTION_FAILED: 'the back end failed while generating; err.cause holds the original error',
+  INVALID_ARGUMENT: 'a value passed by the caller is wrong',
+  DISPOSED: 'the session was disposed; load the model again',
+};
+
+export class AnyformerError extends Error {
+  constructor(code, message, { cause, ...extra } = {}) {
+    super(message, cause === undefined ? undefined : { cause });
+    this.code = code;
+    Object.assign(this, extra);
+  }
+}
+// On the prototype, so stack traces say AnyformerError.
+AnyformerError.prototype.name = 'AnyformerError';
+
+// By name and code, not instanceof: a page can hold two copies of anyformer (CDN imports), and the check still holds.
+export function isAnyformerError(e, code) {
+  return e instanceof Error && e.name === 'AnyformerError' && e.code in ERROR_CODES && (code === undefined || e.code === code);
+}
+
+// Any error into an AnyformerError. An aborted signal wins: fetch rejects with the abort reason, which is not always
+// an AbortError. An AnyformerError keeps its code; anything else gets `code` (the step that failed).
+export function wrap(e, code, signal, where = '') {
+  if (signal?.aborted) return isAnyformerError(e, 'ABORTED') ? e : new AnyformerError('ABORTED', `${where}aborted`, { cause: signal.reason });
+  if (isAnyformerError(e)) return e;
+  if (e?.name === 'AbortError') return new AnyformerError('ABORTED', `${where}aborted`, { cause: e });
+  return new AnyformerError(code, `${where}${e?.message ?? e}`, { cause: e });
+}

package/src/gguf-split.js

@@ -0,0 +1,160 @@
+// A GGUF too big for one ArrayBuffer (wllama reads files of at most 2 GB) becomes shards in llama.cpp's gguf-split
+// layout, the same files `llama-gguf-split --split-max-size` writes: shard 1 carries every metadata key, each shard
+// carries split.no / split.count / split.tensors.count and its own tensor table, and each shard's data section holds
+// its tensors with offsets relative to it. Tensor bytes are copied, never decoded, so any quant type works.
+
+const MAGIC = 0x46554747;                   // 'GGUF'
+const DEFAULT_ALIGNMENT = 32;
+// GGUF metadata value types → byte size (fixed-size types only; 8 = string, 9 = array).
+const SIZE = { 0: 1, 1: 1, 2: 2, 3: 2, 4: 4, 5: 4, 6: 4, 7: 1, 10: 8, 11: 8, 12: 8 };
+
+class Short extends Error {}                // the header runs past the bytes read so far
+
+// Parse a GGUF header from its first bytes. Returns the raw metadata bytes (copied verbatim into shard 1), the tensor
+// table, the alignment and where the data section starts. Throws Short when `u8` stops inside the header.
+export function parseHeader(u8) {
+  const dv = new DataView(u8.buffer, u8.byteOffset, u8.byteLength);
+  let p = 0;
+  const need = (n) => { if (p + n > u8.byteLength) throw new Short(); };
+  const u32 = () => { need(4); const v = dv.getUint32(p, true); p += 4; return v; };
+  const u64 = () => { need(8); const v = dv.getBigUint64(p, true); p += 8; return Number(v); };
+  const str = () => { const n = u64(); need(n); const s = new TextDecoder().decode(u8.subarray(p, p + n)); p += n; return s; };
+  const skip = (type) => {
+    if (type === 8) { const n = u64(); need(n); p += n; return; }
+    if (type === 9) { const t = u32(), n = u64(); if (t in SIZE) { need(n * SIZE[t]); p += n * SIZE[t]; } else for (let i = 0; i < n; i++) skip(t); return; }
+    need(SIZE[type]); p += SIZE[type];
+  };
+  if (u32() !== MAGIC) throw new Error('not a GGUF file');
+  const version = u32();
+  if (version < 2) throw new Error(`GGUF version ${version} is not supported`);
+  const tensorCount = u64(), kvCount = u64();
+  const kvStart = p;
+  let alignment = DEFAULT_ALIGNMENT;
+  for (let i = 0; i < kvCount; i++) {
+    const key = str(), type = u32();
+    if (key === 'general.alignment' && type === 4) { need(4); alignment = dv.getUint32(p, true); }
+    if (key.startsWith('split.')) throw new Error(`already a shard (${key})`);
+    skip(type);
+  }
+  const kvBytes = u8.slice(kvStart, p);
+  const tensors = [];
+  for (let i = 0; i < tensorCount; i++) {
+    const name = str(), nDims = u32(), dims = [];
+    for (let d = 0; d < nDims; d++) dims.push(u64());
+    tensors.push({ name, dims, type: u32(), offset: u64() });
+  }
+  const dataStart = Math.ceil(p / alignment) * alignment;
+  return { version, kvCount, kvBytes, tensors, alignment, dataStart };
+}
+
+// Read the header through `read(start, end)` (bytes [start, end)), widening the window until it parses.
+export async function readHeader(read, fileSize) {
+  for (let n = 8 << 20; ; n *= 2) {
+    try { return parseHeader(await read(0, Math.min(n, fileSize))); } catch (e) {
+      if (!(e instanceof Short) || n >= fileSize) throw e;
+    }
+  }
+}
+
+// Group tensors (in data order) into shards of at most maxBytes of data each. A tensor's bytes run to the next
+// tensor's offset, so the original alignment padding travels with it and every copied offset stays aligned.
+export function planShards(header, fileSize, maxBytes) {
+  const t = [...header.tensors].sort((a, b) => a.offset - b.offset);
+  const dataSize = fileSize - header.dataStart;
+  const sized = t.map((x, i) => ({ ...x, bytes: (i + 1 < t.length ? t[i + 1].offset : dataSize) - x.offset }));
+  const shards = [];
+  let cur = null;
+  for (const x of sized) {
+    if (!cur || (cur.bytes + x.bytes > maxBytes && cur.tensors.length)) shards.push(cur = { tensors: [], bytes: 0 });
+    cur.tensors.push({ ...x, newOffset: cur.bytes });
+    cur.bytes += x.bytes;
+  }
+  for (const s of shards) { s.start = header.dataStart + s.tensors[0].offset; s.end = s.start + s.bytes; }
+  return shards;
+}
+
+// The header bytes of shard i of n: magic, version, counts, metadata (all of it in shard 0, split keys only in the
+// rest), this shard's tensor table, padding to the alignment.
+export function shardHeader(header, shards, i) {
+  const out = [];
+  const enc = new TextEncoder();
+  const u32 = (v) => { const b = new Uint8Array(4); new DataView(b.buffer).setUint32(0, v, true); out.push(b); };
+  const u64 = (v) => { const b = new Uint8Array(8); new DataView(b.buffer).setBigUint64(0, BigInt(v), true); out.push(b); };
+  const u16 = (v) => { const b = new Uint8Array(2); new DataView(b.buffer).setUint16(0, v, true); out.push(b); };
+  const i32 = (v) => { const b = new Uint8Array(4); new DataView(b.buffer).setInt32(0, v, true); out.push(b); };
+  const str = (s) => { const b = enc.encode(s); u64(b.length); out.push(b); };
+  const shard = shards[i];
+  u32(MAGIC); u32(header.version);
+  u64(shard.tensors.length);
+  u64((i === 0 ? header.kvCount : 0) + 3);
+  if (i === 0) out.push(header.kvBytes);
+  str('split.no'); u32(2); u16(i);
+  str('split.count'); u32(2); u16(shards.length);
+  str('split.tensors.count'); u32(5); i32(header.tensors.length);
+  for (const t of shard.tensors) {
+    str(t.name); u32(t.dims.length);
+    for (const d of t.dims) u64(d);
+    u32(t.type); u64(t.newOffset);
+  }
+  // Shards after the first carry no general.alignment, so llama.cpp reads them at the default 32.
+  const align = i === 0 ? header.alignment : DEFAULT_ALIGNMENT;
+  const len = out.reduce((n, b) => n + b.length, 0);
+  out.push(new Uint8Array(Math.ceil(len / align) * align - len));
+  const all = new Uint8Array(out.reduce((n, b) => n + b.length, 0));
+  let p = 0;
+  for (const b of out) { all.set(b, p); p += b.length; }
+  return all;
+}
+
+export const shardName = (base, i, n) => `${base}-${String(i + 1).padStart(5, '0')}-of-${String(n).padStart(5, '0')}.gguf`;
+
+// Split the GGUF at `url` into shards of at most maxBytes in OPFS (anyformer-gguf/<key>/), reading it with HTTP range
+// requests, and return the shards as Files in order. Crash-safe and resumable: a shard of the expected size is kept,
+// anything else is rewritten, and done.json is written last; with done.json present nothing is fetched.
+export const SPLIT_ROOT = 'anyformer-gguf';
+export async function splitToOpfs({ url, size, key, repo, file, maxBytes = 512 << 20, onProgress, signal, fetch: f = globalThis.fetch }) {
+  const root = await (await navigator.storage.getDirectory()).getDirectoryHandle(SPLIT_ROOT, { create: true });
+  const dir = await root.getDirectoryHandle(key, { create: true });
+  const files = async (names) => Promise.all(names.map(async (n) => (await dir.getFileHandle(n)).getFile()));
+  // meta.json first, so an interrupted split still names its repo (src/store.js lists and removes it).
+  const meta = await (await dir.getFileHandle('meta.json', { create: true })).createWritable();
+  await meta.write(JSON.stringify({ repo, file, url, size }));
+  await meta.close();
+  try {
+    const done = JSON.parse(await (await (await dir.getFileHandle('done.json')).getFile()).text());
+    if (done.url === url && done.size === size) return files(done.shards);
+  } catch { /* not split yet, or interrupted */ }
+  const range = async (s, e) => {
+    const r = await f(url, { headers: { Range: `bytes=${s}-${e - 1}` }, signal });
+    if (r.status !== 206 && !(r.ok && s === 0 && e >= size)) throw new Error(`${url}: range request returned ${r.status}`);
+    return r;
+  };
+  const header = await readHeader(async (s, e) => new Uint8Array(await (await range(s, e)).arrayBuffer()), size);
+  const shards = planShards(header, size, maxBytes);
+  const base = file.replace(/\.gguf$/i, '');
+  const names = shards.map((_, i) => shardName(base, i, shards.length));
+  let copied = 0;
+  for (const [i, s] of shards.entries()) {
+    const head = shardHeader(header, shards, i);
+    const expect = head.length + s.bytes;
+    const h = await dir.getFileHandle(names[i], { create: true });
+    if ((await h.getFile()).size === expect) { copied += s.bytes; continue; }
+    const w = await h.createWritable();           // commits atomically on close(); an abort leaves the old file
+    try {
+      await w.write(head);
+      const body = (await range(s.start, s.end)).body.getReader();
+      for (;;) {
+        const { done, value } = await body.read();
+        if (done) break;
+        await w.write(value);
+        copied += value.length;
+        onProgress?.({ status: 'split', file: names[i], loaded: copied, total: size - header.dataStart });
+      }
+      await w.close();
+    } catch (e) { await w.abort().catch(() => {}); throw e; }
+  }
+  const marker = await (await dir.getFileHandle('done.json', { create: true })).createWritable();
+  await marker.write(JSON.stringify({ url, size, repo, file, shards: names }));
+  await marker.close();
+  return files(names);
+}

package/src/load.js

@@ -0,0 +1,106 @@
+import { probe } from './probe.js';
+import { describe } from './describe.js';
+import { resolve } from './resolve.js';
+import { transformersBackend } from './backends/transformers.js';
+import { kohraBackend } from './backends/kohra.js';
+import { diskformerBackend } from './backends/diskformer.js';
+import { localmindBackend } from './backends/localmind.js';
+import { wllamaBackend } from './backends/wllama.js';
+import { AnyformerError, wrap } from './errors.js';
+
+// Back ends land here as each one passes its browser test (test/browser/). The list order breaks ties.
+export const BACKENDS = [transformersBackend, kohraBackend, diskformerBackend, localmindBackend, wllamaBackend];
+
+// 2.5 GB: the smallest budget diskformer's chat gate passes at. Browsers do not report GPU memory, so a caller that
+// knows more passes { budget }.
+export const DEFAULT_BUDGET = 2.5e9;
+
+function noBackend(plan) {
+  const lines = plan.candidates.map((c) => `  ${c.backend} [${c.code}]: ${c.reason}`);
+  return new AnyformerError('NO_BACKEND', `no back end can run ${plan.model.repo} at a ${(plan.budget / 1e9).toFixed(1)} GB budget` +
+    (lines.length ? `:\n${lines.join('\n')}` : ': no back ends registered'), { plan });
+}
+
+// What load() would do, without loading: the descriptor, the caps, and every back end's answer.
+// task: a Hugging Face pipeline task ('text-generation', 'feature-extraction', …); null runs the repo's own.
+// repo: a Hub id, or an entry from MODELS (its task and budget apply unless opts set them).
+const entry = (repo, opts) => (typeof repo === 'object' && repo?.repo
+  ? [repo.repo, { ...(repo.task && { task: repo.task }), ...(repo.budget && { budget: repo.budget }), ...opts }]
+  : [repo, opts]);
+
+export async function plan(repoOrEntry, opts = {}) {
+  let [repo, { budget = DEFAULT_BUDGET, task = null, backends = BACKENDS, caps, model, fetch }] = entry(repoOrEntry, opts);
+  caps ??= await probe();
+  model ??= await describe(repo, fetch ? { fetch } : {});
+  return { model, caps, budget, task, ...(await resolve(model, caps, backends, { budget, task })) };
+}
+
+export async function load(repoOrEntry, options = {}) {
+  const [repo, opts] = entry(repoOrEntry, options);
+  const { backends = BACKENDS, maxCtx, dtype, onProgress, signal } = opts;
+  let p;
+  try { p = await plan(repo, opts); } catch (e) { throw wrap(e, 'LOAD_FAILED', signal, `${repo}: `); }
+  if (!p.chosen) throw noBackend(p);
+  const backend = backends.find((b) => b.id === p.chosen);
+  let session;
+  try {
+    session = await backend.load(p.model, { budget: p.budget, task: p.task, caps: p.caps, maxCtx, dtype, onProgress, signal });
+  } catch (e) { throw wrap(e, 'LOAD_FAILED', signal, `${backend.id}: `); }
+  return { backend: backend.id, task: session.task || p.task || p.model.task || null, ...(session.device && { device: session.device }), plan: p, ...serialize(session) };
+}
+
+// content: a string, or an array of { type, … } blocks (text, image, audio) for multimodal models.
+const validContent = (c) => typeof c === 'string' || (Array.isArray(c) && c.every((b) => typeof b?.type === 'string'));
+const validMessages = (messages) => Array.isArray(messages) && messages.every((m) => typeof m?.role === 'string' && validContent(m?.content));
+
+// One call at a time per session: two overlapping generates would share one KV cache. A call waits for the previous
+// one to finish (or for its caller to stop iterating). dispose() waits for the running call, then frees the GPU;
+// calls after it throw DISPOSED.
+function serialize(session) {
+  let tail = Promise.resolve(), disposed = false;
+  const gone = () => new AnyformerError('DISPOSED', 'this session was disposed; load the model again');
+  const turn = async () => {
+    let release;
+    const mine = new Promise((r) => { release = r; });
+    const prev = tail;
+    tail = prev.then(() => mine);
+    await prev;
+    return release;
+  };
+  const generate = session.generate && async function* (messages, o = {}) {
+    if (disposed) throw gone();
+    if (!validMessages(messages)) throw new AnyformerError('INVALID_ARGUMENT', 'messages must be an array of { role, content }, content a string or an array of { type } blocks');
+    const release = await turn();
+    try {
+      if (disposed) throw gone();
+      for await (const chunk of session.generate(messages, o)) yield chunk;
+    } catch (e) {
+      throw wrap(e, 'EXECUTION_FAILED', o.signal);
+    } finally {
+      release();
+    }
+  };
+  return {
+    ...(generate && { generate }),
+    // The one call every task has. For text generation it is the last chunk of generate().
+    async run(input, o = {}) {
+      if (generate) { let last; for await (const c of generate(input, o)) last = c; return last; }
+      if (disposed) throw gone();
+      const release = await turn();
+      try {
+        if (disposed) throw gone();
+        return await session.run(input, o);
+      } catch (e) {
+        throw wrap(e, 'EXECUTION_FAILED', o.signal);
+      } finally {
+        release();
+      }
+    },
+    async dispose() {
+      if (disposed) return;
+      disposed = true;
+      await tail;
+      await session.dispose?.();
+    },
+  };
+}