diskformer 0.0.1 → 0.2.0

package/index.js

@@ -0,0 +1,8 @@
+// diskformer — a disk tier for in-browser inference. Keep model weights in OPFS and page into WebGPU only
+// what each step needs. Plain ESM, no dependencies, no build step.
+export { OpfsReaderPool, OpfsWriter, canInline, opfsDir, readOpfsText, writeOpfsText, removeOpfs } from './src/opfs-reader.js';
+export { ExpertStreamer, ExpertStreamer as RecordPool } from './src/expert-stream.js';
+export { RowFile, RowCache, fingerprint } from './src/rows.js';
+export { parseGguf, tensorBytes, splitQ8, splitQ4, GGML, Q8_BLOCK, Q4_BLOCK, Q6K_BLOCK } from './src/gguf.js';
+export { ingestGguf, ingestProgress, readHeader, storeKey, removeStore } from './src/ingest.js';
+export { measure } from './src/measure.js';

package/package.json

@@ -1,13 +1,47 @@
 {
   "name": "diskformer",
-  "version": "0.0.1",
-  "description": "A disk tier for in-browser inference: keep model weights in OPFS and page into WebGPU only what each step needs. Pre-release name placeholder; v0.1 is being extracted from LocalMind.",
+  "version": "0.2.0",
+  "description": "Run a model larger than your GPU memory in a browser tab: weights stay in OPFS, WebGPU holds only what each step needs. A disk tier plus two reference MoE engines.",
   "type": "module",
   "license": "MIT",
   "author": "Chirag Patnaik",
   "homepage": "https://github.com/NakliTechie/diskformer.js#readme",
-  "repository": { "type": "git", "url": "git+https://github.com/NakliTechie/diskformer.js.git" },
-  "bugs": { "url": "https://github.com/NakliTechie/diskformer.js/issues" },
-  "keywords": ["webgpu", "opfs", "llm", "inference", "offload", "moe", "browser", "transformers"],
-  "files": ["README.md", "LICENSE"]
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/NakliTechie/diskformer.js.git"
+  },
+  "bugs": {
+    "url": "https://github.com/NakliTechie/diskformer.js/issues"
+  },
+  "keywords": [
+    "webgpu",
+    "opfs",
+    "llm",
+    "inference",
+    "offload",
+    "moe",
+    "browser",
+    "transformers"
+  ],
+  "files": [
+    "index.js",
+    "src/",
+    "engines/",
+    "README.md",
+    "LICENSE",
+    "CHANGELOG.md"
+  ],
+  "main": "index.js",
+  "exports": {
+    ".": "./index.js",
+    "./engines/*": "./engines/*"
+  },
+  "scripts": {
+    "test": "node test/run.mjs",
+    "demo": "node examples/run-demo.mjs",
+    "sync": "node scripts/sync-localmind.mjs"
+  },
+  "engines": {
+    "node": ">=20"
+  }
 }

package/src/expert-stream.js

@@ -0,0 +1,265 @@
+/* moe-expert-stream.js — keep a mixture-of-experts model's routed experts on SSD (OPFS) and
+ * page them into a fixed GPU slot pool on demand. Model-agnostic: an engine describes one
+ * expert record (its byte layout in the OPFS file and which GPU pool buffer each part goes
+ * to); this module owns the slot LRU, the reads, the uploads, the pins and the counters.
+ *
+ *   const xs = new ExpertStreamer({ device, reader, recordBytes, recordOffset, parts, slots,
+ *                                   numLayers, numExperts });
+ *   const slots = await xs.ensure(layer, expertIds);   // resident + pinned, in id order
+ *   …write `slots` into the kernel's slot table, encode, queue.submit()…
+ *   xs.release(slots);                                  // after the submit
+ *   xs.prefetch(layer + 1, predictedIds);               // optional, never blocks
+ *
+ * `parts`: [{ buffer: GPUBuffer, srcOffset, bytes }]. Slot s of a part lives at s × bytes in
+ * that part's buffer, so a kernel finds expert row r of slot s at s × rowsPerExpert + r.
+ *
+ * Safety of reuse: an upload (a staging-ring copy, or queue.writeBuffer) is ordered after every
+ * earlier queue.submit, so a slot may be overwritten as soon as the work that reads it has been
+ * submitted, even if the GPU has not run it yet. Until then the slot is pinned and never chosen
+ * for eviction.
+ */
+
+export class ExpertStreamer {
+  constructor({ device, reader, recordBytes, recordOffset, parts, slots, numLayers, numExperts, maxStaging = 32, uploadRing = 16, evict = 'lru', hotHalfLife = 8, maxPrefetchInflight = 8 }) {
+    Object.assign(this, { device, reader, recordBytes, recordOffset, parts, slots, numLayers, numExperts, maxStaging, evict, maxPrefetchInflight });
+    // Prefetches wait in pfQueue (nearest layer first) and at most maxPrefetchInflight of them
+    // read at once, so a demand read never queues behind a long run of guesses in the workers.
+    this.pfQueue = [];
+    this.pfInflight = 0;
+    // evict 'hot': drop the unpinned expert with the lowest decaying route count, oldest first
+    // on ties (llama.cpp PR #25294's policy). One tick per ensure() call, so the half-life is in
+    // tokens × layers. 'lru' (default): plain least recently used.
+    this.heat = new Float32Array(numLayers * numExperts);
+    this.heatTick = new Float64Array(numLayers * numExperts);
+    this.tick = 0;
+    this.decayPerTick = Math.pow(0.5, 1 / Math.max(1, hotHalfLife * numLayers));
+    // Uploads go through a ring of mapped MAP_WRITE buffers + copyBufferToBuffer: measured
+    // 2026-10-04 on an M4 Pro (Chrome 154) at 27.5 GB/s for 5 MB records with depth 4, against
+    // 1.88 GB/s for queue.writeBuffer. uploadRing = 0 keeps writeBuffer (A/B measurement only).
+    this.ring = [];
+    for (let i = 0; i < uploadRing; i++) {
+      this.ring.push({ buf: device.createBuffer({ size: Math.ceil(recordBytes / 4) * 4, usage: 0x02 | 0x04, mappedAtCreation: true }), ready: null });
+    }
+    this.ringNext = 0;
+    this.slotKey = new Int32Array(slots).fill(-1);
+    this.pinCount = new Int32Array(slots);
+    this.free = [];
+    for (let s = slots - 1; s >= 0; s--) this.free.push(s);
+    this.lru = new Map();        // key → slot, least recently used first
+    this.inflight = new Map();   // key → { p: Promise<slot>, pins }
+    this.staging = [];
+    this.resetStats();
+  }
+
+  resetStats() {
+    const L = this.numLayers;
+    this.stats = {
+      hits: 0, misses: 0, lateHits: 0, prefetchIssued: 0, prefetchUsed: 0, evictions: 0,
+      bytesRead: 0, readMs: 0, uploadMs: 0, waitMs: 0,
+      hitsByLayer: new Uint32Array(L), missesByLayer: new Uint32Array(L),
+    };
+    this.prefetched = new Set();  // keys loaded by prefetch() and not yet demanded
+  }
+
+  key(layer, expert) { return layer * this.numExperts + expert; }
+  resident() { return this.lru.size; }
+
+  pin(s) { this.pinCount[s]++; }
+  release(slots) { for (const s of slots) if (this.pinCount[s] > 0) this.pinCount[s]--; }
+
+  takeStaging() { return this.staging.pop() || new ArrayBuffer(this.recordBytes); }
+  giveStaging(buf) { if (this.staging.length < this.maxStaging && buf.byteLength >= this.recordBytes) this.staging.push(buf); }
+
+  heatOf(key) { return this.heat[key] * Math.pow(this.decayPerTick, this.tick - this.heatTick[key]); }
+  touch(key) { this.heat[key] = this.heatOf(key) + 1; this.heatTick[key] = this.tick; }
+
+  allocSlot() {
+    if (this.free.length) return this.free.pop();
+    if (this.evict === 'hot') {
+      let bestK = -1, bestS = -1, bestH = Infinity;
+      for (const [k, s] of this.lru) {          // LRU order, so the first minimum is the oldest
+        if (this.pinCount[s] !== 0) continue;
+        const h = this.heatOf(k);
+        if (h < bestH) { bestH = h; bestK = k; bestS = s; }
+      }
+      if (bestS >= 0) {
+        this.lru.delete(bestK); this.prefetched.delete(bestK); this.slotKey[bestS] = -1; this.stats.evictions++;
+        return bestS;
+      }
+      throw new Error(`expert pool exhausted: all ${this.slots} slots pinned — raise the pool size`);
+    }
+    for (const [k, s] of this.lru) {
+      if (this.pinCount[s] === 0) {
+        this.lru.delete(k);
+        this.prefetched.delete(k);
+        this.slotKey[s] = -1;
+        this.stats.evictions++;
+        return s;
+      }
+    }
+    throw new Error(`expert pool exhausted: all ${this.slots} slots pinned — raise the pool size`);
+  }
+
+  // Starts the read + upload of one expert into a newly allocated slot. The slot stays pinned
+  // while the load is in flight; `entry.pins` demands registered meanwhile are converted into
+  // pins in the same synchronous step that publishes the slot, so no eviction can slip
+  // between "loaded" and "pinned by its user". Returns the in-flight entry { p, pins }.
+  load(layer, expert, pins = 0) {
+    const key = this.key(layer, expert);
+    const s = this.allocSlot();
+    this.pin(s);
+    const entry = { pins, p: null };
+    entry.p = (async () => {
+      let ok = false;
+      try {
+        const r = await this.reader.read(this.recordOffset(layer, expert), this.recordBytes, this.takeStaging());
+        if (r.got !== this.recordBytes) throw new Error(`short expert read L${layer} E${expert}: ${r.got} of ${this.recordBytes} bytes`);
+        this.stats.readMs += r.ms;
+        this.stats.bytesRead += r.got;
+        const t0 = performance.now();
+        await this.upload(r.buf, s);
+        this.stats.uploadMs += performance.now() - t0;
+        this.giveStaging(r.buf);
+        this.slotKey[s] = key;
+        this.lru.set(key, s);
+        this.pinCount[s] += entry.pins;
+        ok = true;
+        return s;
+      } catch (err) {
+        if (err && err.buf) this.giveStaging(err.buf);
+        throw err;
+      } finally {
+        this.inflight.delete(key);
+        this.pinCount[s]--;
+        if (!ok) { this.pinCount[s] = 0; this.free.push(s); }
+      }
+    })();
+    this.inflight.set(key, entry);
+    return entry;
+  }
+
+  // Copies one record into slot `s` of every part. Queue order makes this as safe as
+  // writeBuffer: the copy is submitted after the work that last read the slot and before the
+  // work that will read the new expert.
+  async upload(buf, s) {
+    if (!this.ring.length) {
+      for (const part of this.parts) this.device.queue.writeBuffer(part.buffer, s * part.bytes, buf, part.srcOffset, part.bytes);
+      return;
+    }
+    const r = this.ring[this.ringNext];
+    this.ringNext = (this.ringNext + 1) % this.ring.length;
+    while (r.ready) { const p = r.ready; await p; if (r.ready === p) r.ready = null; }
+    new Uint8Array(r.buf.getMappedRange(0, r.buf.size)).set(new Uint8Array(buf, 0, this.recordBytes));
+    r.buf.unmap();
+    const enc = this.device.createCommandEncoder();
+    for (const part of this.parts) enc.copyBufferToBuffer(r.buf, part.srcOffset, part.buffer, s * part.bytes, part.bytes);
+    this.device.queue.submit([enc.finish()]);
+    r.ready = r.buf.mapAsync(2);
+  }
+
+  // Makes every expert in `ids` resident for `layer` and pins its slot. Returns the slots in
+  // the same order as `ids`. The caller must release() them after submitting the GPU work.
+  async ensure(layer, ids) {
+    this.tick++;
+    if (this.pfQueue.length) this.pfQueue = this.pfQueue.filter((q) => q.layer > layer);   // not sorted by layer once lookahead >= 3
+    const out = new Uint32Array(ids.length);
+    const waits = [];
+    const st = this.stats;
+    for (let i = 0; i < ids.length; i++) {
+      const key = this.key(layer, ids[i]);
+      this.touch(key);
+      if (this.prefetched.delete(key)) st.prefetchUsed++;
+      const s = this.lru.get(key);
+      if (s !== undefined) {
+        this.lru.delete(key); this.lru.set(key, s);  // most recently used
+        this.pin(s);
+        out[i] = s;
+        st.hits++; st.hitsByLayer[layer]++;
+        continue;
+      }
+      const flying = this.inflight.get(key);
+      if (flying) {
+        st.lateHits++; st.hitsByLayer[layer]++;
+        flying.pins++;
+        waits.push(flying.p.then((slot) => { out[i] = slot; }));
+        continue;
+      }
+      st.misses++; st.missesByLayer[layer]++;
+      waits.push(this.load(layer, ids[i], 1).p.then((slot) => { out[i] = slot; }));
+    }
+    if (waits.length) {
+      const t0 = performance.now();
+      await Promise.all(waits);
+      st.waitMs += performance.now() - t0;
+    }
+    return out;
+  }
+
+  // For an engine that routes on the GPU against a residency map: `ids` of `layer` were found
+  // resident and used without an ensure(). Refreshes their recency and counts them as hits; an
+  // expert evicted since (its slot was reused after that work was submitted) is skipped.
+  touchUsed(layer, ids) {
+    this.tick++;
+    const st = this.stats;
+    for (const e of ids) {
+      const key = this.key(layer, e);
+      const s = this.lru.get(key);
+      if (s === undefined) continue;
+      this.touch(key);
+      if (this.prefetched.delete(key)) st.prefetchUsed++;
+      this.lru.delete(key); this.lru.set(key, s);
+      st.hits++; st.hitsByLayer[layer]++;
+    }
+  }
+
+  // Queues loads for experts that are neither resident, in flight nor queued; never awaits.
+  // Call with the nearest layer first: the queue is served in order, and entries for a layer
+  // are dropped once that layer's demand (ensure) arrives. Returns how many loads it started.
+  prefetch(layer, ids) {
+    const before = this.stats.prefetchIssued;
+    for (const e of ids) {
+      const key = this.key(layer, e);
+      if (this.lru.has(key) || this.inflight.has(key) || this.pfQueue.some((q) => q.key === key)) continue;
+      this.pfQueue.push({ layer, e, key });
+    }
+    this.pump();
+    return this.stats.prefetchIssued - before;
+  }
+
+  // Starts queued prefetches while fewer than maxPrefetchInflight are reading. Stops when the
+  // pool has nothing evictable left.
+  pump() {
+    while (this.pfInflight < this.maxPrefetchInflight && this.pfQueue.length) {
+      const { layer, e, key } = this.pfQueue.shift();
+      if (this.lru.has(key) || this.inflight.has(key)) continue;
+      let entry;
+      try { entry = this.load(layer, e); } catch (_) { this.pfQueue.length = 0; break; }
+      this.prefetched.add(key);
+      this.stats.prefetchIssued++;
+      this.pfInflight++;
+      entry.p.catch(() => {}).finally(() => { this.pfInflight--; this.pump(); });
+    }
+  }
+
+  // Waits for every in-flight load and staging-ring map (before teardown and between runs).
+  async drain() {
+    await Promise.allSettled([...this.inflight.values()].map((f) => f.p));
+    await Promise.allSettled(this.ring.map((r) => r.ready).filter(Boolean));
+  }
+  ringBytes() { return this.ring.reduce((a, r) => a + r.buf.size, 0); }
+
+  // Empties the pool. Waits for loads still reading first: a load that finished after the
+  // reset would claim a slot that is back on the free list.
+  async clear() {
+    this.pfQueue = [];
+    await this.drain();
+    this.lru.clear();
+    this.inflight.clear();
+    this.prefetched.clear();
+    this.slotKey.fill(-1);
+    this.pinCount.fill(0);
+    this.free = [];
+    for (let s = this.slots - 1; s >= 0; s--) this.free.push(s);
+  }
+
+}

package/src/gguf.js

@@ -0,0 +1,96 @@
+/* gguf.js — read a GGUF header and split quantized blocks into GPU-friendly planes. No dependencies.
+ * Extracted unchanged from LocalMind's qwen3_moe_ssd.js (2026-10-05).
+ *
+ *   const { kv, tensors, dataStart } = parseGguf(headerBytes);   // throws { needBytes } if too short
+ *   tensorBytes(tensor)                                            // bytes of one tensor in the file
+ *   splitQ8(src, q, s) / splitQ4(src, q, s)                        // blocks → value plane + f16-scale plane, bit-exact
+ */
+
+export const GGML = { F32: 0, F16: 1, Q4_0: 2, Q8_0: 8, Q6_K: 14 };
+export const Q8_BLOCK = 34;  // f16 scale + 32 × int8
+export const Q4_BLOCK = 18;  // f16 scale + 32 × 4-bit (16 bytes)
+export const Q6K_BLOCK = 210; // 256 values: ql[128] + qh[64] + scales[16] + f16 d
+
+// Parses a GGUF header. Throws { needBytes } when `u8` stops before the header ends.
+export function parseGguf(u8) {
+  const dv = new DataView(u8.buffer, u8.byteOffset, u8.byteLength);
+  let p = 0;
+  const need = (n) => { if (p + n > u8.byteLength) { const e = new Error('GGUF header truncated'); e.needBytes = Math.max(u8.byteLength * 2, p + n + (1 << 20)); throw e; } };
+  const u32 = () => { need(4); const v = dv.getUint32(p, true); p += 4; return v; };
+  const u64 = () => { need(8); const v = Number(dv.getBigUint64(p, true)); p += 8; return v; };
+  // ignoreBOM: keep a leading U+FEFF (Gemma 4's vocab has '#', '//' and '<?' twice, once with a byte-order mark).
+  const dec = new TextDecoder('utf-8', { ignoreBOM: true });
+  const str = () => { const n = u64(); need(n); const s = dec.decode(u8.subarray(p, p + n)); p += n; return s; };
+  const scalar = {
+    0: () => { need(1); return dv.getUint8(p++); }, 1: () => { need(1); return dv.getInt8(p++); },
+    2: () => { need(2); const v = dv.getUint16(p, true); p += 2; return v; }, 3: () => { need(2); const v = dv.getInt16(p, true); p += 2; return v; },
+    4: u32, 5: () => { need(4); const v = dv.getInt32(p, true); p += 4; return v; },
+    6: () => { need(4); const v = dv.getFloat32(p, true); p += 4; return v; },
+    7: () => { need(1); return dv.getUint8(p++) !== 0; },
+    10: u64, 11: () => { need(8); const v = Number(dv.getBigInt64(p, true)); p += 8; return v; },
+    12: () => { need(8); const v = dv.getFloat64(p, true); p += 8; return v; },
+  };
+  const value = (t) => {
+    if (t === 8) return str();
+    if (t === 9) { const at = u32(), n = u64(); const a = new Array(n); for (let i = 0; i < n; i++) a[i] = value(at); return a; }
+    const f = scalar[t];
+    if (!f) throw new Error(`GGUF: unknown value type ${t}`);
+    return f();
+  };
+  need(4);
+  if (dec.decode(u8.subarray(0, 4)) !== 'GGUF') throw new Error('not a GGUF file');
+  p = 4;
+  const version = u32(), nTensors = u64(), nKv = u64();
+  const kv = {};
+  for (let i = 0; i < nKv; i++) { const k = str(); kv[k] = value(u32()); }
+  const tensors = [];
+  for (let i = 0; i < nTensors; i++) {
+    const name = str(), nd = u32(), dims = [];
+    for (let d = 0; d < nd; d++) dims.push(u64());
+    const type = u32(), offset = u64();
+    tensors.push({ name, dims, type, offset });
+  }
+  const align = kv['general.alignment'] || 32;
+  const dataStart = Math.ceil(p / align) * align;
+  return { version, kv, tensors, headerBytes: p, dataStart };
+}
+
+export function tensorBytes(t) {
+  const n = t.dims.reduce((a, b) => a * b, 1);
+  if (t.type === GGML.F32) return n * 4;
+  if (t.type === GGML.F16) return n * 2;
+  if (t.type === GGML.Q8_0) return (n / 32) * Q8_BLOCK;
+  if (t.type === GGML.Q4_0) return (n / 32) * Q4_BLOCK;
+  if (t.type === GGML.Q6_K) return (n / 256) * Q6K_BLOCK;
+  throw new Error(`unsupported tensor type ${t.type} (${t.name})`);
+}
+
+// Q8_0 blocks → an int8 plane + an f16-scale plane, bit-exact. `src` holds whole blocks and
+// starts at an even byte offset, so both planes copy as u16 lanes.
+export function splitQ8(src, q, s) {
+  const nb = src.byteLength / Q8_BLOCK;
+  const a = new Uint16Array(src.buffer, src.byteOffset, nb * 17);
+  const qd = new Uint16Array(q.buffer, q.byteOffset, nb * 16);
+  const sd = new Uint16Array(s.buffer, s.byteOffset, nb);
+  for (let b = 0, i = 0, o = 0; b < nb; b++, i += 17, o += 16) {
+    sd[b] = a[i];
+    qd[o] = a[i + 1]; qd[o + 1] = a[i + 2]; qd[o + 2] = a[i + 3]; qd[o + 3] = a[i + 4];
+    qd[o + 4] = a[i + 5]; qd[o + 5] = a[i + 6]; qd[o + 6] = a[i + 7]; qd[o + 7] = a[i + 8];
+    qd[o + 8] = a[i + 9]; qd[o + 9] = a[i + 10]; qd[o + 10] = a[i + 11]; qd[o + 11] = a[i + 12];
+    qd[o + 12] = a[i + 13]; qd[o + 13] = a[i + 14]; qd[o + 14] = a[i + 15]; qd[o + 15] = a[i + 16];
+  }
+}
+
+// Q4_0 blocks → a nibble plane (16 bytes per 32 values, ggml's order: low nibbles are values 0..15,
+// high nibbles 16..31) + an f16-scale plane, bit-exact.
+export function splitQ4(src, q, s) {
+  const nb = src.byteLength / Q4_BLOCK;
+  const a = new Uint16Array(src.buffer, src.byteOffset, nb * 9);
+  const qd = new Uint16Array(q.buffer, q.byteOffset, nb * 8);
+  const sd = new Uint16Array(s.buffer, s.byteOffset, nb);
+  for (let b = 0, i = 0, o = 0; b < nb; b++, i += 9, o += 8) {
+    sd[b] = a[i];
+    qd[o] = a[i + 1]; qd[o + 1] = a[i + 2]; qd[o + 2] = a[i + 3]; qd[o + 3] = a[i + 4];
+    qd[o + 4] = a[i + 5]; qd[o + 5] = a[i + 6]; qd[o + 6] = a[i + 7]; qd[o + 7] = a[i + 8];
+  }
+}

package/src/ingest.js

@@ -0,0 +1,177 @@
+/* ingest.js — copy a GGUF from a URL into OPFS in an engine's own layout, streaming: HTTP Range requests of
+ * ≤1 GiB, each unit transformed and written while the next bytes arrive (at most 8 writes in flight). OPFS quota
+ * grows with what is written, so files grow by writes rather than one up-front truncate. The manifest is written
+ * incomplete first and complete last, so an interrupted ingest is never taken as valid. After each range request the
+ * manifest records the first unit not yet written; a later call with the same key and the same GGUF header resumes
+ * from there instead of from byte 0.
+ * Extracted from LocalMind's qwen3_moe_ssd.js (2026-10-05); `root` and `format` are new options (LocalMind
+ * passes 'localmind-ssd' and its own format string).
+ *
+ *   const manifest = await ingestGguf({ url, key, plan: { layout, units }, onProgress });
+ *   // OPFS: <root>/<key>/{header.bin, dense.bin, experts.bin, manifest.json}
+ */
+
+import { OpfsWriter, readOpfsText, writeOpfsText, removeOpfs } from './opfs-reader.js';
+import { parseGguf, splitQ8, splitQ4, Q8_BLOCK, Q4_BLOCK } from './gguf.js';
+
+export const ROOT = 'diskformer';
+export const FORMAT = 'diskformer-gguf/1';
+
+// ── Ingest: GGUF (over HTTP Range) → OPFS, in the engine layout ─────────────
+export const storeKey = (file) => file.replace(/\.gguf$/i, '').toLowerCase().replace(/[^a-z0-9._-]+/g, '-');
+
+export async function readHeader(url, fetchFn, signal) {
+  let n = 16 << 20;
+  for (;;) {
+    const r = await fetchFn(url, { headers: { Range: `bytes=0-${n - 1}` }, signal, cache: 'no-store' });
+    if (!r.ok) throw new Error(`GGUF header fetch: HTTP ${r.status}`);
+    const h = (name) => (r.headers && r.headers.get ? r.headers.get(name) : null);
+    const total = /\/(\d+)\s*$/.exec(h('content-range') || '');
+    const file = { size: total ? Number(total[1]) : null, etag: h('etag') };   // for the resume identity
+    const u8 = new Uint8Array(await r.arrayBuffer());
+    try { return { gguf: parseGguf(u8), bytes: u8, file }; } catch (e) { if (!e.needBytes) throw e; n = e.needBytes; }
+  }
+}
+
+// FNV-1a, for the resume identity: over bytes (the GGUF header) or a string (the plan's output layout).
+const fnv = (at, n) => { let h = 0x811c9dc5; for (let i = 0; i < n; i++) h = Math.imul(h ^ at(i), 0x01000193) >>> 0; return `${n}:${h.toString(16)}`; };
+const bytesId = (u8) => fnv((i) => u8[i], u8.length);
+const textId = (s) => fnv((i) => s.charCodeAt(i), s.length);
+
+// plan.layout(gguf) → { experts: { file, bytes, … }, dense: { file, bytes, tensors } }: where every kept byte goes.
+// plan.units(gguf, layout) → sorted units { src, len, file: 'dense' | 'experts', raw } or { …, kind?: 'q4', q, s }:
+// byte ranges of the GGUF, each small enough to transform in memory (raw copies, or Q8_0 / Q4_0 blocks split
+// into a value plane at q and an f16-scale plane at s).
+// An interrupted ingest resumes only when the GGUF header, the plan's output layout, and the source's size and ETag
+// (when the server sends them) match its checkpoint; otherwise it starts over. `rangeBytes` and `io` are for tests:
+// smaller range requests, and fake OPFS files.
+export async function ingestGguf({ url, key, plan, root = ROOT, format = FORMAT, fetch: fetchFn = fetch, onProgress = () => {}, signal, source = {},
+  rangeBytes = 1 << 30, io = { OpfsWriter, readOpfsText, writeOpfsText } }) {
+  if (!plan || !plan.layout || !plan.units) throw new Error('ingestGguf: plan { layout, units } is required');
+  const t0 = performance.now();
+  const { gguf, bytes: headerBuf, file } = await readHeader(url, fetchFn, signal);
+  const layout = plan.layout(gguf);
+  const dir = `${root}/${key}`;
+  const storage = typeof navigator !== 'undefined' && navigator.storage;
+  const est = storage && storage.estimate ? await storage.estimate() : {};
+  const persisted = storage && storage.persist ? await storage.persist().catch(() => false) : false;
+  const needed = layout.experts.bytes + layout.dense.bytes;
+  onProgress({ status: 'ingest-plan', needed, quota: est.quota, usage: est.usage, persisted });
+  const quotaLog = [{ written: 0, quota: est.quota, usage: est.usage }];
+  const units = plan.units(gguf, layout);
+  const identity = [bytesId(headerBuf.subarray(0, gguf.dataStart)), textId(JSON.stringify([layout, units])), file.size ?? '', file.etag ?? ''].join('|');
+  let prior = null;
+  try { prior = JSON.parse(await io.readOpfsText(`${dir}/manifest.json`) || 'null'); } catch (_) { prior = null; }
+  const r0 = prior && !prior.complete && prior.format === format && prior.resume;
+  const startUnit = r0 && r0.identity === identity && r0.unit > 0 && r0.unit < units.length ? r0.unit : 0;
+  const checkpoint = (unit) => io.writeOpfsText(`${dir}/manifest.json`, JSON.stringify({ format, complete: false, resume: { identity, unit } }));
+
+  // Stream the data section in order. Units are filled from the response chunks, transformed,
+  // and written while the next bytes arrive (at most `maxWrites` writes in flight).
+  const first = units[0].src, last = units[units.length - 1].src + units[units.length - 1].len;
+  const resumedFrom = units[startUnit].src - first;
+  const maxUnit = units.reduce((m, u) => Math.max(m, u.len), 0);
+  let ui = startUnit, fill = 0, unitBuf = new Uint8Array(maxUnit);
+  const writers = {};
+  const release = async (name) => { const w = writers[name]; delete writers[name]; await w.close(); };
+  const inflight = new Set(); const maxWrites = 8;
+  const spare = [];
+  const getBuf = (n) => { const i = spare.findIndex((b) => b.byteLength >= n); return i >= 0 ? spare.splice(i, 1)[0] : new ArrayBuffer(n); };
+  const queue = async (w, buf, len, at) => {
+    while (inflight.size >= maxWrites) await Promise.race(inflight);
+    const p = w.writeOwned(buf, len, at).then((b) => { inflight.delete(p); if (spare.length < 16) spare.push(b); });
+    p.catch(() => {});       // a failed write surfaces through the race above or the final Promise.all
+    inflight.add(p);
+  };
+  let done = resumedFrom, written = 0;
+  const flushUnit = async (u) => {
+    const src = unitBuf.subarray(0, u.len);
+    const w = writers[u.file];
+    if (u.raw !== undefined) {
+      const b = getBuf(u.len); new Uint8Array(b, 0, u.len).set(src);
+      await queue(w, b, u.len, u.raw); written += u.len; return;
+    }
+    const q4 = u.kind === 'q4', blk = q4 ? Q4_BLOCK : Q8_BLOCK, qPer = q4 ? 16 : 32;
+    const nb = u.len / blk;
+    const qb = getBuf(nb * qPer), sb = getBuf(nb * 2);
+    (q4 ? splitQ4 : splitQ8)(src, new Uint8Array(qb, 0, nb * qPer), new Uint8Array(sb, 0, nb * 2));
+    await queue(w, qb, nb * qPer, u.q);
+    await queue(w, sb, nb * 2, u.s);
+    written += nb * blk;
+  };
+  try {
+    if (!startUnit) {
+      await checkpoint(0);
+      // Header bytes, for the tokenizer and metadata at every later load.
+      writers.header = await io.OpfsWriter.open(`${dir}/header.bin`, { truncate: true });
+      await writers.header.write(headerBuf.subarray(0, gguf.dataStart), 0);
+      await release('header');
+    } else {
+      onProgress({ status: 'ingest-resume', unit: startUnit, units: units.length, loaded: resumedFrom, total: last - first });
+    }
+    // No up-front truncate to full size: a fresh origin's quota is ~10 GiB and grows with what is
+    // actually written (measured 2026-10-02), so a single 30.8 GB extension would be refused. The
+    // writes below grow each file by at most one layer's records (~642 MB) at a time.
+    writers.dense = await io.OpfsWriter.open(`${dir}/dense.bin`, { truncate: !startUnit });
+    writers.experts = await io.OpfsWriter.open(`${dir}/experts.bin`, { truncate: !startUnit });
+    // Sequential range requests of rangeBytes (1 GiB) keep any one response bounded.
+    let pos = units[startUnit].src;
+    for (let start = pos; start < last; start += rangeBytes) {
+      const end = Math.min(last, start + rangeBytes) - 1;
+      const r = await fetchFn(url, { headers: { Range: `bytes=${start}-${end}` }, signal, cache: 'no-store' });
+      if (r.status !== 206) throw new Error(`GGUF range ${start}-${end}: HTTP ${r.status}`);
+      const reader = r.body.getReader();
+      for (;;) {
+        const { done: eof, value } = await reader.read();
+        if (eof) break;
+        let off = 0;
+        while (off < value.byteLength && ui < units.length) {
+          const u = units[ui];
+          if (pos < u.src) { const skip = Math.min(u.src - pos, value.byteLength - off); pos += skip; off += skip; continue; }
+          const take = Math.min(u.len - fill, value.byteLength - off);
+          unitBuf.set(value.subarray(off, off + take), fill);
+          fill += take; off += take; pos += take;
+          if (fill === u.len) { await flushUnit(u); ui++; fill = 0; }
+        }
+        done = pos - first;
+        onProgress({ status: 'ingest', loaded: done, total: last - first, resumedFrom, written, secs: (performance.now() - t0) / 1000 });
+        if (written - quotaLog[quotaLog.length - 1].written > 4 * 2 ** 30 && storage && storage.estimate) {
+          const e = await storage.estimate();
+          quotaLog.push({ written, quota: e.quota, usage: e.usage });
+        }
+      }
+      // Every unit before ui is written: flush, then record ui as the resume point.
+      if (ui < units.length) {
+        await Promise.all(inflight);
+        await writers.dense.flush(); await writers.experts.flush();
+        await checkpoint(ui);
+      }
+    }
+    if (ui !== units.length) throw new Error(`ingest ended early: ${ui} of ${units.length} units`);
+    await Promise.all(inflight);
+    await release('dense');
+    await release('experts');
+  } catch (e) {
+    // Release every file still open, so a retry in this page can reopen them and resume.
+    await Promise.allSettled([...inflight]);
+    await Promise.allSettled(Object.keys(writers).map(release));
+    throw e;
+  }
+  const manifest = {
+    format, complete: true, ingestedAt: new Date().toISOString(),
+    source: { url, ...source, dataStart: gguf.dataStart, headerBytes: gguf.headerBytes },
+    ingestSecs: (performance.now() - t0) / 1000, resumedAtUnit: startUnit || undefined, quotaBefore: est.quota, persisted, quotaLog,
+    ...layout,
+  };
+  await io.writeOpfsText(`${dir}/manifest.json`, JSON.stringify(manifest));
+  return manifest;
+}
+
+// Ingest progress as hosts read it: a 'weights' event with byte counts. The spread goes first, so the
+// event's own status ('ingest') cannot overwrite 'weights' (LocalMind's 2026-10-04 bug: a host's load
+// watchdog, seeing no progress, killed a 37 GB first download).
+export function ingestProgress(e) {
+  return e && e.status === 'ingest' ? { ...e, status: 'weights', loaded: e.loaded, total: e.total } : e;
+}
+
+export async function removeStore(key, { root = ROOT } = {}) { await removeOpfs(`${root}/${key}`); }

package/src/measure.js

@@ -0,0 +1,61 @@
+/* measure.js — how fast this machine reads an OPFS file the way a disk tier does. Run it where the tier will run
+ * (a page or a worker); OPFS speed varies a lot between machines and browsers.
+ *
+ *   const r = await measure();                         // writes a 256 MB probe, measures, removes it
+ *   const r = await measure({ path: 'diskformer/<key>/experts.bin', recordBytes: 3345408 });   // an existing store
+ *   // → { bytes, sequentialMBps, random: { recordBytes, reads, p50Ms, p90Ms, MBps }, burst8: { p50Ms, MBps } }
+ *
+ * A probe the call just wrote is likely in the OS page cache, so its numbers are an upper bound; point `path` at a
+ * store larger than RAM to see the disk itself.
+ */
+import { OpfsReaderPool, OpfsWriter, removeOpfs } from './opfs-reader.js';
+
+const pct = (a, q) => { const s = [...a].sort((x, y) => x - y); return s[Math.min(s.length - 1, Math.floor(q * s.length))]; };
+
+export async function measure({ path = null, bytes = 256 << 20, recordBytes = 3 << 20, reads = 64, workers = 4, chunkBytes = 64 << 20 } = {}) {
+  const probe = !path;
+  if (probe) {
+    path = 'diskformer/.measure/probe.bin';
+    const w = await OpfsWriter.open(path, { truncate: true });
+    const chunk = new Uint8Array(Math.min(chunkBytes, bytes));
+    for (let i = 0; i < chunk.length; i += 4096) chunk[i] = i & 255;
+    for (let at = 0; at < bytes; at += chunk.length) await w.write(chunk.subarray(0, Math.min(chunk.length, bytes - at)), at);
+    await w.close();
+  }
+  const pool = await OpfsReaderPool.open(path, { workers });
+  try {
+    const size = pool.size;
+    // Sequential: the load pattern (64 MB chunks, one at a time per worker).
+    let t0 = performance.now(), done = 0, inflight = [];
+    for (let at = 0; at < size; at += chunkBytes) {
+      inflight.push(pool.read(at, Math.min(chunkBytes, size - at)).then((r) => { done += r.got; }));
+      if (inflight.length >= workers) { await Promise.all(inflight); inflight = []; }
+    }
+    await Promise.all(inflight);
+    const sequentialMBps = done / 1e6 / ((performance.now() - t0) / 1000);
+    // Random whole records: the per-token pattern, one at a time, then bursts of 8 (one MoE layer's misses).
+    const n = Math.max(1, Math.floor(size / recordBytes));
+    const at = () => Math.floor(Math.random() * n) * recordBytes;
+    // Throughput comes from the total time: browsers coarsen performance.now() (0.1 ms in a worker without
+    // cross-origin isolation), which rounds a single small read to zero.
+    const one = []; let t1 = performance.now();
+    for (let i = 0; i < reads; i++) { const s = performance.now(); await pool.read(at(), recordBytes); one.push(performance.now() - s); }
+    const oneMs = performance.now() - t1;
+    const burst = [], bursts = Math.max(1, Math.round(reads / 8)); t1 = performance.now();
+    for (let i = 0; i < bursts; i++) {
+      const s = performance.now();
+      await Promise.all(Array.from({ length: 8 }, () => pool.read(at(), recordBytes)));
+      burst.push(performance.now() - s);
+    }
+    const burstMs = performance.now() - t1;
+    const mbps = (count, ms) => Math.round((count * recordBytes) / 1e6 / (Math.max(ms, 0.1) / 1000));
+    return {
+      path, bytes: size, workers, sequentialMBps: Math.round(sequentialMBps),
+      random: { recordBytes, reads, p50Ms: +pct(one, 0.5).toFixed(2), p90Ms: +pct(one, 0.9).toFixed(2), MBps: mbps(reads, oneMs) },
+      burst8: { bursts, p50Ms: +pct(burst, 0.5).toFixed(2), MBps: mbps(8 * bursts, burstMs) },
+    };
+  } finally {
+    await pool.close();
+    if (probe) await removeOpfs('diskformer/.measure').catch(() => {});
+  }
+}