@threadedstack/wmd-web 0.0.0-stage → 0.1.12

package/src/fs/upper-pack.js

@@ -0,0 +1,112 @@
+// W62: the writable layer of an Overlay as one pack layer, what `docker commit` keeps: every
+// file the guest made or changed (contents, mode, owner, times, xattrs, devices, links), a
+// whiteout for every lower name it removed, opaque for a directory made over a removed lower
+// one. Booted again with this pack on top of the same image layers (bootWmd's layers, top
+// first), the guest sees the tree it left.
+//
+// The Overlay keeps lower and upper in one inode tree, so the layer is found by walking the
+// directories the guest has read (kids loaded; an unread directory has no changes below it):
+// an inode with `upper` set is written whole, a directory that holds a change is written so
+// the readers can reach it (a pack lists only entries under directory entries of its own).
+// A directory that moved (its lower contents now show under another name: rename of a lower
+// directory) is written opaque with everything under it, since a pack has no redirects.
+//
+// A file the guest still has mapped writable may have changes in the guest's page cache
+// that 9P has not seen: run `sync` in the guest first (cache=mmap writes the rest at once).
+
+import { S_IFMT, S_IFREG } from './pack.js'
+import { writePack } from './pack-write.js'
+
+/**
+ * The upper of fs below `at` ('' the root; 'rootfs' for the guest init's layout, where the
+ * layers show) as pack bytes, and what it holds.
+ * @returns {{bytes: Uint8Array, entries: number, files: number, whiteouts: number, opaque: number, dataBytes: number}}
+ */
+export function exportUpper(fs, { at = '' } = {}) {
+  const top = at ? fs.resolve(at, false) : fs.root
+  if (!top.isDir()) throw new Error(`exportUpper: ${at} is not a directory`)
+  const out = []
+  const links = new Map() // inode -> link id, for a regular file with more than one name written
+  let nextLink = 1
+  const stats = { files: 0, whiteouts: 0, opaque: 0, dataBytes: 0 }
+
+  const entryOf = (n, path, opaque) => {
+    const e = {
+      path, mode: n.mode, uid: n.uid, gid: n.gid, rdev: n.rdev,
+      atime: n.atime, mtime: n.mtime, ctime: n.ctime, target: n.target || '',
+      xattrs: n.xattrs && n.xattrs.size ? n.xattrs : null, opaque: !!opaque,
+      size: 0, link: 0,
+    }
+    if ((n.mode & S_IFMT) === S_IFREG) {
+      e.size = fs.size(n)
+      if (n.nlink > 1) {
+        let id = links.get(n)
+        if (id) e.link = id
+        else {
+          links.set(n, (id = nextLink++))
+          e.link = id
+          e.data = (off, len) => fs.read(n, off, len)
+          stats.dataBytes += e.size
+        }
+      } else {
+        e.data = (off, len) => fs.read(n, off, len)
+        stats.dataBytes += e.size
+      }
+      stats.files++
+    }
+    if (opaque) stats.opaque++
+    return e
+  }
+
+  // Whether a lower layer shows something at path (a whiteout there hides it).
+  const lowerHas = path => {
+    for (const p of fs.layers) {
+      const i = p.index(path)
+      if (i >= 0) return !p.entry(i).whiteout
+    }
+    // (an entry hidden by a parent's whiteout or opaque directory counts too: asked only of a
+    // new directory, it makes it opaque, which hides nothing the guest sees)
+    return false
+  }
+
+  // Writes what changed in directory n at path; whole: everything under it (a moved
+  // directory's subtree). Returns whether anything was written.
+  const visit = (n, path, whole) => {
+    let wrote = false
+    if (n.isDir()) {
+      const moved = !whole && n.lowerDirs && n.lowerDirs.some(d => d.path !== path)
+      const allIn = whole || moved
+      const kids = allIn ? fs.list(n) : n.kids ? [...n.kids] : []
+      const before = out.length
+      if (!allIn && n.whiteouts) {
+        for (const name of n.whiteouts) {
+          if (n.kids && n.kids.has(name)) continue
+          out.push({ path: join(path, name), whiteout: true })
+          stats.whiteouts++
+        }
+      }
+      for (const [name, c] of kids) visit(c, join(path, name), allIn)
+      const inside = out.length > before
+      // a new directory at a lower directory's name hides it (#insert set opaque for one made
+      // there; one renamed there from the upper did not, and in a pack it would merge)
+      const opaque = allIn || n.opaque || (n.upper && n.lowerDirs && n.lowerDirs.length === 0 && path !== '' && lowerHas(path))
+      if (whole || n.upper || inside || opaque) {
+        out.push(entryOf(n, path, allIn || opaque))
+        wrote = true
+      }
+      return wrote || inside
+    }
+    if (whole || n.upper) {
+      out.push(entryOf(n, path, false))
+      return true
+    }
+    return false
+  }
+
+  // the root is written whenever anything under it is (with its current attributes)
+  visit(top, '', false)
+  const bytes = writePack(out)
+  return { bytes, entries: out.length, ...stats }
+}
+
+const join = (dir, name) => (dir === '' ? name : `${dir}/${name}`)

package/src/host/base-cache.js

@@ -0,0 +1,195 @@
+// P-B6: where a downloaded base snapshot is kept so the next page load does not fetch it again.
+//
+//   opfsBaseCache()   OPFS directory `wmd-base`, one file per snapshot (the file as downloaded: codec and all), written by
+//                     base-io.js; read back with getFile().stream(). The file is valid only once its magic is in (written last).
+//   memoryBaseCache() the same interface over memory (tests, a page without OPFS)
+//
+// open(name) -> {stream, size} | null        null when there is no complete file (a partial one is deleted)
+// writer(name) -> {write(u8), commit(), abort()}   the tee of a download: pieces in file order; write() applies backpressure
+// A cache is an optimisation: every failure is the caller's to log and ignore (the snapshot is verified again on every read).
+import { BASE_MAGIC } from './base-file.js'
+
+const MAGIC = new TextEncoder().encode(BASE_MAGIC)
+const FLUSH = 4 << 20
+const MAX_INFLIGHT = 64 << 20
+
+/** Wraps a raw `put(at, u8)` (async) into the writer: zeroes the first 8 bytes until commit, batches, bounds what is in flight. */
+export function makeWriter({ put, commit, abort }) {
+  let at = 0
+  let pending = []
+  let pendingBytes = 0
+  let inflight = 0
+  const waiters = []
+  let failed = null
+  let first = true
+  const ship = async () => {
+    if (!pendingBytes) return
+    const buf = new Uint8Array(pendingBytes)
+    let o = 0
+    for (const p of pending) (buf.set(p, o), (o += p.length))
+    pending = []
+    pendingBytes = 0
+    if (first) {
+      if (buf.length < 8 || MAGIC.some((b, i) => buf[i] !== b)) throw new Error('not a base snapshot: it is not cached')
+      buf.fill(0, 0, 8) // the magic goes in at commit
+      first = false
+    }
+    const here = at
+    const n = buf.length // read before put(): the buffer is transferred and its length reads 0 afterwards
+    at += n
+    inflight += n
+    put(here, buf).then(
+      () => {
+        inflight -= n
+        waiters.splice(0).forEach(w => w())
+      },
+      e => {
+        failed = e
+        inflight -= n
+        waiters.splice(0).forEach(w => w())
+      },
+    )
+    while (inflight > MAX_INFLIGHT && !failed) await new Promise(r => waiters.push(r))
+  }
+  const settle = async () => {
+    await ship()
+    while (inflight > 0 && !failed) await new Promise(r => waiters.push(r))
+    if (failed) throw failed
+  }
+  return {
+    async write(u8) {
+      if (failed) throw failed
+      pending.push(u8)
+      pendingBytes += u8.length
+      if (pendingBytes >= FLUSH) await ship()
+    },
+    async commit() {
+      await settle()
+      await commit(at)
+      return { bytes: at }
+    },
+    async abort() {
+      pending = []
+      await abort().catch(() => {})
+    },
+    get bytes() {
+      return at + pendingBytes
+    },
+  }
+}
+
+export function opfsBaseCache({ workerUrl, dirName = 'wmd-base' } = {}) {
+  let w = null
+  let next = 1
+  const pending = new Map()
+  const call = (msg, transfer = []) => {
+    if (!w) {
+      // a literal URL when no workerUrl is given, so bundlers bundle the Worker (2.15)
+      w = workerUrl
+        ? new Worker(workerUrl, { type: 'module', name: 'wmd-base-io' })
+        : new Worker(new URL('./base-io.js', import.meta.url), { type: 'module', name: 'wmd-base-io' })
+      w.onmessage = e => {
+        const p = pending.get(e.data.id)
+        if (!p) return
+        pending.delete(e.data.id)
+        if (e.data.ok) p.resolve(e.data)
+        else p.reject(Object.assign(new Error(e.data.error), { code: e.data.code }, e.data.code === 'quota' ? { name: 'QuotaExceededError' } : {}))
+      }
+      w.onerror = err => {
+        for (const p of pending.values()) p.reject(new Error(`the cache Worker failed: ${err.message || err}`))
+        pending.clear()
+      }
+    }
+    return new Promise((resolve, reject) => {
+      const id = next++
+      pending.set(id, { resolve, reject })
+      w.postMessage({ ...msg, id }, transfer)
+    })
+  }
+  const dir = async () => (await navigator.storage.getDirectory()).getDirectoryHandle(dirName, { create: true })
+  return {
+    kind: 'opfs',
+    async open(name) {
+      let file
+      try {
+        file = await (await (await dir()).getFileHandle(name)).getFile()
+      } catch (e) {
+        if (e.name === 'NotFoundError') return null
+        throw e
+      }
+      const head = new Uint8Array(await file.slice(0, 8).arrayBuffer())
+      if (file.size < 16 || MAGIC.some((b, i) => head[i] !== b)) {
+        await call({ op: 'delete', name }).catch(() => {}) // an unfinished copy: not usable, not kept
+        return null
+      }
+      return { stream: file.stream(), size: file.size }
+    },
+    async writer(name) {
+      await call({ op: 'open', name })
+      return makeWriter({
+        put: (at, buf) => call({ op: 'write', name, at, buf: buf.buffer }, [buf.buffer]),
+        commit: () => call({ op: 'commit', name, magic: MAGIC.slice().buffer }, []),
+        abort: () => call({ op: 'abort', name }),
+      })
+    },
+    async list() {
+      return (await call({ op: 'list' })).items
+    },
+    async delete(name) {
+      await call({ op: 'delete', name })
+    },
+    async clear() {
+      await call({ op: 'clear' })
+    },
+    close() {
+      w?.terminate()
+      w = null
+    },
+  }
+}
+
+export function memoryBaseCache() {
+  const files = new Map() // name -> {buf: Uint8Array, done}
+  return {
+    kind: 'memory',
+    files,
+    async open(name) {
+      const f = files.get(name)
+      if (!f || !f.done) {
+        files.delete(name)
+        return null
+      }
+      const b = f.buf
+      return { stream: new ReadableStream({ start(c) { c.enqueue(b.slice()); c.close() } }), size: b.length }
+    },
+    async writer(name) {
+      const f = { buf: new Uint8Array(0), done: false }
+      files.set(name, f)
+      return makeWriter({
+        put: async (at, u8) => {
+          if (at + u8.length > f.buf.length) {
+            const nb = new Uint8Array(at + u8.length)
+            nb.set(f.buf)
+            f.buf = nb
+          }
+          f.buf.set(u8, at)
+        },
+        commit: async () => {
+          f.buf.set(MAGIC, 0)
+          f.done = true
+        },
+        abort: async () => void files.delete(name),
+      })
+    },
+    async list() {
+      return [...files].map(([name, f]) => ({ name, size: f.buf.length }))
+    },
+    async delete(name) {
+      files.delete(name)
+    },
+    async clear() {
+      files.clear()
+    },
+    close() {},
+  }
+}

package/src/host/base-file.js

@@ -0,0 +1,328 @@
+// P-B6: the base snapshot file, a checkpoint made to be downloaded (a static file next to an image's packs).
+//
+//   [0, 8)        magic "WMDBASE1" (an OPFS copy is written with these 8 bytes zero and the magic put in last)
+//   [8, 12)       u32 LE format version
+//   [12, 16)      u32 LE length n of the header JSON
+//   [16, 16+n)    header JSON (see below), then its SHA-256 (32 bytes)
+//   then          the chunks, back to back, in the order of header.sections, each `clen` bytes (the codec's output)
+//
+// header JSON: the checkpoint's header (vmKey, bootKey, bootDesc, memPages, jit, svcPorts, stats, ...) plus
+//   {base: 1, image, codec: 'none'|'gzip', createdAt, dataLen,
+//    sections: [{name, chunks: [{len, clen, sha256}]}]}
+// `len` is the raw size of a chunk, `clen` the size in the file, `sha256` the digest of the RAW bytes: a reader checks the
+// header against its own SHA-256 (and, when it has it, the manifest's), then every chunk after decoding. A file cut short is
+// refused as soon as the stream ends early (dataLen is in the header), a flipped byte by its chunk's digest or the decoder.
+//
+// The reader works on a stream, so a page decodes and verifies while the rest downloads; chunks are small (8 MiB by default)
+// so decoding overlaps the transfer. codec 'gzip' uses DecompressionStream (all three browsers); a host that serves the file
+// brotli-compressed (Content-Encoding: br) uses codec 'none': the browser decodes that layer and the digests still apply.
+import { CheckpointError, concat, sha256Hex, sha256Sync } from './ckpt-file.js'
+
+export const BASE_MAGIC = 'WMDBASE1'
+export const BASE_VERSION = 1
+export const BASE_MAX_HEADER = 4 << 20
+export const BASE_CHUNK = 8 << 20
+const enc = new TextEncoder()
+const dec = new TextDecoder()
+const hex = u8 => [...u8].map(b => b.toString(16).padStart(2, '0')).join('')
+
+/** A checkpoint's chunks split at `limit` bytes (a multiple of the 64 KiB page, so memory chunks stay whole pages). */
+function split(c, limit) {
+  if (c.length <= limit) return [c]
+  const out = []
+  for (let o = 0; o < c.length; o += limit) out.push(c.subarray(o, Math.min(c.length, o + limit)))
+  return out
+}
+
+/**
+ * Encodes a checkpoint ({header, sections}) as a base file. `compress(u8) -> Promise<Uint8Array>` is the codec's encoder
+ * (omit with codec 'none'). Resolves {head, chunks, header, bytes}: write `head` then every `chunks[i]` in order.
+ */
+export async function encodeBase(ckpt, { image, codec = 'none', compress = async x => x, chunkBytes = BASE_CHUNK, now = () => new Date().toISOString() } = {}) {
+  if (chunkBytes % (64 << 10)) throw new Error('chunkBytes must be a multiple of 64 KiB')
+  const chunks = []
+  const sections = []
+  let dataLen = 0
+  for (const s of ckpt.sections) {
+    const entries = []
+    for (const c0 of s.chunks) {
+      for (const c of split(c0, chunkBytes)) {
+        const z = codec === 'none' ? c : await compress(c)
+        entries.push({ len: c.length, clen: z.length, sha256: await sha256Hex(c.buffer instanceof SharedArrayBuffer ? c.slice() : c) })
+        chunks.push(z)
+        dataLen += z.length
+      }
+    }
+    sections.push({ name: s.name, chunks: entries })
+  }
+  const { sections: _drop, fileLen: _fl, ...rest } = ckpt.header
+  const header = { ...rest, base: 1, image, codec, createdAt: now(), dataLen, sections }
+  const json = enc.encode(JSON.stringify(header))
+  const head = new Uint8Array(16 + json.length + 32)
+  head.set(enc.encode(BASE_MAGIC), 0)
+  const dv = new DataView(head.buffer)
+  dv.setUint32(8, BASE_VERSION, true)
+  dv.setUint32(12, json.length, true)
+  head.set(json, 16)
+  head.set(sha256Sync(json), 16 + json.length)
+  return { head, chunks, header, bytes: head.length + dataLen, headerSha256: hex(sha256Sync(json)) }
+}
+
+/**
+ * A pull queue over a ReadableStream: a background pump keeps reading (up to `cap` bytes queued) so the network stays busy
+ * while the consumer decodes. read(n) gives exactly n bytes; feed(n, sink) passes n bytes on in pieces without a copy.
+ */
+export class ByteQueue {
+  constructor(stream, { cap = 16 << 20, onBytes = null, signal = null } = {}) {
+    this.reader = stream.getReader()
+    this.q = [] // Uint8Array pieces
+    this.queued = 0
+    this.total = 0 // bytes received from the stream
+    this.ended = false
+    this.error = null
+    this.cap = cap
+    this.onBytes = onBytes
+    this.wake = null // resolves a waiting consumer
+    this.room = null // resolves a waiting pump
+    this.waitMs = 0
+    this.signal = signal
+    if (signal) {
+      // an abort stops the pump and wakes a reader waiting for bytes: it fails at once instead of decoding on
+      const stop = () => {
+        this.cancelled = true
+        this.error = this.error || signal.reason || new DOMException('aborted', 'AbortError')
+        this.ended = true
+        this.reader.cancel(signal.reason).catch(() => {})
+        this.room?.()
+        this.notify()
+      }
+      if (signal.aborted) stop()
+      else signal.addEventListener('abort', stop, { once: true })
+    }
+    this.pumping = this.pump()
+  }
+
+  notify() {
+    const w = this.wake
+    this.wake = null
+    w?.()
+  }
+
+  async pump() {
+    try {
+      for (;;) {
+        while (this.queued >= this.cap && !this.cancelled) await new Promise(r => (this.room = r))
+        if (this.cancelled) return
+        const { done, value } = await this.reader.read()
+        if (done) break
+        if (!value || !value.length) continue
+        this.total += value.length
+        if (this.onBytes) await this.onBytes(value)
+        this.q.push(value)
+        this.queued += value.length
+        this.notify()
+      }
+    } catch (e) {
+      this.error = e
+    }
+    this.ended = true
+    this.notify()
+  }
+
+  async need(n) {
+    while (this.queued < n) {
+      if (this.error || this.ended) return false
+      const t = performance.now()
+      await new Promise(r => (this.wake = r))
+      this.waitMs += performance.now() - t
+    }
+    return true
+  }
+
+  /** Sets an error message for a stream that stopped (called by the reader with what it was reading). */
+  short(what, want) {
+    const got = this.queued
+    const why = this.error ? `the download failed (${this.error.name || 'Error'}: ${this.error.message || this.error})` : 'the download ended'
+    return new CheckpointError('truncated', `the base snapshot is cut short: ${why} after ${this.total} bytes, while reading ${what} (wanted ${want} more bytes, ${got} left)`)
+  }
+
+  take(n) {
+    const out = new Uint8Array(n)
+    this.feedSync(n, (p, o) => out.set(p, o))
+    return out
+  }
+
+  feedSync(n, sink) {
+    let done = 0
+    while (done < n) {
+      const p = this.q[0]
+      const k = Math.min(p.length, n - done)
+      sink(k === p.length ? p : p.subarray(0, k), done)
+      done += k
+      if (k === p.length) this.q.shift()
+      else this.q[0] = p.subarray(k)
+      this.queued -= k
+    }
+    if (this.room && this.queued < this.cap) {
+      const r = this.room
+      this.room = null
+      r()
+    }
+  }
+
+  async read(n, what) {
+    if (!(await this.need(n))) throw this.short(what, n)
+    return this.take(n)
+  }
+
+  /** Passes n bytes on in pieces as they arrive (a piece is a view of what the network gave: do not keep it). */
+  async feed(n, what, sink) {
+    let left = n
+    while (left > 0) {
+      if (!this.queued && !(await this.need(1))) throw this.short(what, left)
+      const k = Math.min(this.queued, left)
+      const pieces = []
+      this.feedSync(k, p => pieces.push(p))
+      for (const p of pieces) await sink(p)
+      left -= k
+    }
+  }
+
+  /** True when the stream ended cleanly with nothing left over; false when bytes remain; throws if the stream failed. */
+  async atEnd() {
+    while (!this.queued && !this.ended) await new Promise(r => (this.wake = r))
+    if (this.queued) return false
+    if (this.error) throw new CheckpointError('truncated', `the download failed at its very end (${this.error.name || 'Error'}: ${this.error.message || this.error}) after ${this.total} bytes`)
+    return true
+  }
+
+  async cancel(reason) {
+    this.cancelled = true
+    this.room?.()
+    try {
+      await this.reader.cancel(reason)
+    } catch {}
+  }
+}
+
+/** Decodes one chunk: `clen` compressed bytes from the queue into `len` raw bytes. */
+async function decodeChunk(q, codec, len, clen, what) {
+  if (codec === 'none') {
+    if (clen !== len) throw new CheckpointError('malformed', `${what}: codec none but clen ${clen} != len ${len}`)
+    return q.read(len, what)
+  }
+  if (codec !== 'gzip') throw new CheckpointError('malformed', `${what}: unknown codec ${JSON.stringify(codec)}`)
+  const ds = new DecompressionStream('gzip')
+  const w = ds.writable.getWriter()
+  const r = ds.readable.getReader()
+  const out = new Uint8Array(len)
+  let got = 0
+  const drain = (async () => {
+    for (;;) {
+      const { done, value } = await r.read()
+      if (done) break
+      if (got + value.length > len) throw new CheckpointError('corrupt', `${what} expands past its ${len} bytes: the file is damaged`)
+      out.set(value, got)
+      got += value.length
+    }
+    if (got !== len) throw new CheckpointError('corrupt', `${what} decodes to ${got} bytes, the header says ${len}: the file is damaged`)
+  })()
+  drain.catch(() => {})
+  try {
+    await q.feed(clen, what, p => w.write(p))
+    await w.close()
+    await drain
+  } catch (e) {
+    w.abort().catch(() => {})
+    r.cancel().catch(() => {})
+    drain.catch(() => {})
+    if (e instanceof CheckpointError) throw e
+    throw new CheckpointError('corrupt', `${what} does not decode (${e && e.name}: ${e && e.message}): the file is damaged`)
+  }
+  return out
+}
+
+/**
+ * Reads a base file from a stream: {header, sections: [{name, chunks}], stats}. header is the checkpoint's header
+ * (restorable: bootWmd takes {header, sections}). Everything is verified: the header's SHA-256 (and `expect.headerSha256`
+ * when given), every chunk's, the length of the stream. `onHeader(header)` runs as soon as the header is in (before any
+ * chunk is read): it may throw to refuse the file (a different vm.wasm, boot key), and the download is cancelled.
+ */
+export async function readBase(stream, { expect = {}, onHeader = null, onBytes = null, onProgress = null, cap, signal = null } = {}) {
+  const q = new ByteQueue(stream, { cap, onBytes, signal })
+  const stats = { decodeMs: 0, digestMs: 0, chunks: 0, bytes: 0 }
+  try {
+    const top = await q.read(16, 'the file header')
+    const magic = dec.decode(top.subarray(0, 8))
+    if (magic !== BASE_MAGIC) {
+      const head = [...top.subarray(0, 8)].every(b => b === 0)
+      if (head) throw new CheckpointError('incomplete', 'the base snapshot was never finished (its first bytes are zero: a copy whose write did not complete)')
+      throw new CheckpointError('foreign', `not a wmd-web base snapshot (magic ${JSON.stringify(magic.replace(/[^\x20-\x7e]/g, '?'))})`)
+    }
+    const dv = new DataView(top.buffer)
+    const version = dv.getUint32(8, true)
+    if (version !== BASE_VERSION) throw new CheckpointError('version', `base snapshot format ${version}; this build reads format ${BASE_VERSION}`)
+    const n = dv.getUint32(12, true)
+    if (n < 2 || n > BASE_MAX_HEADER) throw new CheckpointError('malformed', `base snapshot header length ${n} is not possible`)
+    const rest = await q.read(n + 32, 'the header')
+    const json = rest.subarray(0, n)
+    const got = sha256Sync(json)
+    const want = rest.subarray(n)
+    if (want.some((b, i) => b !== got[i])) throw new CheckpointError('corrupt', 'the base snapshot header does not match its SHA-256: the section table or a count in it is damaged')
+    if (expect.headerSha256 && hex(got) !== expect.headerSha256) {
+      throw new CheckpointError('corrupt', `the base snapshot is not the file the manifest names (header SHA-256 ${hex(got).slice(0, 12)}, the manifest says ${expect.headerSha256.slice(0, 12)})`)
+    }
+    let header
+    try {
+      header = JSON.parse(dec.decode(json))
+    } catch (e) {
+      throw new CheckpointError('malformed', `base snapshot header is not JSON: ${e.message}`)
+    }
+    if (header.base !== 1 || !Array.isArray(header.sections) || typeof header.dataLen !== 'number') throw new CheckpointError('malformed', 'base snapshot header is missing its section table')
+    let sum = 0
+    for (const s of header.sections) for (const c of s.chunks) {
+      if (!(Number.isInteger(c.len) && c.len >= 0 && Number.isInteger(c.clen) && c.clen >= 0 && typeof c.sha256 === 'string')) throw new CheckpointError('malformed', `section ${s.name} has a bad chunk entry`)
+      // a raw chunk is read whole from the queue: one longer than the queue's read-ahead cap could never be satisfied (it would park until a false stall)
+      if (header.codec === 'none' && c.clen > q.cap) throw new CheckpointError('malformed', `section ${s.name} has a raw chunk of ${c.clen} bytes; this reader takes at most ${q.cap} (re-pack with smaller chunks)`)
+      sum += c.clen
+    }
+    if (sum !== header.dataLen) throw new CheckpointError('malformed', `base snapshot header: its chunks add up to ${sum} bytes and dataLen says ${header.dataLen}`)
+    stats.fileBytes = 16 + n + 32 + header.dataLen
+    stats.headerAt = performance.now()
+    if (onHeader) await onHeader(header)
+    const sections = []
+    for (const s of header.sections) {
+      const chunks = []
+      for (let i = 0; i < s.chunks.length; i++) {
+        const c = s.chunks[i]
+        const what = `section ${s.name}, chunk ${i}`
+        const t0 = performance.now()
+        const raw = await decodeChunk(q, header.codec, c.len, c.clen, what)
+        const t1 = performance.now()
+        const sha = await sha256Hex(raw)
+        stats.decodeMs += t1 - t0 // includes the time spent waiting for the network
+        stats.digestMs += performance.now() - t1
+        if (sha !== c.sha256) throw new CheckpointError('corrupt', `base snapshot ${what} does not match its SHA-256: the file is damaged`)
+        chunks.push(raw)
+        stats.chunks++
+        stats.bytes += raw.length
+        onProgress?.(stats)
+      }
+      sections.push({ name: s.name, chunks })
+    }
+    if (!(await q.atEnd())) throw new CheckpointError('malformed', 'the base snapshot has bytes after its last chunk')
+    stats.netBytes = q.total
+    stats.netWaitMs = q.waitMs
+    return { header, sections, stats }
+  } catch (e) {
+    await q.cancel(e)
+    if (signal?.aborted) {
+      const r = signal.reason
+      throw r instanceof CheckpointError ? r : new CheckpointError('aborted', `the base snapshot read was stopped (${(r && r.message) || 'aborted'}) after ${q.total} bytes`)
+    }
+    throw e
+  }
+}
+
+/** The pieces of a base file as one byte array (tests, small files). */
+export const baseBytes = ({ head, chunks }) => concat([head, ...chunks])