@threadedstack/wmd-web 0.0.0-stage → 0.1.12

package/src/base.js

@@ -0,0 +1,325 @@
+// P-B6: base snapshots, the page's side. openBase() starts fetching (or reading from the OPFS cache) an image's base
+// snapshot at once and returns the lazy restore bootWmd takes as `restoreLazy`: the boot downloads the kernel and packs
+// meanwhile, checks the snapshot's header against its own vm.wasm and boot key as soon as it has them (a different one is
+// refused, the download cancelled, and the boot goes on cold with the reason in timings.baseFallback), and restores from the
+// snapshot once it is whole and verified.
+//
+//   const lazy = openBase({ manifestUrl: '/fixtures/images/python/base.json', cache: opfsBaseCache(), vmKey: vmKeyOf('/vm.wasm'), log })
+//   const vm = await bootWmd({ ...options, checkpoint: true, restoreLazy: lazy, restoreFallback: true })
+//   vm.timings.restore ? 'restored from the base snapshot' : vm.timings.baseFallback   // {code, message}
+//
+// The manifest (base.json next to the packs): {version: 1, image, entries: [{file, vmKey, bootKey, headerSha256, bytes, codec,
+// createdAt}]}. With `vmKey` the entry is chosen by vm.wasm key (the last entry of that build); a manifest with none for this
+// build is refused at once, without downloading the file. A cache entry is named after the header's SHA-256, so a new
+// snapshot never reads an old copy; the manifest's headerSha256 pins the header, the header pins every chunk and the packs.
+//
+// Lifecycle (what bootWmd does with the lazy object):
+//   header          resolves when the snapshot's header is in; the reader then WAITS for proceed() (or abort) before it decodes
+//                   a chunk, so a header the boot refuses (another vm.wasm) costs only what the pump had queued (a few MB)
+//   proceed()       the vm key matched: the reader may go on
+//   ckpt            resolves with a box {take(), drop(), stats}: take() hands the snapshot over once
+//   accept()        the restore SUCCEEDED: the cache copy (if this was a download) is committed
+//   abort(reason)   the boot does not want it (any exit before accept): the download stops, the box is dropped, the pending
+//                   cache copy is discarded; an entry read from the cache is deleted when the reason is a refusal of its content
+//   reject(err)     the restore failed after the snapshot was taken over: abort, and the cache entry is removed whatever its source
+// A download that stops making progress for `stallMs`, or runs longer than `maxMs`, is aborted with code 'stalled'.
+import { opfsBaseCache } from './host/base-cache.js'
+import { readBase } from './host/base-file.js'
+import { CheckpointError, sha256Hex } from './host/ckpt.js'
+
+export { opfsBaseCache, memoryBaseCache } from './host/base-cache.js'
+export { readBase, encodeBase } from './host/base-file.js'
+
+/** A cache file name: the image with the characters a file name may not have replaced (no '-' in it), '-', the header hash. */
+export const safeImage = image => String(image || 'base').replace(/[^A-Za-z0-9_.]/g, '_')
+const cacheName = (image, entry) => `${safeImage(image)}-${entry.headerSha256.slice(0, 24)}.wmdbase`
+const REFUSALS = ['stale', 'version', 'malformed', 'corrupt', 'foreign', 'incomplete']
+
+/** The SHA-256 of a vm.wasm URL's bytes (what bootWmd's vmKey is), for openBase's `vmKey`. */
+export async function vmKeyOf(url, fetchFn = (...a) => fetch(...a)) {
+  const r = await fetchFn(url)
+  if (!r.ok) throw new Error(`GET ${url}: ${r.status}`)
+  return sha256Hex(new Uint8Array(await r.arrayBuffer()))
+}
+
+/**
+ * @param {object} o {manifestUrl, image?, cache?, query? (appended to the file's URL: test hooks), log?, fetchFn?, vmKey? (a key or a promise of one),
+ *   pick? (entries -> entry), stallMs? (20000), maxMs? (600000), gate? (true: the reader waits for proceed() after the header)}
+ */
+export function openBase({ manifestUrl, image, cache = null, query = '', log = () => {}, fetchFn = (...a) => fetch(...a), vmKey = null, pick = es => es.at(-1), stallMs = 20_000, maxMs = 600_000, gate = true } = {}) {
+  const ac = new AbortController()
+  const info = { source: null, manifestMs: null, cacheWrite: null, attempts: [], netBytes: 0, stalled: null }
+  let resolveHeader
+  let rejectHeader
+  const header = new Promise((res, rej) => ((resolveHeader = res), (rejectHeader = rej)))
+  header.catch(() => {})
+  let headerSettled = false
+  const t0 = performance.now()
+  let lastProgress = t0
+  let gateWaiting = false
+  let openGate
+  const gateP = gate ? new Promise(r => (openGate = r)) : Promise.resolve()
+  let pending = null // {tee, name, image}: a verified download whose cache copy waits for accept()
+  let box = null
+  let aborted = false
+  let cur = { name: null, source: null }
+  let committed = null // the cache file name once accept() wrote it
+  const touch = () => (lastProgress = performance.now())
+
+  const reasonOf = () => {
+    const r = ac.signal.reason
+    return r instanceof CheckpointError ? r : new CheckpointError('aborted', `the base snapshot was abandoned (${(r && r.message) || 'aborted'})`)
+  }
+  const fail = err => {
+    if (ac.signal.aborted) return
+    if (err.code === 'stalled') info.stalled = err.message
+    ac.abort(err)
+    openGate?.()
+  }
+  // no bytes for stallMs (while the reader is not waiting for the boot), or more than maxMs in all: refuse, so the boot goes on cold
+  const dog = setInterval(() => {
+    const now = performance.now()
+    if (now - t0 > maxMs) fail(new CheckpointError('stalled', `the base snapshot did not finish in ${Math.round(maxMs / 1000)} s (${info.netBytes} bytes received): giving up on it`))
+    else if (!gateWaiting && now - lastProgress > stallMs) fail(new CheckpointError('stalled', `the base snapshot stalled: no bytes for ${Math.round(stallMs / 1000)} s after ${info.netBytes} bytes (the host stopped answering?): giving up on it`))
+  }, Math.max(20, Math.min(1000, stallMs / 4)))
+  const stopDog = () => clearInterval(dog)
+
+  async function dropCacheEntry(why) {
+    const name = committed || cur.name
+    if (!cache || !name) return
+    await cache.delete(name).catch(() => {})
+    log(`base snapshot: cache entry ${name} removed (${why})`)
+  }
+
+  async function accept() {
+    const p = pending
+    pending = null
+    if (!p || aborted) return
+    const tc = performance.now()
+    try {
+      const w = await p.tee.commit()
+      committed = p.name
+      info.cacheWrite = { ok: true, bytes: w.bytes, ms: performance.now() - tc }
+      pruneOthers(cache, p.image, p.name).catch(() => {})
+    } catch (e) {
+      p.tee.abort().catch(() => {}) // never leave a half-committed file
+      info.cacheWrite = { ok: false, name: e && e.name, error: `${e && e.name}: ${e && e.message}` }
+      log(`base snapshot: the cache write did not finish (${e && e.message})`)
+    }
+  }
+
+  function abort(reason) {
+    if (aborted) return
+    aborted = true
+    const err = reason instanceof CheckpointError ? reason : reason ? new CheckpointError('aborted', `the base snapshot was abandoned (${reason.message || reason})`) : new CheckpointError('aborted', 'the base snapshot was abandoned')
+    if (!ac.signal.aborted) ac.abort(err)
+    openGate?.()
+    stopDog()
+    const p = pending
+    pending = null
+    p?.tee.abort().catch(() => {})
+    box?.drop()
+    // a cached file this page refuses on its content (another vm.wasm, a damaged chunk) is removed, so it is not read again
+    if (cur.source === 'opfs' && err.code && REFUSALS.includes(err.code)) dropCacheEntry(err.code)
+  }
+
+  async function reject(err) {
+    abort(err)
+    await dropCacheEntry(`the restore failed: ${(err && err.message) || err}`)
+  }
+
+  async function run() {
+    const tm = performance.now()
+    let manifest
+    try {
+      const r = await fetchFn(manifestUrl, { cache: 'no-store', signal: ac.signal })
+      if (!r.ok) throw new CheckpointError('unavailable', `no base snapshot: GET ${manifestUrl} answered ${r.status}`)
+      manifest = await r.json()
+    } catch (e) {
+      if (ac.signal.aborted) throw reasonOf()
+      if (e instanceof CheckpointError) throw e
+      throw new CheckpointError('unavailable', `no base snapshot: ${manifestUrl} could not be read (${e && e.name}: ${e && e.message})`)
+    }
+    touch()
+    info.manifestMs = performance.now() - tm
+    let entries = (manifest.entries || []).filter(e => !image || e.image === image || !e.image)
+    if (vmKey) {
+      let vk = null
+      // the page's vm key may be slow or never settle: the wait is bounded (stallMs), ends at once on an abort, and the watchdog does not
+      // count it as a download stall (like the gate)
+      gateWaiting = true
+      let timer
+      try {
+        vk = await Promise.race([
+          vmKey,
+          new Promise((_, rej) => {
+            timer = setTimeout(() => rej(new CheckpointError('stalled', `the page's vm.wasm key did not arrive in ${Math.round(stallMs / 1000)} s: giving up on the base snapshot`)), stallMs)
+          }),
+          new Promise((_, rej) => {
+            if (ac.signal.aborted) rej(reasonOf())
+            else ac.signal.addEventListener('abort', () => rej(reasonOf()), { once: true })
+          }),
+        ])
+      } catch (e) {
+        if (e instanceof CheckpointError) throw e
+        log(`base snapshot: this page's vm.wasm key is unknown (${e && e.message}); the header decides`)
+      } finally {
+        clearTimeout(timer)
+        gateWaiting = false
+        touch()
+      }
+      if (vk) {
+        const mine = entries.filter(e => e.vmKey === vk)
+        if (entries.length && !mine.length) {
+          throw new CheckpointError('stale', `no base snapshot for this vm.wasm (${vk.slice(0, 12)}): the manifest ${manifestUrl} lists snapshots of vm.wasm ${[...new Set(entries.map(e => String(e.vmKey).slice(0, 12)))].join(', ')} only (its memory image is another emulator's); take a new one`)
+        }
+        entries = mine
+      }
+    }
+    const entry = entries.length ? pick(entries) : null
+    if (!entry || !entry.file || typeof entry.headerSha256 !== 'string') throw new CheckpointError('unavailable', `no base snapshot: the manifest ${manifestUrl} lists none${image ? ` for ${image}` : ''}`)
+    info.entry = entry
+    const name = cacheName(image || manifest.image, entry)
+    const url = new URL(entry.file, new URL(manifestUrl, globalThis.location?.href || 'http://localhost/')).href + query
+    const opts = {
+      expect: { headerSha256: entry.headerSha256 },
+      signal: ac.signal,
+      onProgress: touch,
+      onHeader: async h => {
+        if (!headerSettled) {
+          headerSettled = true
+          resolveHeader(h)
+        }
+        if (gate) {
+          gateWaiting = true
+          await gateP // the boot checks the vm key; a refusal aborts (abort() opens the gate)
+          gateWaiting = false
+          touch()
+        }
+      },
+    }
+    // attempt 1: the cache (if it holds the file); attempt 2 (or 1): the network. A cached file that fails verification is deleted and the network is tried.
+    for (const source of cache ? ['opfs', 'network'] : ['network']) {
+      let stream = null
+      let tee = null
+      const att = { source }
+      info.attempts.push(att)
+      const ta = performance.now()
+      info.netBytes = 0
+      cur = { name, source }
+      try {
+        if (source === 'opfs') {
+          const c = await cache.open(name).catch(e => {
+            att.cacheError = String(e && e.message)
+            return null
+          })
+          if (!c) {
+            att.miss = true
+            continue
+          }
+          stream = c.stream
+          att.size = c.size
+        } else {
+          const r = await fetchFn(url, { signal: ac.signal })
+          if (!r.ok || !r.body) throw new CheckpointError('unavailable', `the base snapshot ${url} answered ${r.status}`)
+          touch()
+          att.headers = r.headers.get('content-length')
+          att.contentEncoding = r.headers.get('content-encoding')
+          if (cache) {
+            try {
+              tee = await cache.writer(name)
+            } catch (e) {
+              info.cacheWrite = { ok: false, name: e && e.name, error: `${e && e.name}: ${e && e.message}` }
+              log(`base snapshot: not cached (${e && e.message})`)
+            }
+          }
+          stream = r.body
+        }
+        const res = await readBase(stream, {
+          ...opts,
+          onBytes: async u8 => {
+            info.netBytes += u8.length // live: a page can watch how far the download is
+            touch()
+            if (!tee) return
+            try {
+              await tee.write(u8)
+            } catch (e) {
+              // the cache failed (cleared, full, another tab): the download goes on without it
+              info.cacheWrite = { ok: false, name: e && e.name, error: `${e && e.name}: ${e && e.message}` }
+              log(`base snapshot: the cache write failed, going on without it (${e && e.message})`)
+              tee.abort().catch(() => {})
+              tee = null
+            }
+          },
+        })
+        att.ms = performance.now() - ta
+        info.source = source
+        info.stats = res.stats
+        info.totalMs = performance.now() - t0
+        stopDog()
+        // the cache copy is committed only when the restore has succeeded (accept()): a snapshot this page refuses, or fails to restore, is not kept
+        if (tee) {
+          if (aborted) {
+            tee.abort().catch(() => {})
+            throw new CheckpointError('aborted', 'the base snapshot was refused before it finished downloading')
+          }
+          pending = { tee, name, image: image || manifest.image }
+        }
+        if (aborted) throw reasonOf()
+        let held = res
+        box = {
+          take() {
+            const r = held
+            held = null
+            return r
+          },
+          drop() {
+            held = null
+          },
+          stats: res.stats,
+        }
+        return box
+      } catch (e) {
+        att.error = `${e && e.name}: ${e && e.message}`
+        att.code = e && e.code
+        if (tee) tee.abort().catch(() => {}) // never leave a partial file that looks whole (its magic is not in)
+        if (ac.signal.aborted) throw reasonOf()
+        if (source === 'opfs') {
+          log(`base snapshot: the cached copy was not usable (${e && e.message}); deleting it and downloading`)
+          await cache.delete(name).catch(() => {})
+          continue
+        }
+        throw e
+      }
+    }
+    throw new CheckpointError('unavailable', 'no base snapshot could be read')
+  }
+
+  const ckpt = run().then(
+    r => r,
+    e => {
+      stopDog()
+      if (!headerSettled) {
+        headerSettled = true
+        rejectHeader(e)
+      }
+      throw e
+    },
+  )
+  ckpt.catch(() => {})
+  return {
+    header,
+    ckpt,
+    info,
+    /** The vm key matched: the reader may decode (it waits here after the header). */
+    proceed: () => openGate?.(),
+    accept,
+    abort,
+    reject,
+  }
+}
+
+async function pruneOthers(cache, image, keep) {
+  const prefix = `${safeImage(image)}-`
+  for (const f of await cache.list()) if (f.name !== keep && f.name.startsWith(prefix)) await cache.delete(f.name)
+}