@threadedstack/wmd-web 0.1.18 → 0.1.20

package/src/boot.js

@@ -192,6 +192,13 @@ export const WEBKIT_CODE_BUDGET_MIB = 144
  */
 export const WEBKIT_JIT_THRESHOLD = 1000
 
+/**
+ * D-551: how deep compiled guest calls may nest in WebKit (vm.wasm's WMD_MIRROR_DEPTH; 32, and 64 for region
+ * calls, elsewhere). An amd64 go build in WebKit trapped with "Maximum call stack size exceeded" at about 33
+ * frames of block code, which killed the machine; Chromium and Firefox ran the same build at 32.
+ */
+export const WEBKIT_MIRROR_DEPTH = 8
+
 const MiB = 1 << 20
 
 /**
@@ -342,7 +349,7 @@ export function fmaFuses() {
 
 /**
  * W32: the environment vm.wasm starts with in this browser: vmEnv's, plus
- * WMD_JIT_THRESHOLD=WEBKIT_JIT_THRESHOLD in WebKit, W30's WMD_TRACE_MAX for the engine and
+ * WMD_JIT_THRESHOLD=WEBKIT_JIT_THRESHOLD and D-551's WMD_MIRROR_DEPTH=WEBKIT_MIRROR_DEPTH in WebKit, W30's WMD_TRACE_MAX for the engine and
  * W33's WMD_RESCUE_MS, D-539's WMD_FMA_FUSED=1 where the engine fuses relaxed_madd (fmaFuses) and, in
  * Chromium, D-542's WMD_RGN_DUP=1 (regions copy sampled-hot compiled blocks of their page; Firefox's go ran
  * slower with it, WebKit did not move), each unless the caller sets it. Kept out of the boot key (bootKeyOf hashes vmEnv's list), so a checkpoint or base
@@ -355,6 +362,7 @@ export function browserEnv(env, nav = globalThis.navigator, fused = fmaFuses())
   const has = k => env.some(e => e.startsWith(`${k}=`))
   const add = []
   if (isWebKit(nav) && !has('WMD_JIT_THRESHOLD')) add.push(`WMD_JIT_THRESHOLD=${WEBKIT_JIT_THRESHOLD}`)
+  if (isWebKit(nav) && !has('WMD_MIRROR_DEPTH')) add.push(`WMD_MIRROR_DEPTH=${WEBKIT_MIRROR_DEPTH}`)   // D-551
   if (!has('WMD_TRACE_MAX')) add.push(`WMD_TRACE_MAX=${BROWSER_TRACE_MAX[engineOf(nav)]}`)
   if (!has('WMD_RESCUE_MS')) add.push(`WMD_RESCUE_MS=${BROWSER_RESCUE_MS}`)
   if (!has('WMD_FMA_FUSED') && fused) add.push('WMD_FMA_FUSED=1')   // D-539

package/src/build/builder.js

@@ -6,7 +6,8 @@
 // the image back as a Docker tar on buildctl's stdout and converts it with loadImageTar. A Dockerfile with no RUN
 // is built in the page instead (fast.js), unless fast: false.
 
-import { bootWmd } from '../boot.js'
+import { blobBytes } from '../blob.js'
+import { bootWmd, isWebKit } from '../boot.js'
 import { MAIN_PROC_ID, containerBundle, fillRunRequest } from '../host/ctl.js'
 import { DiskError, deleteDisk, openDisk, sweepDisks } from '../host/disk.js'
 import { p9MountOptions } from '../host/p9mount.js'
@@ -15,6 +16,8 @@ import { packContext, readTarFile } from './context.js'
 import { BUILDER_PINS } from './pin.js'
 import { FastRefused, fastBuild, fastEligible } from './fast.js'
 
+export { blobBytes } // D-554: read the tar Blob a build returns with it (WebKit cannot read 128 MiB or more whole)
+
 export const ARCHES = {
   arm64: {
     platform: 'linux/arm64',
@@ -41,13 +44,68 @@ const FULL_MIB = 64 // /bk with less free than this during a failed build: the b
 const enc = new TextEncoder()
 /** host/jit.js's line when the engine's code pool is full (D-515). */
 export const POOL_FULL_RE = /wmd-web: jit: the engine's code pool is full/
+const JIT_LINE_RE = /^(?:vm\.wasm: JIT|wmd-web: jit:)/
+const JIT_LINES = 60 // guest/host JIT messages kept (a ring)
+const JIT_EVENTS = 200 // 'jit' events sent in a builder's life: the vm repeats some of them at every release
+const JIT_KINDS = [
+  [/the engine's code pool is full/, 'pool-full'],
+  [/the engine compiles code again/, 'pool-back'],
+  [/blocks run in the interpreter from now on/, 'interpreter-from-now-on'], // vm cache_release: release_dead
+  [/waiting before releasing again/, 'release-deferred'], // D-445 grace window
+  [/JIT release/, 'release'],
+  [/module budget reached|budget \d+ KiB from now on|refused more code/, 'budget'],
+]
+/** What a JIT message line says, or null for a line that is not one. */
+export function jitKind(line) {
+  if (!JIT_LINE_RE.test(line)) return null
+  return JIT_KINDS.find(([re]) => re.test(line))?.[1] || 'other'
+}
+/**
+ * The guest's stderr into whole lines: the JIT's messages are kept in a ring with the time (performance.now, ms)
+ * and sent as {t:'jit', what, line} events, so a build that crawls in the interpreter can be told from one that hangs.
+ */
+export function jitWatch({ onEvent = () => {}, max = JIT_LINES, maxEvents = JIT_EVENTS, now = () => Math.round(performance.now()) } = {}) {
+  const ring = []
+  let buf = ''
+  let events = 0
+  let skipped = 0
+  const line = l => {
+    const what = jitKind(l)
+    if (!what) return
+    const at = now()
+    ring.push({ at, what, line: l.slice(0, 400) })
+    if (ring.length > max) ring.shift()
+    if (what === 'other') return
+    if (events < maxEvents || what === 'interpreter-from-now-on') {
+      events++
+      onEvent({ t: 'jit', what, line: l.slice(0, 400) })
+    } else skipped++
+  }
+  return {
+    push(text) {
+      buf += text
+      let i
+      while ((i = buf.indexOf('\n')) >= 0) {
+        line(buf.slice(0, i))
+        buf = buf.slice(i + 1)
+      }
+      if (buf.length > 4096) buf = buf.slice(-4096)
+    },
+    lines: () => ring.slice(),
+    skipped: () => skipped,
+  }
+}
 
 export class BuildError extends Error {
-  constructor(message, { step = null, tail = '', reason = null } = {}) {
+  constructor(message, { step = null, tail = '', reason = null, jit = null, machine = null } = {}) {
     super(message)
     this.name = 'BuildError'
     this.step = step
     this.tail = tail
+    this.jit = jit // the builder's last JIT messages ([{at, what, line}]), when it kept them
+    // when buildctl's stream ended without an exit (code 125): why it closed, whether the machine ended, the
+    // machine's stderr tail and the start script's last lines
+    this.machine = machine
     this.reason = reason // 'no-space' | 'quota' | 'disk' | 'timeout' | 'pin' | 'secret' | null
   }
 }
@@ -220,7 +278,7 @@ async function indexDigest(index) {
 async function bytesOf(x, fetchFn) {
   if (x instanceof Uint8Array) return x
   if (x instanceof ArrayBuffer) return new Uint8Array(x)
-  if (typeof Blob !== 'undefined' && x instanceof Blob) return new Uint8Array(await x.arrayBuffer())
+  if (typeof Blob !== 'undefined' && x instanceof Blob) return blobBytes(x)
   const r = await fetchFn(x)
   if (!r.ok) throw new Error(`${x}: ${r.status} ${r.statusText}`)
   return new Uint8Array(await r.arrayBuffer())
@@ -348,10 +406,88 @@ function aborted(signal) {
  * @param {string[]} [o.registryMirrors]    Docker Hub mirrors (host[:port], in order) BuildKit tries before docker.io; list a plain-http one
  *                                           in httpRegistries too
  * @param {Record<string,string>} [o.registryCAs]  host -> PEM: CAs BuildKit checks that registry's certificate against
+ * @param {boolean} [o.nodeInterpreted]     D-553: node in RUN steps runs its JavaScript in the interpreter (a preload turns
+ *                                           off V8's compilers; WebAssembly stays). Default: on in WebKit, off elsewhere.
+ *                                           Without it, WebKit cannot keep up with the code node's JIT writes, and an npm
+ *                                           install of 1,600 packages does not end in 30 minutes.
  * @param {Function} [o.onEvent]
  * @param {AbortSignal} [o.signal]
  * @param {object} [o.boot]                  extra bootWmd options (tests)
  */
+/**
+ * D-553: the preload node runs in a RUN step (NODE_OPTIONS --require). It turns off V8's compilers, so node runs its
+ * JavaScript in the interpreter and writes no new machine code; WebAssembly stays (node-gyp's undici needs it, which
+ * --jitless removed). Each flag only where that V8 has it (V8 prints an error for an unknown one): measured on node
+ * 12, 14, 16, 18, 20, 22 and 24.
+ */
+export const NODE_PRELOAD = `try {
+  var v = process.versions.v8.split('.').map(Number), m = v[0] * 100 + v[1], f = ['--no-opt']
+  if (m >= 708) f.push('--regexp-interpret-all')
+  if (m >= 904) f.push('--no-sparkplug', '--liftoff-only')
+  if (m >= 1100) f.push('--no-maglev')
+  require('v8').setFlagsFromString(f.join(' '))
+} catch (e) {}
+`
+
+/**
+ * D-553: buildkit-runc that makes node in a RUN step load NODE_PRELOAD (runc ... --bundle <dir> ...). BuildKit writes
+ * config.json on one line (json.Encoder). awk walks it as JSON, strings and escapes included (inside a string a quote
+ * is \", so the keys it looks for cannot occur there):
+ * - it appends a read-only bind of /usr/local/lib/wmd-node.js at /dev/.wmd-node.js to the end of "mounts", after
+ *   BuildKit's /dev tmpfs, so the file is in that tmpfs and never in the step's layer;
+ * - then it adds --require=/dev/.wmd-node.js to the first "env":[ (process.env)'s NODE_OPTIONS, or a NODE_OPTIONS
+ *   element at the front.
+ * Without a "mounts" array it changes nothing (a --require of a missing file would stop node).
+ */
+export const RUNC_NODE = String.raw`#!/bin/sh
+b=
+p=
+for a in "$@"; do
+  case "$a" in --bundle=*) b=$(printf %s "$a" | cut -c10-) ;; esac
+  case "$p" in --bundle|-b) b=$a ;; esac
+  p=$a
+done
+c="$b/config.json"
+if [ -n "$b" ] && [ -f "$c" ]; then
+  awk -v M='{"destination":"/dev/.wmd-node.js","type":"bind","source":"/usr/local/lib/wmd-node.js","options":["rbind","ro"]}' -v R='--require=/dev/.wmd-node.js' '
+  function shut(s, st,   n, i, ch, ins, esc, d) {
+    n = length(s); ins = 0; esc = 0; d = 0
+    for (i = st; i <= n; i++) {
+      ch = substr(s, i, 1)
+      if (ins) { if (esc) esc = 0; else if (ch == "\\") esc = 1; else if (ch == "\"") ins = 0 }
+      else if (ch == "\"") ins = 1
+      else if (ch == "[" || ch == "{") d++
+      else if (ch == "]" || ch == "}") { if (!d) return i; d-- }
+    }
+    return 0
+  }
+  {
+    s = $0
+    k = index(s, ",\"mounts\":["); if (!k) k = index(s, "{\"mounts\":[")
+    if (!k || index(s, "\"destination\":\"/dev/.wmd-node.js\"")) { print s; next }
+    st = k + 11; e = shut(s, st); if (!e) { print s; next }
+    s = substr(s, 1, e - 1) (e == st ? "" : ",") M substr(s, e)
+    k = index(s, "\"env\":["); if (!k) { print s; next }
+    st = k + 7; n = length(s); ins = 0; esc = 0; found = 0; q = 0; L = length(R) + 1
+    for (i = st; i <= n; i++) {
+      ch = substr(s, i, 1)
+      if (ins) {
+        if (esc) esc = 0
+        else if (ch == "\\") esc = 1
+        else if (ch == "\"") {
+          ins = 0
+          if (substr(s, q, 14) == "\"NODE_OPTIONS=") { found = 1; s = substr(s, 1, i - 1) " " R substr(s, i); i += L; n += L }
+        }
+      } else if (ch == "\"") { ins = 1; q = i }
+      else if (ch == "]") break
+    }
+    if (!found) s = substr(s, 1, st - 1) "\"NODE_OPTIONS=" R "\"" (substr(s, st, 1) == "]" ? "" : ",") substr(s, st)
+    print s
+  }' "$c" > "$c.wmd" && mv "$c.wmd" "$c"
+fi
+exec /usr/bin/buildkit-runc "$@"
+`
+
 export async function createBuilder(o) {
   const archName = o.arch || 'arm64'
   const arch = ARCHES[archName]
@@ -359,11 +495,13 @@ export async function createBuilder(o) {
   if (o.disk !== false) checkNamespace(o.disk?.namespace)
   if (!o.relay?.url) throw new TypeError('relay: {url}: the builder pulls base images through it')
   const fetchFn = o.fetch || globalThis.fetch.bind(globalThis)
+  // every event carries `at`: performance.now() in ms (`ms` is a duration in some events)
   const emit = ev => {
     try {
-      o.onEvent?.(ev)
+      o.onEvent?.({ at: Math.round(performance.now()), ...ev })
     } catch {}
   }
+  const jit = jitWatch({ onEvent: emit })
   const ram = o.ram || 1536
   const cpus = o.cpus || 2
   const t0 = performance.now()
@@ -398,7 +536,16 @@ export async function createBuilder(o) {
     ...(o.registryMirrors?.length ? [`WMD_BK_MIRRORS=${o.registryMirrors.map(checkHost).join(',')}`] : []),
     ...(o.registryCAs && Object.keys(o.registryCAs).length ? [`WMD_BK_CA=${Object.entries(o.registryCAs).map(([h, pem]) => `${checkHost(h)}=${b64utf8(pem)}`).join(' ')}`] : []),
   ]
-  const argv = ['/usr/local/bin/wmd-buildkitd']
+  const interp = o.nodeInterpreted ?? isWebKit()
+  // D-553: BuildKit starts each RUN step with buildkit-runc from PATH; a wrapper earlier in PATH makes node in the step
+  // load a preload that turns off V8's compilers, then runs the real one. The image and the cache keys stay as they are.
+  const argv = interp
+    ? ['/bin/sh', '-c', 'mkdir -p /usr/local/lib && printf %s "$WMD_NODE_PRELOAD" > /usr/local/lib/wmd-node.js && printf %s "$WMD_RUNC_WRAP" > /usr/local/bin/buildkit-runc && chmod 0755 /usr/local/bin/buildkit-runc && exec /usr/local/bin/wmd-buildkitd']
+    : ['/usr/local/bin/wmd-buildkitd']
+  if (interp) {
+    env.push(`WMD_RUNC_WRAP=${RUNC_NODE}`, `WMD_NODE_PRELOAD=${NODE_PRELOAD}`)
+    emit({ t: 'node-interpreted', message: "node in RUN steps runs its JavaScript in the interpreter: WebKit's code pool cannot keep up with node's JIT" })
+  }
   const b = containerBundle({ args: argv, env, cwd: '/', hostname: 'buildkitsandbox', networkDisabled: false, ip: '192.168.127.2', gateway: '192.168.127.1', privileged: true })
   const append = [...arch.console, 'rdinit=/sbin/init', 'loglevel=4', 'norandmaps', 'wmd.init', 'wmd.p9tag=wmdroot', 'wmd.control=vport', 'wmd.exit'].join(' ')
   const args = [
@@ -425,6 +572,7 @@ export async function createBuilder(o) {
       ...(disk ? { disk } : {}),
       onStderr: s => {
         stderr = (stderr + s).slice(-16384)
+        jit.push(s)
         // WebKit: the engine's code pool filled, so new code waits and the guest runs slower (D-515; plan 5.10)
         if (!slowSaid && POOL_FULL_RE.test(s)) {
           slowSaid = true
@@ -482,6 +630,8 @@ export async function createBuilder(o) {
     throw e
   }
   emit({ t: 'ready', ms: Math.round(performance.now() - t0), platform: arch.platform })
+  let endedWith = null
+  ended.then(r => (endedWith = { code: r?.code ?? null, reason: r?.reason ?? null }), () => {})
 
   let seq = 0
   let busy = false
@@ -539,14 +689,14 @@ export async function createBuilder(o) {
         clearTimeout(killTimer)
         signal?.removeEventListener('abort', kill)
       }
-      if (!binary) return { code: r.code ?? null, out, err }
+      if (!binary) return { code: r.code ?? null, out, err, error: r.error ?? null }
       const bytes = new Uint8Array(size)
       let at = 0
       for (const c of chunks) {
         bytes.set(c, at)
         at += c.length
       }
-      return { code: r.code ?? null, out: bytes, err }
+      return { code: r.code ?? null, out: bytes, err, error: r.error ?? null }
     })()
   }
   const sh = (cmd, opts) => inGuest(['/bin/sh', '-c', cmd], opts)
@@ -563,6 +713,15 @@ export async function createBuilder(o) {
     inGuest,
     sh,
     log: () => mainLog.lines,
+    jitLines: () => jit.lines(),
+    /** The machine's state for a failure report: {ended ({code, reason} or null), stderr tail, start script's tail}. */
+    machineState: () => ({ ended: endedWith, stderr: stderr.slice(-4000), main: mainLog.tail.slice(-3000) }),
+    /** The guest network's counters and its last 40 events (resets, refusals, D-551 redials), or null. */
+    netStats: async () => {
+      const n = await vm.net?.stats?.().catch(() => null)
+      if (!n?.stack) return n ?? null
+      return { stack: { ...n.stack, events: (n.stack.events || []).slice(-40) }, relay: n.relay ?? null }
+    },
     /** Free MiB and free inodes on /bk. */
     async room() {
       const r = await sh("df -Pm /bk | awk 'NR==2{print $4}'; df -Pi /bk | awk 'NR==2{print $4, $2}'")
@@ -609,6 +768,7 @@ async function keepBuilt(store, index, packs) {
 async function runBuild(builder, req, emit) {
   const t0 = performance.now()
   const onEvent = ev => {
+    ev = { at: Math.round(performance.now()), ...ev }
     emit(ev)
     try {
       req.onEvent?.(ev)
@@ -650,7 +810,18 @@ async function runBuild(builder, req, emit) {
   const dir = `/run/ctx/${id}`
   // req.signal and timeoutMs cover everything from here: the context, the upload, buildctl and the conversion
   const ac = new AbortController()
-  const timer = req.timeoutMs ? setTimeout(() => ac.abort(new BuildError(`the build ran past ${req.timeoutMs} ms`, { reason: 'timeout' })), req.timeoutMs) : null
+  let lastStep = null
+  const timer = req.timeoutMs
+    ? setTimeout(
+        () => {
+          const e = new BuildError(`the build ran past ${req.timeoutMs} ms${lastStep ? `, in the step ${lastStep}` : ''}`, { reason: 'timeout', step: lastStep, tail: log.tail.slice(-4000), jit: builder.jitLines?.() ?? null })
+          ac.abort(e)
+          // the network's counters join the error while buildctl is stopped (SIGINT, then 5 s), before it is thrown
+          builder.netStats?.().then(n => (e.net = n), () => {})
+        },
+        req.timeoutMs,
+      )
+    : null
   const onAbort = () => ac.abort(new BuildCancelled())
   req.signal?.addEventListener('abort', onAbort, { once: true })
   const check = () => {
@@ -698,7 +869,6 @@ async function runBuild(builder, req, emit) {
     if (req.push?.auth) xenv.push(`DOCKER_CONFIG=${dir}/auth`)
     args.push('--output', `type=docker,${req.tag ? `name=${req.tag},` : ''}dest=-`)
     if (req.push) args.push('--output', `type=image,name=${req.push.name},push=true${req.push.insecure ? ',registry.insecure=true' : ''}`)
-    let lastStep = null
     const errLines = [] // buildctl's own error lines (no step), not the steps' output
     const pr = progressReader({
       onStep: ev => {
@@ -737,8 +907,22 @@ async function runBuild(builder, req, emit) {
       if (builder.disk?.events?.slice?.(diskMark).some(e => e.reason === 'quota')) reason = 'quota'
       if (!reason && /input\/output error|read-only file system/i.test(tail)) reason = 'disk'
       const failed = [...pr.vertexes.values()].find(v => v.error)
-      const why = failed?.error || tail.split('\n').filter(Boolean).pop() || `buildctl exited ${r.code}`
-      throw new BuildError(`the build failed${failed ? ` at ${failed.name}` : ''}: ${why.slice(0, 500)}`, { step: failed?.name || lastStep, tail, reason })
+      const why = failed?.error || tail.split('\n').filter(Boolean).pop() || `buildctl exited ${r.code}${r.error ? ` (${r.error})` : ''}`
+      // a stream that ended with no exit frame: the machine's side says why
+      let machine = null
+      if (r.code === 125 && r.error) {
+        await new Promise(res => setTimeout(res, 2000)) // let a machine that is going down report it
+        machine = { stream: r.error, ...(builder.machineState?.() ?? {}) }
+      }
+      const err = new BuildError(`the build failed${failed ? ` at ${failed.name}` : ''}: ${why.slice(0, 500)}`, { step: failed?.name || lastStep, tail, reason, jit: builder.jitLines?.() ?? null, machine })
+      err.net = (await builder.netStats?.().catch(() => null)) ?? null
+      // buildkitd's side (a session it closed, a worker error): its last log lines, when the machine still answers
+      if (r.code !== 125) {
+        const d = await Promise.race([builder.sh('tail -n 40 /run/buildkitd.log').catch(() => null), new Promise(res => setTimeout(res, 10_000, null))])
+        err.daemonLog = d?.out ? d.out.slice(-6000) : null
+      }
+      check() // a cancel or timeout during these waits wins
+      throw err
     }
     onEvent({ t: 'export', bytes: r.out.length })
     const t1 = performance.now()
@@ -784,7 +968,8 @@ async function ensureRoom(builder, onEvent, signal) {
 export async function buildImage(o) {
   const builder = await createBuilder(o)
   try {
-    return await builder.build(o)
+    // o.onEvent already hears every build event through the builder: given to build() too, each would come twice
+    return await builder.build({ ...o, onEvent: undefined })
   } finally {
     await builder.close()
   }

package/src/build/context.js

@@ -1,6 +1,7 @@
 // The build context (docs/plans/web-buildkit-impl.md 5.6): turn a file source, a plain object or a tar into the
 // one tar the page sends to the builder. .dockerignore is applied here with a port of moby's patternmatcher, so
 // ignored files are never read; BuildKit applies it again and stays the source of truth.
+import { blobBytes } from '../blob.js'
 import { matcher, parseDockerignore } from './dockerignore.js'
 import { decodeText, scanTar, tarWriter } from './tar.js'
 
@@ -108,7 +109,7 @@ function tarReader(input) {
   const u8 = input instanceof ArrayBuffer ? new Uint8Array(input) : input
   const total = blob ? input.size : u8.length
   const read = blob
-    ? async (o, n) => new Uint8Array(await input.slice(o, o + n).arrayBuffer())
+    ? (o, n) => blobBytes(input.slice(o, o + n))
     : (o, n) => u8.subarray(o, o + n)
   return { blob, u8, total, read }
 }
@@ -175,7 +176,7 @@ async function packTar(input, dockerfile, lim) {
     if (err instanceof ContextError) throw err
     throw new ContextError('bad-source', '', `build context: ${err.message}`)
   }
-  const tar = blob ? new Uint8Array(await input.arrayBuffer()) : u8
+  const tar = blob ? await blobBytes(input) : u8
   return { tar, files, bytes, ignored: 0, dockerfile }
 }
 

package/src/host/disk-worker.js

@@ -71,7 +71,14 @@ function serve(mailbox, disk, closeFiles) {
   let ops = 0
   const post = () => self.postMessage({ t: 'stats', stats: { ...stats, ...disk.usage() } })
   for (;;) {
-    if (Atomics.load(i32, 5) === 1) break
+    const ctl = Atomics.load(i32, 5)
+    if (ctl === 1) break
+    if (ctl === 2) {
+      // tests (gate 9): the Worker ends with no word, as a crashed one does; a vCPU waiting on it gives up after WAIT_MS
+      closeFiles()
+      self.close()
+      return
+    }
     const s = Atomics.load(i32, 1)
     if (s === STATE.GONE) break
     if (s !== STATE.POSTED) {

package/src/host/disk.js

@@ -10,7 +10,7 @@
 //   i32[2] op       0 read, 1 write, 2 flush, 3 discard (blk.c's BLK_OP_*)
 //   i32[3] len      bytes (u32)
 //   i32[4] result   0 or a negative errno
-//   i32[5] control  1: the page asks the Worker to flush and close
+//   i32[5] control  1: the page asks the Worker to flush and close; 2 (tests): it ends at once and answers nothing
 //   f64[3] off      byte offset (bytes 24..31)
 //   data from byte 64, DATA bytes; longer reads and writes go in pieces
 //

package/src/host/jit.js

@@ -69,6 +69,8 @@ const POOL_CHECK_MS = 2000 // D-515: at most one code pool check a thread in thi
 const POOL_PROBE_N = 1500000 // D-515: loop passes of the check: about 2 ms compiled, 40 ms interpreted
 const POOL_SLOW_X = 6 // D-515: a check this many times slower than this thread's fastest means the pool is full
 const POOL_LEARN = 3 / 4 // D-515: the budget from then on, of the modules live when the pool filled
+const POOL_RECOVER_N = 5 // a learned budget grows again after this many checks in a row that found room (10 s at POOL_CHECK_MS)
+const POOL_RECOVER_X = 4 / 3 // ... by this factor each time (the inverse of POOL_LEARN), never past maxLiveBytes
 
 // counters after the header: each a pair of Int32 cells (low word, unsigned; high word).
 // D-419: not BigInt64 cells: Atomics.add on a BigInt64Array returns a new BigInt, garbage a
@@ -404,6 +406,38 @@ export function createJitDoor({ store, memory, ram = null, log = () => {}, compi
 
   let poolNext = 0
   let poolFast = Infinity
+  let poolRoom = 0 // checks in a row that found room, with the shared budget unchanged
+  let budgetSeen = -1 // the shared budget at the last check: a change (another Worker grew or lowered it) starts the count again
+  /**
+   * The engine has had room for POOL_RECOVER_N checks in a row: a budget learned while released code
+   * was not yet freed (WebKit frees it 30-40 s late, D-445) grows back by POOL_RECOVER_X, and is
+   * dropped once it reaches maxLiveBytes (the original cap). Without a cap there is nothing to
+   * return to.
+   */
+  function recoverBudget() {
+    if (!maxLiveBytes) return
+    // the budget is shared and every Worker checks: without this, N Workers would grow it N times as fast
+    const seen = Atomics.load(hdr, H_BUDGET_KIB)
+    if (seen !== budgetSeen) {
+      budgetSeen = seen
+      poolRoom = 0
+    }
+    if (++poolRoom < POOL_RECOVER_N) return
+    poolRoom = 0
+    if (!seen) return
+    lock()
+    try {
+      const cur = Atomics.load(hdr, H_BUDGET_KIB)
+      if (!cur) return
+      const next = Math.ceil(cur * POOL_RECOVER_X)
+      const done = next * 1024 >= maxLiveBytes
+      Atomics.store(hdr, H_BUDGET_KIB, done ? 0 : next)
+      budgetSeen = done ? 0 : next
+      log(`wmd-web: jit: the engine has had room for ${POOL_RECOVER_N} checks; budget ${done ? `back to its cap of ${maxLiveBytes >> 10} KiB` : `${next} KiB`}`)
+    } finally {
+      unlock()
+    }
+  }
   /**
    * D-515: before a new module is compiled. While the engine's code pool is full (codePool, checked
    * at most once each poolEvery) the answer is JIT_LATER: the guest interprets the blocks and asks
@@ -423,8 +457,10 @@ export function createJitDoor({ store, memory, ram = null, log = () => {}, compi
     if (ms > poolFast * POOL_SLOW_X) ms = Math.min(ms, codePool()) // not a pause of this thread
     poolFast = Math.min(poolFast, Math.max(ms, 0.05))
     const full = ms > poolFast * POOL_SLOW_X
+    if (full) poolRoom = 0
     const at = () => new Date().toISOString().slice(11, 19)
     if (!full) {
+      recoverBudget()
       if (was && Atomics.compareExchange(hdr, H_POOL_FULL, 1, 0) === 1) {
         add('poolBack')
         log(`wmd-web: jit: the engine compiles code again (${at()})`)

package/src/image-files.js

@@ -4,6 +4,7 @@
 // `USER name`, devcontainer `remoteUser` or `--chown=app:app` (resolveUser). W66: another image's files,
 // for `COPY --from` and multi-stage builds (walk).
 
+import { blobBytes } from './blob.js'
 import { ENOENT, FsError } from './fs/errno.js'
 import { Overlay } from './fs/overlay.js'
 import { Pack, bytesSource } from './fs/pack.js'
@@ -19,7 +20,7 @@ async function layerBytes(l, i, fetchFn) {
   }
   if (l instanceof Uint8Array) return l
   if (l instanceof ArrayBuffer || (typeof SharedArrayBuffer !== 'undefined' && l instanceof SharedArrayBuffer)) return new Uint8Array(l)
-  if (typeof Blob !== 'undefined' && l instanceof Blob) return new Uint8Array(await l.arrayBuffer())
+  if (typeof Blob !== 'undefined' && l instanceof Blob) return blobBytes(l)
   throw new TypeError(`layer ${i}: a URL, bytes or a Blob, not ${Object.prototype.toString.call(l)}`)
 }