@daugasauron/dolly 0.1.4 → 0.1.5-rc.0

package/src/process-supervisor.mjs

@@ -13,6 +13,7 @@ import {
   DOLLY_PROCESS_SPAWN_INTERACTIVE,
 } from "./process-constants.mjs";
 import { hex } from "./static-asset.mjs";
+import { DOLLY_FS_BLOCK_SIZE } from "../host/runtime/abi.mjs";
 
 const encoder = new TextEncoder();
 const spawnHeaderSize = DOLLY_PROCESS_SIZEOF.dolly_process_spawn_request;
@@ -26,23 +27,58 @@ const interruptedSystemCall = -BigInt(DOLLY_ERRNO.EINTR);
 // the memory of a 61st process of 8 GiB was refused.
 const processWorkerLimit = 56;
 const processMemoryBudgetBytes = 4 * 1024 * 1024 * 1024;
-// Chromium releases the address space of a terminated Worker's memory some
-// time after terminate(): a memory it cannot reserve is asked for again for
-// a while before the process is refused. A spawn the count refuses waits as
-// long, like any blocking call, asked again whenever a process ends and at
-// each service tick; then it fails EAGAIN, a full process table's answer. A
-// spawn the budget refuses waits until a process ends, however long: make and
-// Cargo end a build at a spawn's EAGAIN. So a pipeline started over the
-// budget waits forever if its first stage fills the pipe before the next
-// stage is admitted; a signal (Ctrl+C) ends the wait.
-const reservationRetryMilliseconds = 50;
+// A spawn the count refuses waits as long as a Worker asks again for a memory
+// Chromium cannot reserve yet (src/process-worker.mjs), like any blocking
+// call, asked again whenever a process ends and at each service tick; then it
+// fails EAGAIN, a full process table's answer. A spawn the budget refuses
+// waits until a process ends, however long: make and Cargo end a build at a
+// spawn's EAGAIN. So a pipeline started over the budget waits forever if its
+// first stage fills the pipe before the next stage is admitted; a signal
+// (Ctrl+C) ends the wait.
 const reservationPatienceMilliseconds = 5000;
+// A Worker ended mid-copy into its mailbox finishes the copy within
+// microseconds; one that holds the mailbox this long keeps it.
+const mailboxSealMilliseconds = 1000;
 const interruptGraceMilliseconds = 500;
 const compiledModuleCacheEntries = 64;
 const compiledModuleCacheBytes = 256 * 1024 * 1024;
 // File versions known to hold a cached module's bytes (dolly_process_image_version).
 const compiledVersionEntries = 1024;
 const compilationNoticeMilliseconds = 250;
+// A large executable's module is compiled and held in a Worker of its own,
+// which goes once neither the cache nor a process holds the module: a Module
+// held here would go only at this Worker's next collection, which compiled
+// code does not hasten. Four 96 MB executables launched in turn kept five
+// compiled in Firefox, where the cache held two. An idle Worker costs about
+// 2.5 MB, so a small module is compiled and held here.
+const heldModuleBytes = 16 * 1024 * 1024;
+const holderUrl = URL.createObjectURL(new Blob([`let module;
+onmessage = async ({ data: { bytes, port } }) => {
+  if (port) return port.postMessage(module);
+  try {
+    module = await WebAssembly.compile(bytes);
+    postMessage(null);
+  } catch (error) {
+    postMessage(String(error?.message ?? error));
+  } finally {
+    bytes.transfer(0);
+  }
+};`], { type: "text/javascript" }));
+// Each holder until it answers: Firefox ends a Worker that only its own
+// handlers reach once its loop is idle, as after it posts its answer, and
+// that answer is lost.
+const compiling = new Set();
+// The Worker holding the module compiled from BYTES, which it takes.
+function compileHeld(bytes) {
+  const holder = new Worker(holderUrl, { name: "dolly-module" });
+  compiling.add(holder);
+  return new Promise((resolve, reject) => {
+    const fail = message => { holder.terminate(); reject(new Error(message)); };
+    holder.onmessage = ({ data }) => data === null ? resolve(holder) : fail(data);
+    holder.onerror = event => { event.preventDefault(); fail(event.message || "module Worker failed"); };
+    holder.postMessage({ bytes: bytes.buffer }, [bytes.buffer]);
+  }).finally(() => compiling.delete(holder));
+}
 const largeInteractiveProcessBytes = 128 * 1024 * 1024;
 const workerReclamationMilliseconds = 500;
 
@@ -126,15 +162,13 @@ function encodeSpawn(arguments_, flags) {
   return packet;
 }
 
-function createProcessMemory({ initial, maximum }) {
-  return new WebAssembly.Memory({
-    initial,
-    maximum,
-    shared: true,
-    address: "i64",
-  });
-}
-
+// A process's memory lives in its own Workers alone, never in this one: a
+// shared memory's pages go when the last agent holding it drops it, and a
+// living Worker drops one only when it collects garbage, which a small heap
+// rarely needs (tasks/20261101-102515-retained-process-memory). So each
+// Worker makes or receives the memory, holds the gate and copies its requests
+// into a mailbox of its own in kernel memory, and the memory goes with the
+// Worker. This Worker only names that mailbox to the kernel.
 export class DollyProcessSupervisor {
   constructor(kernel, kernelMemory, gateModule, workerUrl, processContract, hostAbi, serviceHost, threadContract, threadHost, processModules) {
     if (!(kernelMemory instanceof WebAssembly.Memory) ||
@@ -227,7 +261,7 @@ export class DollyProcessSupervisor {
     if (this.processes.size >= processWorkerLimit) return "count";
     let committed = 0, fanning = false;
     for (const process of this.processes.values()) {
-      committed += process.memory?.buffer.byteLength ?? 0;
+      committed += process.memoryBytes;
       fanning ||= process.parent === parent && !process.retiring;
     }
     return committed < processMemoryBudgetBytes || !fanning ? null : "budget";
@@ -260,8 +294,8 @@ export class DollyProcessSupervisor {
       throw new Error(`kernel supplied invalid process ${pid}`);
     }
     const process = {
-      pid, parent, resolve, reject, worker: null, gate: null, control: null,
-      memory: null, messageHandler: null, errorHandler: null,
+      pid, parent, resolve, reject, worker: null, control: null, mailbox: 0,
+      memoryBytes: 0, messageHandler: null, errorHandler: null,
       messageErrorHandler: null, started: false,
       failure: null, interactive: (flags & DOLLY_PROCESS_SPAWN_INTERACTIVE) !== 0, retiring: false,
       interruptTimer: null, deadlineTimer: null, retirementTimer: null, lastSignal: 0,
@@ -317,15 +351,18 @@ export class DollyProcessSupervisor {
   }
 
   #deliverSignal(process, signalNumber = DOLLY_PROCESS_SIGINT) {
-    if (!process || process.retiring || this.processes.get(process.pid) !== process) return false;
+    if (!process || process.retiring || this.processes.get(process.pid) !== process) return;
     const result = this.kernel.dolly_process_signal(process.pid, signalNumber);
     process.lastSignal = signalNumber;
-    // Resize notification is not an interrupt request and must never acquire
-    // a forced-termination deadline, including before program entry.
-    if (signalNumber === DOLLY_PROCESS_SIGWINCH && (result !== 0 || !process.started)) return result === 0;
-    if (signalNumber === DOLLY_PROCESS_SIGKILL || result !== 0 || !process.started) {
-      return this.#forceExit(process.pid, 128 + signalNumber, signalNumber);
+    // The kernel decides what the signal does: it may have ended the process.
+    if (this.kernel.dolly_process_exited(process.pid) !== 0) {
+      this.#retire(process);
+      return;
     }
+    // A program not yet entered has no call to interrupt; what the kernel
+    // keeps for it, it takes when it starts. One that ignores the signal
+    // (result 1) has nothing to answer.
+    if (result !== 0 || !process.started) return;
     const receiver = process.threaded
       ? [...process.threads.values()].sort((a, b) => a.tid - b.tid)[0] : process;
     const deferred = this.deferred.get(receiver);
@@ -342,7 +379,6 @@ export class DollyProcessSupervisor {
       this.#signal(receiver, message.sequence, interruptedSystemCall);
     }
     if (signalNumber !== DOLLY_PROCESS_SIGWINCH) this.#armInterrupt(process, signalNumber);
-    return true;
   }
 
   // A program that does not finish a delivered signal in time is ended by it.
@@ -395,15 +431,18 @@ export class DollyProcessSupervisor {
   }
 
   async #compileProcess(bytes) {
+    const byteLength = bytes.byteLength;
     const digest = hex(await crypto.subtle.digest("SHA-256", bytes));
-    const key = `${bytes.byteLength}:${digest}`;
+    const key = `${byteLength}:${digest}`;
     if (this.compiledModules.has(key)) return this.#cachedModule(key);
+    // What this module will replace in the cache goes before it is compiled.
+    if (byteLength <= compiledModuleCacheBytes) this.#evict(byteLength);
 
     const started = performance.now();
     let noticeShown = false;
     const notice = setTimeout(() => {
       noticeShown = true;
-      const mebibytes = (bytes.byteLength / (1024 * 1024)).toFixed(1);
+      const mebibytes = (byteLength / (1024 * 1024)).toFixed(1);
       this.#writeTerminal(
         `\r\ndolly: preparing ${mebibytes} MiB WebAssembly executable for this session...\r\n`,
       );
@@ -422,7 +461,7 @@ export class DollyProcessSupervisor {
       checkHostAbi(requirements, this.hostAbi);
       threaded = validateThreadProfile(parsed, requirements, this.threadContract);
       largeStack = parsed.customSections.includes("dolly.process.stack");
-      module = await WebAssembly.compile(bytes);
+      module = byteLength >= heldModuleBytes ? await compileHeld(bytes) : await WebAssembly.compile(bytes);
       processInterface = { imports: parsed.imports, exports: parsed.exports,
         castEmulationParameters: parsed.castEmulationParameters };
       prepared = true;
@@ -436,21 +475,35 @@ export class DollyProcessSupervisor {
         );
       }
     }
-    const compiled = { key, module, memoryRequirements, processInterface, threaded, largeStack, requirements, byteLength: bytes.byteLength };
-    if (bytes.byteLength <= compiledModuleCacheBytes) {
+    const compiled = { key, module, memoryRequirements, processInterface, threaded, largeStack, requirements, byteLength };
+    if (byteLength <= compiledModuleCacheBytes) {
+      // An exec's compile may have run beside this one.
+      this.#evict(byteLength);
       this.compiledModules.set(key, compiled);
-      this.compiledModuleBytes += bytes.byteLength;
-      while (this.compiledModules.size > compiledModuleCacheEntries ||
-             this.compiledModuleBytes > compiledModuleCacheBytes) {
-        const oldestKey = this.compiledModules.keys().next().value;
-        const oldest = this.compiledModules.get(oldestKey);
-        this.compiledModules.delete(oldestKey);
-        this.compiledModuleBytes -= oldest.byteLength;
-      }
+      this.compiledModuleBytes += byteLength;
     }
     return compiled;
   }
 
+  // Evicts the least recently used modules until one of byteLength fits.
+  #evict(byteLength) {
+    while (this.compiledModules.size && (this.compiledModules.size >= compiledModuleCacheEntries ||
+           this.compiledModuleBytes + byteLength > compiledModuleCacheBytes)) {
+      const [oldestKey, oldest] = this.compiledModules.entries().next().value;
+      this.compiledModules.delete(oldestKey);
+      this.compiledModuleBytes -= oldest.byteLength;
+      this.#release(oldest);
+    }
+  }
+
+  // Ends the Worker of a held module (heldModuleBytes) that neither the cache
+  // nor a process holds.
+  #release(compiled) {
+    if (!(compiled.module instanceof Worker) || this.compiledModules.get(compiled.key) === compiled) return;
+    for (const process of this.processes.values()) if (process.compiled === compiled) return;
+    compiled.module.terminate();
+  }
+
   #writeTerminal(text) {
     const bytes = encoder.encode(text);
     const address = this.kernel.malloc(BigInt(Math.max(1, bytes.byteLength)));
@@ -470,42 +523,38 @@ export class DollyProcessSupervisor {
   async #launchPending() {
     for (let pid; (pid = this.kernel.dolly_process_next_launch()) !== 0;) {
       const process = this.processes.get(pid);
+      let compiled;
       try {
         if (!process) throw new Error(`kernel supplied unregistered process ${pid}`);
-        const compiled = await this.#compiledImage(pid);
-        if (!this.#canLaunch(process)) continue;
-        const local = this.#admit(compiled);
-        for (let waited = 0; ; waited += reservationRetryMilliseconds) {
-          try {
-            this.#start(process, compiled, local);
-            break;
-          } catch (error) {
-            if (!(error instanceof RangeError) || process.memory || waited >= reservationPatienceMilliseconds) throw error;
-          }
-          await new Promise(resolve => setTimeout(resolve, reservationRetryMilliseconds));
-          if (!this.#canLaunch(process)) break;
-        }
+        compiled = await this.#compiledImage(pid);
+        if (this.#canLaunch(process)) this.#start(process, compiled, this.#admit(compiled));
       } catch (error) {
         // A failed launch ends only its own process; exiting it in the kernel
         // also stops next_launch from naming it again.
         if (process) this.#fail(process, error);
         else this.kernel.dolly_process_worker_exited(pid, 126, 0);
+      } finally {
+        if (compiled) this.#release(compiled);
       }
     }
   }
 
-  // The bytes of the executable the kernel opened for a process, which it then closes.
+  // The bytes of the executable the kernel opened for a process, copied from
+  // the file's own blocks (the kernel makes no copy), which it then closes.
+  // One copy: Web Crypto rejects views of shared memory.
   #takeImage(pid) {
-    const read = this.kernel.dolly_process_image_read(pid);
-    if (read !== 0) throw new Error(`cannot read the executable of process ${pid}: ${read}`);
-    const address = Number(this.kernel.dolly_process_image_address(pid));
     const size = Number(this.kernel.dolly_process_image_size(pid));
-    if (!Number.isSafeInteger(address) || !Number.isSafeInteger(size) ||
-        address <= 0 || size < 8 || address > this.kernelMemory.buffer.byteLength - size) {
-      throw new Error(`kernel supplied an invalid executable for process ${pid}`);
-    }
+    if (size < 0) throw new Error(`cannot read the executable of process ${pid}: ${size}`);
+    if (!Number.isSafeInteger(size) || size < 8) throw new Error(`kernel supplied an invalid executable for process ${pid}`);
     const bytes = new Uint8Array(size);
-    bytes.set(new Uint8Array(this.kernelMemory.buffer, address, size));
+    for (let offset = 0; offset < size; offset += DOLLY_FS_BLOCK_SIZE) {
+      const length = Math.min(DOLLY_FS_BLOCK_SIZE, size - offset);
+      const address = Number(this.kernel.dolly_process_image_block(pid, BigInt(offset / DOLLY_FS_BLOCK_SIZE)));
+      if (!Number.isSafeInteger(address) || address <= 0 || address > this.kernelMemory.buffer.byteLength - length) {
+        throw new Error(`kernel supplied an invalid executable for process ${pid}`);
+      }
+      bytes.set(new Uint8Array(this.kernelMemory.buffer, address, length), offset);
+    }
     if (this.kernel.dolly_process_image_consumed(pid) !== 0) {
       throw new Error(`kernel did not release executable ${pid}`);
     }
@@ -513,26 +562,19 @@ export class DollyProcessSupervisor {
   }
 
   // Checked at every launch, not only when compiled: nena@0 may have disabled
-  // a module since. The modules served in this executable's own Worker each
-  // hold one Worker's state, such as its function table: one goes with threads
-  // only where it serves every thread. Returns those modules.
+  // a module since. Returns the modules served in this executable's own
+  // Workers, each thread's among them (dso@0, stack@0).
   #admit({ requirements, threaded }) {
     checkHostAbi(requirements, this.hostAbi);
     const local = [...requirements.keys()].filter(requirement => this.processModules.has(requirement));
-    const unthreaded = local.filter(requirement => !this.processModules.get(requirement).threads);
-    if (threaded && unthreaded.length) throw new TypeError(`a program using threads@0 cannot use ${unthreaded.join(", ")}`);
     if (threaded && !this.threadHost) throw new Error("threads@0 is unavailable");
     return local;
   }
 
-  #start(process, { module, memoryRequirements, processInterface, threaded, largeStack }, local) {
-    const memory = createProcessMemory(memoryRequirements);
-    process.memory = memory;
-    const gate = new WebAssembly.Instance(this.gateModule, {
-      process: { memory },
-      kernel: { memory: this.kernelMemory },
-    });
-    Object.assign(process, { gate, module, processInterface, threaded, largeStack, local });
+  #start(process, compiled, local) {
+    const { memoryRequirements, processInterface, threaded, largeStack } = compiled;
+    Object.assign(process, { memoryRequirements, memoryBytes: Number(memoryRequirements.initial) * 65536,
+      maximumBytes: Number(memoryRequirements.maximum) * 65536, compiled, processInterface, threaded, largeStack, local });
     if (threaded) {
       const tid = this.kernel.dolly_threads_attach(process.pid);
       if (tid <= 0) throw new Error(`kernel could not reserve main thread: ${tid}`);
@@ -551,10 +593,12 @@ export class DollyProcessSupervisor {
     const execution = { thread, message };
     process.execution = execution;
     let compiled, local;
+    const unused = () => compiled && this.#release(compiled);
     try {
       compiled = await this.#compiledImage(process.pid);
       if (process.execution === execution) local = this.#admit(compiled);
     } catch (error) {
+      unused();
       if (process.execution !== execution) return;
       process.execution = null;
       const line = encoder.encode(`dolly: process ${process.pid} cannot run its new program: ` +
@@ -564,10 +608,11 @@ export class DollyProcessSupervisor {
       this.#signal(thread, message.sequence, -BigInt(DOLLY_ERRNO.ENOEXEC));
       return;
     }
-    if (process.execution !== execution) return;
+    if (process.execution !== execution) return unused();
     process.execution = null;
     const signalAtStart = this.kernel.dolly_process_exec_commit(process.pid);
     if (signalAtStart < 0) {
+      unused();
       this.#fail(process, new Error(`kernel rejected process ${process.pid} exec: ${signalAtStart}`));
       return;
     }
@@ -577,39 +622,60 @@ export class DollyProcessSupervisor {
     const delay = this.#reclamationDeadline(process) - performance.now();
     if (process.interruptTimer !== null) clearTimeout(process.interruptTimer);
     process.interruptTimer = null;
-    this.#disposeWorker(process);
+    this.#disposeWorker(process, compiled);
     Object.assign(process, { started: false, threaded: false, tid: 0, threads: new Map(), signalAtStart });
     // Its descriptors with CLOEXEC are closed: a reader of their pipes goes on now.
     this.#deliverRaised();
     if (this.kernel.dolly_process_take_wakeup()) this.serviceDeferred();
     const start = () => {
-      if (!this.#canLaunch(process)) return;
-      try { this.#start(process, compiled, local); } catch (error) { this.#fail(process, error); }
+      try {
+        if (this.#canLaunch(process)) this.#start(process, compiled, local);
+      } catch (error) { this.#fail(process, error); } finally { this.#release(compiled); }
     };
     if (delay > 0) setTimeout(start, delay);
     else start();
   }
 
-  #launchWorker(process, thread, argument = undefined) {
-    const { pid, module, memory, processInterface, threaded, local } = process;
-    const control = new SharedArrayBuffer(16);
+  // port: for a thread, the channel the spawning Worker sends the memory on.
+  #launchWorker(process, thread, argument = undefined, port = undefined) {
+    const { pid, compiled: { module }, memoryRequirements, processInterface, threaded, local } = process;
     const name = `dolly-process-${pid}-thread-${thread.tid}`;
     const worker = threaded ? this.threadHost.create(pid, this.workerUrl, name)
       : new Worker(this.workerUrl, { type: "module", name });
     if (!worker) throw new Error("thread Worker capacity exhausted");
+    // Words: the call sequence, the answered sequence, the result's two
+    // halves, the copy word (src/process-worker.mjs copyRequest) and a
+    // thread's start word (#disposeThread).
+    const control = new SharedArrayBuffer(32);
+    thread.mailbox = Number(this.kernel.malloc(BigInt(packetLimit)));
+    if (!thread.mailbox) { worker.terminate(); throw new Error("kernel memory for a process mailbox"); }
+    if (this.kernel.dolly_process_worker_mailbox(pid, thread.tid, BigInt(thread.mailbox)) !== 0) {
+      worker.terminate();
+      this.kernel.free(BigInt(thread.mailbox));
+      thread.mailbox = 0;
+      throw new Error(`kernel does not know thread ${thread.tid} of process ${pid}`);
+    }
     thread.worker = worker;
     thread.control = new Int32Array(control);
     thread.messageHandler = event => this.#message(process, event.data, thread);
-    thread.errorHandler = event => this.#fail(process, new Error(event.message || `${name} failed`));
+    // Cancelled, or the browser reports the Worker's uncaught error again in
+    // this Worker, whose end it would be (src/browser.mjs).
+    thread.errorHandler = event => { event.preventDefault(); this.#fail(process, new Error(event.message || `${name} failed`)); };
     thread.messageErrorHandler = () => this.#fail(process, new Error(`${name} message could not be decoded`));
     worker.addEventListener("message", thread.messageHandler);
     worker.addEventListener("error", thread.errorHandler, { once: true });
     worker.addEventListener("messageerror", thread.messageErrorHandler, { once: true });
     // local: the modules this executable records that are served in its own
     // Worker (host/dso), each as the bundle it imports and its configuration.
-    worker.postMessage({ type: "configure", pid, tid: thread.tid, argument, threaded, module, memory, control,
+    // A held module comes from its Worker, on a channel of this Worker's own.
+    const channel = module instanceof Worker ? new MessageChannel() : null;
+    channel && module.postMessage({ port: channel.port1 }, [channel.port1]);
+    worker.postMessage({ type: "configure", pid, tid: thread.tid, argument, threaded, control,
+      module: channel ? channel.port2 : module,
+      memory: memoryRequirements, port, mailbox: thread.mailbox, kernelMemory: this.kernelMemory, gateModule: this.gateModule,
       promisingEntry: this.promisingEntry && process.largeStack, stackDepths: this.stackDepths,
-      clockOrigin: performance.timeOrigin, processInterface, local: local.map(requirement => this.processModules.get(requirement)) });
+      clockOrigin: performance.timeOrigin, processInterface, local: local.map(requirement => this.processModules.get(requirement)) },
+    [port, channel?.port2].filter(Boolean));
   }
 
   #canLaunch(process) {
@@ -653,8 +719,11 @@ export class DollyProcessSupervisor {
     this.deferred.delete(thread);
   }
 
+  // The thread's Worker has copied the request into its mailbox (thread.mailbox)
+  // and reads the response from it once answered; what this reads of the
+  // mailbox, it reads before it answers.
   #syscall(process, message, retry = false, thread = process) {
-    const refusedSince = this.deferred.get(thread)?.refusedSince;
+    const pending = this.deferred.get(thread);
     this.#clearDeferred(thread);
     const values = [
       message.sequence,
@@ -663,36 +732,40 @@ export class DollyProcessSupervisor {
       message.requestSize,
       message.responseAddress,
       message.responseCapacity,
+      message.memoryBytes,
     ];
+    const spawnsThread = process.threaded && message.operation === DOLLY_THREAD_SPAWN;
     if (message.pid !== process.pid || message.tid !== thread.tid ||
         values.some((value) => !Number.isSafeInteger(value) || value < 0) ||
         message.requestSize > packetLimit || message.responseCapacity > packetLimit ||
+        spawnsThread !== (message.port instanceof MessagePort) ||
         Atomics.load(thread.control, 0) !== message.sequence ||
         Atomics.load(thread.control, 1) === message.sequence) {
       this.#fail(process, new Error(`process ${process.pid} sent an invalid syscall`));
       return;
     }
+    // The size the budget counts: a threaded process keeps the one it reported
+    // before its memory took its maximum (src/process-worker.mjs).
+    if (!(process.threaded && message.memoryBytes >= process.maximumBytes)) process.memoryBytes = message.memoryBytes;
     let result;
     let signalDelivery;
     let threadArgument;
+    let portTaken = false;
     try {
-      process.gate.exports.request(
-        BigInt(message.requestAddress),
-        BigInt(this.mailboxAddress),
-        BigInt(message.requestSize),
-      );
+      // A deferred call is retried against the request still in the mailbox:
+      // a handler that defers has written nothing (src/process-kernel.h).
+      const mailbox = () => new DataView(this.kernelMemory.buffer, thread.mailbox, 8);
       if (message.operation === DOLLY_THREAD_SPAWN && message.requestSize === 8)
-        threadArgument = new DataView(this.kernelMemory.buffer, this.mailboxAddress, 8).getBigUint64(0, true);
+        threadArgument = mailbox().getBigUint64(0, true);
       const args = [process.pid, message.operation, BigInt(message.requestSize), BigInt(message.responseCapacity)];
       if (process.threaded) args.splice(1, 0, thread.tid);
       const waiting = message.operation === DOLLY_PROCESS_SPAWN ? this.#spawnWait(process.pid) : null;
-      const now = performance.now(), since = waiting === "count" ? refusedSince ?? now : undefined;
+      const now = performance.now(), since = waiting === "count" ? pending?.refusedSince ?? now : undefined;
       if (waiting === "budget" || waiting === "count" && now - since < reservationPatienceMilliseconds) {
         this.deferred.set(thread, { process, thread, message, timer: null, refusedSince: since });
         return;
       }
-      const exhausted = waiting === "count" ||
-        process.threaded && message.operation === DOLLY_THREAD_SPAWN && !this.threadHost?.available(process.pid);
+      const exhausted = waiting === "count" || spawnsThread && !this.threadHost?.available(process.pid);
       // Firefox can enter a direct Wasm call twice. Use the generic call path
       // so a completed syscall is never replayed against its response packet.
       result = exhausted ? -BigInt(DOLLY_ERRNO.EAGAIN)
@@ -723,20 +796,14 @@ export class DollyProcessSupervisor {
         if (result > BigInt(message.responseCapacity)) {
           throw new Error(`kernel overfilled process ${process.pid} response`);
         }
-        process.gate.exports.response(
-          BigInt(this.mailboxAddress),
-          BigInt(message.responseAddress),
-          result,
-        );
         if (message.operation === DOLLY_THREAD_SPAWN) {
           if (result !== 8n || threadArgument === undefined) throw new Error("invalid kernel thread response");
-          const response = new DataView(this.kernelMemory.buffer, this.mailboxAddress, 8);
-          const tid = response.getUint32(0, true);
-          if (!tid || tid > 0x7fffffff || response.getUint32(4, true) || process.threads.has(tid))
+          const tid = mailbox().getUint32(0, true);
+          if (!tid || tid > 0x7fffffff || mailbox().getUint32(4, true) || process.threads.has(tid))
             throw new Error("invalid kernel thread identity");
-          const child = { tid, worker: null, control: null };
+          const child = { tid, worker: null, control: null, mailbox: 0 };
           process.threads.set(tid, child);
-          try { this.#launchWorker(process, child, threadArgument); }
+          try { this.#launchWorker(process, child, threadArgument, message.port); portTaken = true; }
           catch (error) {
             this.#disposeThread(process, child);
             this.kernel.dolly_threads_unstarted(process.pid, tid);
@@ -745,33 +812,32 @@ export class DollyProcessSupervisor {
         }
         if (message.operation === DOLLY_PROCESS_SPAWN) {
           if (result !== 8n) throw new Error("invalid kernel spawn response");
-          const response = new DataView(this.kernelMemory.buffer, this.mailboxAddress, 8);
-          const pid = response.getUint32(0, true);
-          if (pid === 0 || pid > 0x7fffffff || response.getUint32(4, true) !== 0) {
+          const pid = mailbox().getUint32(0, true);
+          if (pid === 0 || pid > 0x7fffffff || mailbox().getUint32(4, true) !== 0) {
             throw new Error("invalid kernel spawn identity");
           }
           this.#registerProcess(pid, process.pid);
         }
         if (message.operation === DOLLY_PROCESS_SIGNAL) {
           if (result !== 8n) throw new Error("invalid kernel signal response");
-          const response = new DataView(this.kernelMemory.buffer, this.mailboxAddress, 8);
-          signalDelivery = { pid: response.getUint32(0, true), signal: response.getUint32(4, true) };
+          signalDelivery = { pid: mailbox().getUint32(0, true), signal: mailbox().getUint32(4, true) };
           if (signalDelivery.pid === 0 || signalDelivery.pid > 0x7fffffff) {
             throw new Error("invalid kernel signal target");
           }
         }
+        if (message.operation === DOLLY_PROCESS_SIGNAL_ACKNOWLEDGE && result === 4n &&
+            mailbox().getInt32(0, true) === 0 && process.interruptTimer !== null) {
+          clearTimeout(process.interruptTimer);
+          process.interruptTimer = null;
+        }
       }
     } catch (error) {
+      message.port?.close();
       this.#fail(process, error);
       return;
     }
+    if (!portTaken) message.port?.close();
     this.#signal(thread, message.sequence, result);
-    if (message.operation === DOLLY_PROCESS_SIGNAL_ACKNOWLEDGE && result === 4n &&
-        new DataView(this.kernelMemory.buffer, this.mailboxAddress, 4).getInt32(0, true) === 0 &&
-        process.interruptTimer !== null) {
-      clearTimeout(process.interruptTimer);
-      process.interruptTimer = null;
-    }
     if (signalDelivery?.signal) {
       this.#deliverSignal(this.processes.get(signalDelivery.pid), signalDelivery.signal);
     }
@@ -826,7 +892,7 @@ export class DollyProcessSupervisor {
 
   #reclamationDeadline(process) {
     return Math.max(process.reclamationDeadline,
-      process.interactive && (process.memory?.buffer.byteLength ?? 0) >= largeInteractiveProcessBytes
+      process.interactive && process.memoryBytes >= largeInteractiveProcessBytes
         ? performance.now() + workerReclamationMilliseconds : 0);
   }
 
@@ -949,10 +1015,27 @@ export class DollyProcessSupervisor {
       // for two seconds. Answered with the deferred result, which no program
       // is handed, a waiting Worker leaves Wasm and closes itself.
       if (thread.control) this.#signal(thread, Atomics.load(thread.control, 0), deferredResult);
+      // Chrome keeps what was posted to a Worker terminated before it runs, the
+      // memory a thread's Worker is sent among it, until this Worker collects
+      // garbage. One that has not started (control word 5 still 0) is told so
+      // and ends itself (src/process-worker.mjs processMemory).
+      const unstarted = thread !== process && Atomics.compareExchange(thread.control, 5, 0, 2) === 0;
+      if (unstarted) Atomics.notify(thread.control, 5);
       // threads@0 disabled since has already terminated its Workers.
-      if (process.threaded && this.threadHost) this.threadHost.release(thread.worker);
+      if (process.threaded && this.threadHost) this.threadHost.release(thread.worker, !unstarted);
       else thread.worker.terminate();
     }
+    // The Worker copies a request into its mailbox under control word 4
+    // (src/process-worker.mjs copyRequest); sealed, it copies no more, and the
+    // mailbox goes back to the kernel. Termination has no completion event: a
+    // Worker ended inside a copy finishes it first.
+    if (thread.mailbox) {
+      this.kernel.dolly_process_worker_mailbox(process.pid, thread.tid, 0n);
+      while (Atomics.compareExchange(thread.control, 4, 0, 2) === 1 &&
+             Atomics.wait(thread.control, 4, 1, mailboxSealMilliseconds) !== "timed-out");
+      if (Atomics.load(thread.control, 4) === 2) this.kernel.free(BigInt(thread.mailbox));
+    }
+    thread.mailbox = 0;
     thread.worker = null;
     thread.control = null;
     thread.messageHandler = thread.errorHandler = thread.messageErrorHandler = null;
@@ -967,10 +1050,13 @@ export class DollyProcessSupervisor {
     this.#disposeWorker(process);
   }
 
-  #disposeWorker(process) {
+  // next: the module an exec starts next, which stays even if it is this one.
+  #disposeWorker(process, next = null) {
     for (const thread of [...process.threads.values()]) this.#disposeThread(process, thread);
     this.#disposeThread(process, process);
-    process.gate = process.memory = process.module = process.processInterface = null;
+    const { compiled } = process;
+    process.compiled = process.processInterface = null;
+    if (compiled && compiled !== next) this.#release(compiled);
   }
 
   serviceDeferred() {
@@ -993,6 +1079,7 @@ export class DollyProcessSupervisor {
     }
     this.processes.clear();
     this.deferred.clear();
+    for (const compiled of this.compiledModules.values()) if (compiled.module instanceof Worker) compiled.module.terminate();
     this.compiledModules.clear();
     this.compiledModuleBytes = 0;
     this.compiledVersions.clear();

package/src/runtime-worker.mjs

@@ -1,17 +1,16 @@
 // The runtime Worker boot() starts: it restores an image of the image directory
 // from its packs into a new kernel and runs its ENTRY (worker-image.mjs).
-import { DOLLY_IMAGES } from "./images.mjs";
-import { loadPackagedSnapshotMetadata, streamPackagedSystemSnapshot } from "./packaged-snapshot.mjs";
+import { imageDefinition } from "./images.mjs";
+import { loadPackagedSnapshotMetadata, streamPackagedSystemSnapshot, verifyVisibleRecipes } from "./packaged-snapshot.mjs";
 import { siteReference, hex } from "./static-asset.mjs";
 import { checkedMemoryRange, runImage } from "./worker-image.mjs";
 
-await runImage(async ({ image, mode, files: given = {} }, host) => {
-  const definition = DOLLY_IMAGES.find(candidate => candidate.image === image);
-  if (!definition || mode !== "snapshot") throw new Error("invalid Dolly image selection");
-  const metadata = await loadPackagedSnapshotMetadata(image);
-  const declared = metadata.hostRequirements ?? definition.hostRequirements ?? [];
-  host.require(declared);
-  host.admit(declared);
+await runImage(async ({ image, files: given = {} }, host) => {
+  const definition = imageDefinition(image);
+  if (!definition) throw new Error("invalid Dolly image selection");
+  const [metadata] = await Promise.all([loadPackagedSnapshotMetadata(image), verifyVisibleRecipes(image)]);
+  host.require(definition.hostRequirements);
+  host.admit(definition.hostRequirements);
   return {
     name: image,
     prepare({ files }) {
@@ -31,7 +30,7 @@ await runImage(async ({ image, mode, files: given = {} }, host) => {
       if (kernel.dolly_bootstrap_snapshot_begin(BigInt(metadata.byteLength)) !== 0)
         throw new Error("Dolly snapshot stream initialization failed");
       let loaded = 0, reported = 0;
-      await streamPackagedSystemSnapshot(image, metadata, bytes => {
+      await streamPackagedSystemSnapshot(metadata, bytes => {
         for (let offset = 0; offset < bytes.length; offset += capacity) {
           const chunk = bytes.subarray(offset, offset + capacity);
           staging().set(chunk);