@nimbus-sh/fabric 0.1.0

package/dist/loader-pool.js

@@ -0,0 +1,666 @@
+/**
+ * loader-pool.ts — Nimbus loader-isolate pool based on cloudflare-parallel.
+ *
+ * Adds Nimbus-specific behavior to the upstream pool design:
+ *   1. **Stable-slot isolate reuse**. Upstream's #counter++ gives every
+ *      dispatch a fresh isolate — fine for one-off AI calls, terrible for
+ *      running 67 npm tarball extractions (cold-start dominates). We pin
+ *      each job to `slot = cursor % concurrency` and use stable loader
+ *      IDs `nfp:${fnHash}:slot-${i}:g${generation}`, so a pool of
+ *      concurrency=4 keeps at most 4 warm isolates rather than N fresh ones.
+ *   2. **Nimbus defaults**: compatibilityDate = CF_COMPAT_DATE (matches
+ *      the supervisor worker), compatibilityFlags = ['nodejs_compat'],
+ *      globalOutbound = undefined (inherit parent network so the facet can
+ *      reach https://registry.npmjs.org without a proxy binding).
+ *   3. **Supervisor autoinjection**. The pool grabs the embedder's
+ *      registered supervisor entrypoint stub (see `supervisorEntrypoint` in
+ *      ctx-exports.ts) and forwards it as `env.SUPERVISOR` to every facet,
+ *      same pattern as git-network-facet.ts. Callers can add more bindings
+ *      via `extraBindings`.
+ *   4. **Fail-loud defaults**: timeout 60s, retries 0, onError 'throw'.
+ *      Caller opts in to leniency.
+ *
+ * The vendored directory contains only the upstream serialization, error,
+ * and binding types used by this implementation.
+ */
+import { CF_COMPAT_DATE } from '@nimbus-sh/core/constants.js';
+import { supervisorEntrypoint } from './ctx-exports.js';
+import { disposeRpcResource } from '@nimbus-sh/core/_shared/rpc-dispose.js';
+import { serializeFunction, hashSource } from './vendor/serialize.js';
+import { beginLoaderFetch, recordLoaderId, withDynamicWorkerCapNamed } from './loader-ledger.js';
+import { assertModuleMapWithinCodeLimit } from './workerd-facet-host.js';
+import { recordFailure, setLastFacetId, getLastRpcFrame } from '@nimbus-sh/core/observability/oom-discriminator.js';
+import { classifyError } from '@nimbus-sh/core/observability/oom-classify.js';
+import { BindingError, ExecutionError, RetryExhaustedError, TimeoutError, } from './vendor/errors.js';
+/**
+ * esbuild runtime helpers re-declared at the top of every generated facet
+ * module. esbuild emits `__name(fn, "fn")` wrappers around every named
+ * function or arrow-with-binding-name; `fn.toString()` yields a body that
+ * references `__name` by bare identifier. The supervisor bundle declares
+ * `__name` at its own top level, but the binding does NOT cross isolate
+ * boundaries — the facet's worker.js must re-declare it.
+ *
+ * The shim is bytes-stable so it doesn't perturb the loader-cache key;
+ * if esbuild ever emits a new helper we'll see a "<name> is not defined"
+ * error in the facet, add it here, and every slot rebuilds.
+ */
+const ESBUILD_RUNTIME_SHIM = [
+    'const __defProp = Object.defineProperty;',
+    'const __name = (target, value) => __defProp(target, "name", { value, configurable: true });',
+    'const __nimbusDisposeRpcResult = (value) => {',
+    '  if ((typeof value !== "object" && typeof value !== "function") || value === null) return;',
+    '  const dispose = value[Symbol.dispose];',
+    '  if (typeof dispose === "function") { try { dispose.call(value); } catch {} }',
+    '};',
+    'const __nimbusUseRpcResult = async (promise, use) => {',
+    '  const value = await promise;',
+    '  try { return await use(value); }',
+    '  finally { __nimbusDisposeRpcResult(value); }',
+    '};',
+].join('\n');
+/** Assemble the exact JavaScript module parsed by a dynamic loader worker. */
+export function assembleLoaderWorkerModuleSource(options) {
+    const lines = [
+        'import { WorkerEntrypoint } from "cloudflare:workers";',
+    ];
+    const wasmEntries = options.wasmEntries ?? [];
+    if (wasmEntries.length > 0) {
+        lines.push('');
+        lines.push('// ── Pool-injected WebAssembly modules ─────────────────────');
+        for (const entry of wasmEntries) {
+            lines.push(`import __NIMBUS_WASM_${entry.id} from './${entry.name}';`);
+        }
+        lines.push('globalThis.__NIMBUS_WASM = globalThis.__NIMBUS_WASM || {};');
+        for (const entry of wasmEntries) {
+            lines.push(`globalThis.__NIMBUS_WASM[${JSON.stringify(entry.name)}] = __NIMBUS_WASM_${entry.id};`);
+        }
+        lines.push('// ── End pool-injected WebAssembly modules ─────────────────');
+    }
+    lines.push('', '// ── esbuild runtime shim ──────────────────────────────────', '// When Nimbus is bundled by wrangler/esbuild, our facet function', '// is transformed into `__name(async function …, "…")` at emit', '// time. `fn.toString()` then yields the wrapped function body,', '// but `__name` and its helpers are module-local in the SUPERVISOR', '// bundle and do NOT cross into the facet isolate. Redeclare them', '// here so facet bodies survive the toString() round-trip.', ESBUILD_RUNTIME_SHIM, '// ── End esbuild runtime shim ──────────────────────────────', '');
+    if (options.preamble) {
+        lines.push('// ── Preamble (pool-level helpers) ─────────────────────────', options.preamble, '// ── End preamble ──────────────────────────────────────────', '');
+    }
+    lines.push(`const __fn__ = ${options.fnSource};`);
+    lines.push('');
+    const callExpr = options.hasBindings
+        ? '__fn__(...args, this.env)'
+        : '__fn__(...args)';
+    lines.push('export default class extends WorkerEntrypoint {', '  execute(...args) {', `    const result = ${callExpr};`, '    if (result instanceof Promise) return result;', '    return result;', '  }', '}');
+    return lines.join('\n');
+}
+/**
+ * Nimbus-scoped parallel dispatch over `env.LOADER`. Tasks are pure
+ * functions whose last argument is an `env` object containing the
+ * forwarded bindings (default: `{ SUPERVISOR }`).
+ *
+ * Typical use:
+ *
+ *   const pool = new LoaderPool(env, ctx, {
+ *     concurrency: 4,
+ *     tag: 'npm-install',
+ *   });
+ *   const results = await pool.map(
+ *     async (pkg, env) => env.SUPERVISOR.writeBatch(buildPayload(pkg)),
+ *     toFetch,
+ *   );
+ */
+export class LoaderPool {
+    loader;
+    /** The hosting actor, as the loader-ledger's per-DO key. */
+    ctx;
+    concurrency;
+    defaultTimeoutMs;
+    defaultRetries;
+    tag;
+    slotGenerations = new Map();
+    bindings;
+    preamble;
+    preambleHash;
+    /**
+     * WASM modules to ship in the LOADER `modules` map. See
+     * LoaderPoolOptions.wasmModules for the rationale. Stored in
+     * insertion order so the per-import preamble we generate matches
+     * across pool dispatches (cache-key stability).
+     */
+    wasmModules;
+    /** Hash of (name + byte length + first/last bytes) of every wasm
+     *  module, folded into the loader cache key so changes invalidate
+     *  warm slots. Hashing the FULL bytes would be O(20+ MiB) per dispatch
+     *  and is unnecessary — wasm bytes are pinned at deploy time, the
+     *  length+endpoints are a strong-enough fingerprint. */
+    wasmHash;
+    /**
+     * Short prefix of the owning DO's id, baked into the loader.get()
+     * cache key so warm isolates are scoped to ONE session. Without this,
+     * session A's pool and session B's pool (same `tag` + `fnHash`) share
+     * an isolate — which means B's writeBatch RPCs routed through A's
+     * env.SUPERVISOR binding (minted with A's doId at construction
+     * time). B's install reports success but the writes land in A's VFS,
+     * leaving B with only the git-clone seed files (~119 instead of ~1491).
+     * 12 chars is enough entropy for DO ids to collide-free per process.
+     */
+    doIdShort;
+    constructor(env, ctx, opts) {
+        // A host hands its whole env over; the binding is claimed here and the
+        // claim is checked on the next line.
+        const loader = env?.LOADER;
+        if (!loader || typeof loader.get !== 'function') {
+            throw new BindingError('LoaderPool: env.LOADER binding missing or invalid. ' +
+                'Add a [[worker_loaders]] entry to wrangler.jsonc.');
+        }
+        this.loader = loader;
+        this.ctx = ctx;
+        this.concurrency = Math.max(1, opts?.concurrency ?? 4);
+        this.defaultTimeoutMs = opts?.timeoutMs ?? 60_000;
+        this.defaultRetries = Math.max(0, opts?.retries ?? 0);
+        this.tag = opts?.tag ?? 'facet';
+        this.preamble = opts?.preamble;
+        // Include preamble in the cache-bucket key so changes to bundled helpers
+        // invalidate warm slots. Empty preamble → '0' suffix (stable).
+        this.preambleHash = this.preamble ? hashSource(this.preamble) : '0';
+        this.doIdShort = opts?.cacheScope === 'global'
+            ? 'global'
+            : ctx.id.toString().slice(0, 12);
+        // Materialise the wasm-modules table. Sanitise each name into a
+        // valid JS identifier for the static import binding; key collisions
+        // (e.g. 'esbuild.wasm' and 'esbuild_wasm' both sanitise to
+        // 'esbuild_wasm') are rejected loudly because the generated worker
+        // would otherwise have duplicate imports. Order is preserved.
+        const wasmEntries = [];
+        const seenIds = new Set();
+        if (opts?.wasmModules) {
+            for (const [name, bytes] of Object.entries(opts.wasmModules)) {
+                if (!(bytes instanceof ArrayBuffer)) {
+                    // Reached only when a caller broke the declared option type, so the
+                    // value is whatever it really was rather than the ArrayBuffer here.
+                    const got = bytes?.constructor?.name;
+                    throw new BindingError(`LoaderPool: wasmModules['${name}'] must be ArrayBuffer ` +
+                        `(got ${got || typeof bytes}).`);
+                }
+                const id = name.replace(/[^A-Za-z0-9_]/g, '_').replace(/^[^A-Za-z_]/, '_');
+                if (seenIds.has(id)) {
+                    throw new BindingError(`LoaderPool: wasmModules key '${name}' collides with another after ` +
+                        `identifier-sanitisation (id='${id}'). Pick distinct module names.`);
+                }
+                seenIds.add(id);
+                wasmEntries.push({ name, id, bytes });
+            }
+        }
+        this.wasmModules = wasmEntries;
+        // Fingerprint: name + length + first/last byte of each module.
+        // Hashing 20+ MiB of wasm per dispatch would be wasteful; this
+        // fingerprint is bytes-stable for a given deployed bundle and only
+        // changes when the wasm itself changes (deploy-time event).
+        if (wasmEntries.length === 0) {
+            this.wasmHash = '0';
+        }
+        else {
+            const fp = wasmEntries
+                .map((w) => {
+                const u = new Uint8Array(w.bytes);
+                const len = u.byteLength;
+                const first = len > 0 ? u[0] : 0;
+                const last = len > 0 ? u[len - 1] : 0;
+                return `${w.name}:${len}:${first}:${last}`;
+            })
+                .join('|');
+            this.wasmHash = hashSource(fp);
+        }
+        const bindings = { ...(opts?.extraBindings ?? {}) };
+        if (!opts?.omitSupervisor) {
+            const supervisorRpc = supervisorEntrypoint();
+            if (supervisorRpc) {
+                // INSTALL-HONESTY: peer-DO branch supplies coordinator's doId
+                // via supervisorDoIdOverride so SUPERVISOR.* RPCs route back
+                // to the user's session DO, not the peer DO. Default to the
+                // local ctx.id (single-DO callers and the in-DO in-DO fanout path).
+                const supDoId = opts?.supervisorDoIdOverride ?? ctx.id.toString();
+                bindings.SUPERVISOR = supervisorRpc({
+                    props: { doId: supDoId, pid: opts?.supervisorPid ?? 0 },
+                });
+            }
+            else {
+                // Supervisor entrypoint unavailable — running without ctx.exports
+                // (e.g. unit-test harness, or LOADER.load contexts where the
+                // bindings.SUPERVISOR auto-wire isn't set up). We still construct
+                // the pool but the facet will get env.SUPERVISOR === undefined.
+                // Callers that need SUPERVISOR should check availability before
+                // dispatch. (A facet that tries to call env.SUPERVISOR.writeBatch
+                // will throw a plain TypeError; that's the clearest failure mode.)
+            }
+        }
+        this.bindings = Object.keys(bindings).length > 0 ? bindings : undefined;
+    }
+    /** Effective concurrency used when no per-call override is supplied. */
+    get defaultConcurrency() {
+        return this.concurrency;
+    }
+    #resolve(opts) {
+        return {
+            timeoutMs: Math.max(0, opts?.timeoutMs ?? this.defaultTimeoutMs),
+            retries: Math.max(0, opts?.retries ?? this.defaultRetries),
+        };
+    }
+    /**
+     * Materialise per-call wasm bytes into the {name,id,bytes} shape
+     * the import-build path expects. Mirrors the constructor's logic
+     * (identifier sanitisation + collision check) but ALSO rejects
+     * collisions with constructor-time entries.
+     *
+     * Returns [] when there are no per-call entries — that's the hot
+     * path (every existing pool dispatch).
+     */
+    #materialisePerCallWasm(perCall) {
+        if (!perCall)
+            return [];
+        const out = [];
+        const ctorIds = new Set(this.wasmModules.map((w) => w.id));
+        const seen = new Set();
+        for (const [name, bytes] of Object.entries(perCall)) {
+            if (!(bytes instanceof ArrayBuffer)) {
+                const got = bytes?.constructor?.name;
+                throw new BindingError(`LoaderPool: per-call wasmModules['${name}'] must be ` +
+                    `ArrayBuffer (got ${got || typeof bytes}).`);
+            }
+            const id = name.replace(/[^A-Za-z0-9_]/g, '_').replace(/^[^A-Za-z_]/, '_');
+            if (ctorIds.has(id)) {
+                throw new BindingError(`LoaderPool: per-call wasmModules key '${name}' (sanitised ` +
+                    `id='${id}') collides with a constructor-time wasm module. ` +
+                    `Per-call modules cannot shadow pool-defaults. Pick a distinct name.`);
+            }
+            if (seen.has(id)) {
+                throw new BindingError(`LoaderPool: per-call wasmModules key '${name}' (sanitised ` +
+                    `id='${id}') collides with another per-call key. Pick distinct names.`);
+            }
+            seen.add(id);
+            out.push({ name, id, bytes });
+        }
+        return out;
+    }
+    /**
+     * Compute a stable fingerprint of a list of {name,bytes} entries.
+     * Returns '0' for the empty list (cache-key bytes-stable for the
+     * common no-per-call-wasm case). Same fingerprinting strategy as
+     * the constructor's `wasmHash`: name + length + first/last byte
+     * per module. Hashing all bytes would be O(20+ MiB) per dispatch
+     * for nothing — the length+endpoints fingerprint is a strong-
+     * enough discriminator and identical bytes produce identical
+     * fingerprints (warm reuse).
+     *
+     * Per-call wasm fingerprinting MUST be content-sensitive: when a user
+     * compiles two .c files (e.g. `clang a.c -o a` then `clang b.c -o b`)
+     * the resulting .wasm binaries may differ by only a few bytes deep
+     * inside the code section. The old fingerprint
+     * `name + len + first + last` collided for such cases, returning the
+     * same cache key and forcing a warm-isolate reuse that served the
+     * FIRST binary's WebAssembly.Module on the second dispatch (verified
+     * in prod: ./a and ./b were both 7572 bytes with identical first/last
+     * bytes, differing only at offset 651 — clang-state-fix wave repro).
+     *
+     * Fix: hash the ACTUAL bytes via djb2 over the full content. Per-call
+     * wasm is typically the user's compiled binary (KBs to a few MiB);
+     * djb2 of a few MiB takes microseconds on the supervisor side and
+     * runs once per dispatch (not per request — warm-reuse-on-match still
+     * works for the legitimate "same bytes" case). The savings of NOT
+     * hashing the wasm were marginal; the correctness cost was severe.
+     */
+    #fingerprintWasm(entries) {
+        if (entries.length === 0)
+            return '0';
+        const parts = [];
+        for (const w of entries) {
+            const u = new Uint8Array(w.bytes);
+            const len = u.byteLength;
+            // djb2 over the bytes. Faster than crypto.subtle.digest at small
+            // sizes, deterministic, and good enough for cache-key
+            // disambiguation (NOT cryptographic — the loader cache doesn't
+            // protect against malicious inputs).
+            let h = 5381;
+            for (let i = 0; i < len; i++) {
+                h = ((h << 5) + h + u[i]) | 0;
+            }
+            parts.push(`${w.name}:${len}:${(h >>> 0).toString(36)}`);
+        }
+        return hashSource(parts.join('|'));
+    }
+    /**
+     * Build the WorkerCode blob that the loader callback will return.
+     * Same bytes every time for a given function and slot, allowing workerd
+     * to reuse the isolate.
+     *
+     * Always prepends the ESBUILD_RUNTIME_SHIM so stringified functions that
+     * reference esbuild-emitted helpers (__name, __defProp, etc.) don't
+     * crash the facet with "__name is not defined". User preambles are
+     * appended below the shim.
+     */
+    #buildCode(fnSource, perCallWasmEntries) {
+        const workerOpts = {
+            compatibilityDate: CF_COMPAT_DATE,
+            compatibilityFlags: ['nodejs_compat'],
+            // Inherit parent network so the facet can reach registry.npmjs.org.
+            globalOutbound: undefined,
+            env: this.bindings,
+        };
+        // ── WASM module imports ───────────────────────────────────────────
+        // Each entry in `wasmModules` (constructor-time + per-call) is
+        // registered in the LOADER's modules map (below) as
+        // `{ wasm: ArrayBuffer }`. workerd compiles each during the
+        // worker's module-load phase (eval permitted there) and the
+        // standard ESM import binding receives the resulting
+        // WebAssembly.Module. We expose them on `globalThis.__NIMBUS_WASM`
+        // so the user fn can read them at request time without having to
+        // re-import (the user fn is serialized via fn.toString and doesn't
+        // carry import statements).
+        //
+        // Per-call entries (passed via LoaderCallOptions.wasmModules
+        // — used by the wasm-runner shell command) are appended to the same
+        // table. Naming collision with constructor entries is rejected
+        // upstream in #materialisePerCallWasm so the import block here
+        // doesn't have to deduplicate.
+        const allWasmEntries = [
+            ...this.wasmModules,
+            ...(perCallWasmEntries ?? []),
+        ];
+        const moduleSource = assembleLoaderWorkerModuleSource({
+            fnSource,
+            preamble: this.preamble,
+            wasmEntries: allWasmEntries,
+            hasBindings: this.bindings !== undefined,
+        });
+        // Modules map: the entry worker.js source plus any wasm modules the
+        // pool was constructed with. Workerd parses the modules map at
+        // worker-load time and resolves the static `import` statements
+        // we generated above against this map. The wasm-shape entry
+        // (`{ wasm: ArrayBuffer }`) tells workerd to compile during the
+        // module-load phase — the only phase where wasm code generation
+        // is permitted in this deploy.
+        //
+        // Per-call entries are appended after constructor entries so the
+        // map order matches the import order in the generated worker.js
+        // (matters only for human-readable diffs; workerd doesn't care).
+        const modules = { 'worker.js': moduleSource };
+        for (const w of allWasmEntries) {
+            modules[w.name] = { wasm: w.bytes };
+        }
+        assertModuleMapWithinCodeLimit(modules);
+        return {
+            compatibilityDate: workerOpts.compatibilityDate,
+            compatibilityFlags: workerOpts.compatibilityFlags,
+            mainModule: 'worker.js',
+            modules,
+            env: workerOpts.env,
+            // globalOutbound: undefined = inherit parent network; omitting the key
+            // from the returned object has the same effect (codegen treats
+            // absence as inherit when the key is explicitly stated; here we keep
+            // it absent to match the cloudflare-parallel semantics).
+        };
+    }
+    /**
+     * Dispatch a single task to the slot isolate. `slotIndex` picks which
+     * warm isolate services the call; callers round-robin slots themselves.
+     */
+    async #dispatchSlot(fnSource, fnHash, slotIndex, args, resilience, perCallWasm) {
+        // Per-call wasm fingerprint. Mixed into the cache key so two calls
+        // with different bytes hit different slots (no cache poisoning).
+        // For the common case (no per-call wasm) the fingerprint is '0',
+        // which is bytes-stable so warm reuse is unaffected.
+        const perCallWasmEntries = this.#materialisePerCallWasm(perCallWasm);
+        const perCallWasmHash = this.#fingerprintWasm(perCallWasmEntries);
+        // Cache key includes the short DO id so warm isolates are scoped to
+        // ONE session. See the doIdShort field comment for why — without it,
+        // a later session's pool reuses the warm worker from a previous
+        // session (which still carries the old session's env.SUPERVISOR
+        // binding), and writeBatch RPCs land in the wrong DO's VFS.
+        const buildId = (generation) => `nfp:${this.tag}:${this.doIdShort}:${fnHash}:${this.preambleHash}:${this.wasmHash}:${perCallWasmHash}:slot-${slotIndex}:g${generation}`;
+        let id = buildId(this.slotGenerations.get(slotIndex) ?? 0);
+        const code = this.#buildCode(fnSource, perCallWasmEntries);
+        // W5 Lever 5: record the dispatch so /api/_diag/memory shows the
+        // last-facet-id even on a hang or silent kill. Bounded — single
+        // slot updated on every dispatch.
+        try {
+            setLastFacetId(id, slotIndex);
+        }
+        catch { /* best-effort */ }
+        const runOnce = async () => {
+            // loader.get() is synchronous from the caller's POV; the callback
+            // is only invoked on cache miss. We wrap the callback tightly so a
+            // retry doesn't rebuild workerCode — that's already stable here.
+            //
+            // The returned `stub` is the cached worker reference. We deliberately
+            // do NOT dispose it: loader.get() is designed for warm-slot reuse
+            // across dispatches (same `id` returns the same cached worker),
+            // and disposing would invalidate that cache.
+            //
+            // We USED to also dispose the per-dispatch `entrypoint` stub in a
+            // finally block here (added in 3c47b44 to prevent QueueState::ACTIVE
+            // during cold-start install). Empirically that broke dispatch on
+            // every slot after the first: once the finally ran
+            // `entrypoint[Symbol.dispose]()`, subsequent dispatches on the
+            // same cached slot hung — SupervisorRPC.writeBatch logged
+            // 'canceled', the DO-side _rpcWriteBatch completed OK, but the
+            // pool never saw the result and every task stalled to the 60s
+            // per-task timeout at 0/13 packages. Removing the per-dispatch
+            // dispose restored 13/13 in ~2.3s in dev.
+            //
+            // The pool-level dispose() method below (also from 3c47b44) is
+            // fine and stays — it only tears down the long-lived SUPERVISOR
+            // binding stub once the whole pool is done, which does NOT
+            // invalidate any in-flight slot's entrypoint reference.
+            const stub = this.loader.get(id, async () => code);
+            recordLoaderId(this.ctx, id);
+            const entrypoint = stub.getEntrypoint();
+            // Direct property call, awaited by this frame — bracketed, never
+            // wrapped. See beginLoaderFetch for the measured DO-poisoning hazard.
+            const endFetch = beginLoaderFetch(this.ctx);
+            try {
+                const out = await entrypoint.execute(...args);
+                return out;
+            }
+            catch (err) {
+                if (err instanceof Error) {
+                    throw new ExecutionError(err.message, err.stack);
+                }
+                throw new ExecutionError(String(err));
+            }
+            finally {
+                endFetch();
+            }
+        };
+        const maxAttempts = 1 + resilience.retries;
+        let lastError;
+        let retriedCloneRefusal = false;
+        let attempt = 0;
+        while (attempt < maxAttempts) {
+            try {
+                if (resilience.timeoutMs > 0) {
+                    // Race runOnce() against a settable timer. CRITICAL: clear
+                    // the timer in a finally so the timer's reject closure (which
+                    // transitively roots `args` — i.e. the per-task payload sent
+                    // to the slot, including 28 MiB pre-bundle slices) doesn't
+                    // hold its references for the full timeoutMs after the race
+                    // settles.
+                    //
+                    // Before this fix: a facet OOM at t=0 left the slice rooted
+                    // for the remaining timeoutMs (default 60s for pre-bundle).
+                    // With concurrency=2, two consecutive OOMs could pin
+                    // ~56 MiB of slice memory in the supervisor heap for a full
+                    // minute — alongside an in-flight cirrus-real boot, that's
+                    // enough to push a shared isolate over the 128 MiB cap.
+                    // See plan in close-plan-2026-04-28.
+                    let timerId;
+                    try {
+                        return await Promise.race([
+                            runOnce(),
+                            new Promise((_, reject) => {
+                                timerId = setTimeout(() => reject(new TimeoutError(resilience.timeoutMs)), resilience.timeoutMs);
+                            }),
+                        ]);
+                    }
+                    finally {
+                        if (timerId !== undefined)
+                            clearTimeout(timerId);
+                    }
+                }
+                return await runOnce();
+            }
+            catch (err) {
+                lastError = err instanceof Error ? err : new Error(String(err));
+                const cause = classifyError(lastError);
+                // W5 Lever 5: classify + record. We push on EVERY failed
+                // attempt (not just the final retry-exhausted throw) so the
+                // ring captures transient SQLITE_NOMEM / clone-refused
+                // patterns that still ultimately succeed. Ring is bounded
+                // (50 entries) so noise is self-limiting.
+                try {
+                    recordFailure({
+                        at: Date.now(),
+                        phase: 'rpc',
+                        cause,
+                        rssEstimateBytes: 0, heapUsedBytes: 0,
+                        lruBytes: 0, inFlightBytes: 0,
+                        lastRpcFrame: getLastRpcFrame(),
+                        lastFacetId: { codeId: id, slotIndex, atMs: Date.now() },
+                        message: lastError.message,
+                    });
+                }
+                catch { /* fail-soft */ }
+                if (cause === 'clone_refused' && !retriedCloneRefusal) {
+                    retriedCloneRefusal = true;
+                    const generation = (this.slotGenerations.get(slotIndex) ?? 0) + 1;
+                    this.slotGenerations.set(slotIndex, generation);
+                    id = buildId(generation);
+                    try {
+                        setLastFacetId(id, slotIndex);
+                    }
+                    catch { /* best-effort */ }
+                    // If the supervisor DO is stale, a newer loader still cannot
+                    // deserialize back into it; only recycling that DO heals the
+                    // reverse direction. This refresh targets the stale-loader case.
+                    continue;
+                }
+                if (attempt < maxAttempts - 1) {
+                    // 100 * 2^attempt, capped at 2s so retries don't compound waiting.
+                    const delay = Math.min(2000, 100 * Math.pow(2, attempt));
+                    await new Promise((r) => setTimeout(r, delay));
+                }
+                attempt++;
+            }
+        }
+        // On the way out only, so retries do not stack the annotation: a cap hit
+        // carries the ledger — which ids hold slots, and that a keyed id never
+        // gives one back — instead of the platform's bare message.
+        const named = withDynamicWorkerCapNamed(this.ctx, lastError);
+        if (maxAttempts > 1) {
+            throw new RetryExhaustedError(maxAttempts, named);
+        }
+        throw named;
+    }
+    #prepare(fn) {
+        const fnSource = serializeFunction(fn);
+        const fnHash = hashSource(fnSource);
+        return { fnSource, fnHash };
+    }
+    /**
+     * Run `fn` once with `arg` on a slot isolate. Returns the result or
+     * throws TimeoutError / RetryExhaustedError / ExecutionError.
+     */
+    async submit(fn, arg, opts) {
+        const { fnSource, fnHash } = this.#prepare(fn);
+        const resilience = this.#resolve(opts);
+        return (await this.#dispatchSlot(fnSource, fnHash, 0, [arg], resilience, opts?.wasmModules));
+    }
+    /**
+     * Run `fn` on every item in `items`, at most `concurrency` at a time,
+     * pinned to stable slots so warm isolates are reused.
+     *
+     * Results are returned in input order. Failure handling per `onError`.
+     */
+    async map(fn, items, opts) {
+        if (items.length === 0)
+            return [];
+        const { fnSource, fnHash } = this.#prepare(fn);
+        return this.#mapInternal(fnSource, fnHash, items, opts);
+    }
+    /**
+     * Same shape as `map`, but accepts a pre-serialized function source
+     * string instead of a live function reference. Used by
+     * `FanoutPool`'s peer-DO leg, where the function was already
+     * serialized on the coordinator side and forwarded over RPC.
+     *
+     * The fnSource MUST be the output of `serializeFunction(fn)`
+     * (typically forwarded directly from a coordinator RPC). Bytes-
+     * stable invariants:
+     *   - `fnHash = hashSource(fnSource)` must be deterministic so
+     *     warm slots are correctly keyed.
+     *   - `fnSource` must NOT reference `this` — same rule as
+     *     `serializeFunction`.
+     *
+     * No fn-validation runs here (it already ran on the coordinator);
+     * the peer trusts the caller to forward a valid serialization.
+     */
+    async mapSource(fnSource, items, opts) {
+        if (items.length === 0)
+            return [];
+        const fnHash = hashSource(fnSource);
+        return this.#mapInternal(fnSource, fnHash, items, opts);
+    }
+    async #mapInternal(fnSource, fnHash, items, opts) {
+        const resilience = this.#resolve(opts);
+        const concurrency = Math.max(1, Math.min(opts?.concurrency ?? this.concurrency, items.length));
+        const onError = opts?.onError ?? 'throw';
+        const settled = new Array(items.length);
+        let cursor = 0;
+        const runSlot = async (slotIndex) => {
+            while (true) {
+                const idx = cursor++;
+                if (idx >= items.length)
+                    return;
+                try {
+                    const value = (await this.#dispatchSlot(fnSource, fnHash, slotIndex, [items[idx]], resilience, opts?.wasmModules));
+                    settled[idx] = { ok: true, value };
+                }
+                catch (err) {
+                    const error = err instanceof Error ? err : new Error(String(err));
+                    if (onError === 'throw')
+                        throw error;
+                    settled[idx] = { ok: false, error };
+                }
+            }
+        };
+        await Promise.all(Array.from({ length: concurrency }, (_, slotIndex) => runSlot(slotIndex)));
+        if (onError === 'null') {
+            return settled.map((s) => (s.ok ? s.value : null));
+        }
+        if (onError === 'skip') {
+            return settled
+                .filter((s) => s.ok)
+                .map((s) => s.value);
+        }
+        // onError === 'throw' — all slots succeeded.
+        return settled.map((s) => s.value);
+    }
+    /**
+     * Release any RPC stubs held by the pool. Call this once the caller
+     * is done with the pool (post-`map`/`submit`) so the underlying
+     * stubs don't linger in workerd's deferred-destruction queue.
+     *
+     * Primary target: the SUPERVISOR binding stub we minted at
+     * construction time (via the registered supervisor entrypoint). It's
+     * a cross-isolate RPC stub — without explicit disposal it stays
+     * referenced until the parent isolate's event-handler context
+     * finishes, which during npm install means "until the whole install
+     * completes" — long enough to accumulate alongside other leaked
+     * stubs and trip the QueueState::ACTIVE fatal.
+     *
+     * Safe to call more than once; idempotent.
+     */
+    dispose() {
+        if (!this.bindings)
+            return;
+        for (const key of Object.keys(this.bindings)) {
+            disposeRpcResource(this.bindings[key]);
+        }
+        // Prevent double-dispose from re-running the loop.
+        this.bindings = undefined;
+    }
+}