@nimbus-sh/fabric 0.1.0

package/src/loader-pool.ts

@@ -0,0 +1,984 @@
+/**
+ * loader-pool.ts — Nimbus loader-isolate pool based on cloudflare-parallel.
+ *
+ * Adds Nimbus-specific behavior to the upstream pool design:
+ *   1. **Stable-slot isolate reuse**. Upstream's #counter++ gives every
+ *      dispatch a fresh isolate — fine for one-off AI calls, terrible for
+ *      running 67 npm tarball extractions (cold-start dominates). We pin
+ *      each job to `slot = cursor % concurrency` and use stable loader
+ *      IDs `nfp:${fnHash}:slot-${i}:g${generation}`, so a pool of
+ *      concurrency=4 keeps at most 4 warm isolates rather than N fresh ones.
+ *   2. **Nimbus defaults**: compatibilityDate = CF_COMPAT_DATE (matches
+ *      the supervisor worker), compatibilityFlags = ['nodejs_compat'],
+ *      globalOutbound = undefined (inherit parent network so the facet can
+ *      reach https://registry.npmjs.org without a proxy binding).
+ *   3. **Supervisor autoinjection**. The pool grabs the embedder's
+ *      registered supervisor entrypoint stub (see `supervisorEntrypoint` in
+ *      ctx-exports.ts) and forwards it as `env.SUPERVISOR` to every facet,
+ *      same pattern as git-network-facet.ts. Callers can add more bindings
+ *      via `extraBindings`.
+ *   4. **Fail-loud defaults**: timeout 60s, retries 0, onError 'throw'.
+ *      Caller opts in to leniency.
+ *
+ * The vendored directory contains only the upstream serialization, error,
+ * and binding types used by this implementation.
+ */
+
+import { CF_COMPAT_DATE } from '@nimbus-sh/core/constants.js';
+import { supervisorEntrypoint } from './ctx-exports.js';
+import { disposeRpcResource } from '@nimbus-sh/core/_shared/rpc-dispose.js';
+import { serializeFunction, hashSource } from './vendor/serialize.js';
+import { beginLoaderFetch, recordLoaderId, withDynamicWorkerCapNamed } from './loader-ledger.js';
+import { assertModuleMapWithinCodeLimit } from './workerd-facet-host.js';
+import { recordFailure, setLastFacetId, getLastRpcFrame } from '@nimbus-sh/core/observability/oom-discriminator.js';
+import { classifyError } from '@nimbus-sh/core/observability/oom-classify.js';
+import {
+  BindingError,
+  ExecutionError,
+  RetryExhaustedError,
+  TimeoutError,
+} from './vendor/errors.js';
+import type { FacetBindings } from '@nimbus-sh/core/runtime/facet-host.js';
+import type { ModuleContent, WorkerLoader } from './vendor/types.js';
+
+/**
+ * A function dispatched into a facet isolate, with the bindings that facet was
+ * minted with as its second argument.
+ *
+ * Declared through a method so the bindings parameter compares BIVARIANTLY: a
+ * task body annotates the exact surface it calls (`env.SUPERVISOR` is the
+ * embedder's RPC class, which the fabric cannot name), and accepting that
+ * narrowing is the whole point of handing the bindings over.
+ */
+export type FacetTaskFn<A, R> = {
+  task(args: A, env: FacetBindings): R | Promise<R>;
+}['task'];
+
+/** The one binding a pool needs off whichever env its host hands it. */
+export interface LoaderPoolEnv {
+  LOADER?: WorkerLoader;
+}
+
+/** Options handed to LoaderPool's constructor. */
+export interface LoaderPoolOptions {
+  /** Maximum concurrent in-flight facets. Default 4. */
+  concurrency?: number;
+  /** Per-task timeout in ms. Default 60_000. */
+  timeoutMs?: number;
+  /**
+   * Per-task retry attempts AFTER the initial failure. Default 0.
+   * Set to a small number only if transient RPC errors are common.
+   */
+  retries?: number;
+  /**
+   * Additional bindings forwarded to each facet. These merge on top of the
+   * default `{ SUPERVISOR: supervisorRpc({ doId, pid:0 }) }`. Use this to
+   * give facets access to KV, R2, AI, or additional supervisor-level APIs.
+   */
+  extraBindings?: Record<string, unknown>;
+  /**
+   * Optional tag used in loader IDs for debugging (e.g. "npm-install").
+   * Does NOT affect isolate identity — same fn + same tag = same slot.
+   */
+  tag?: string;
+  /**
+   * If true, omit the default SUPERVISOR binding. Use this for pools
+   * that don't need DO callbacks (e.g. a pure CPU compute pool).
+   */
+  omitSupervisor?: boolean;
+  /**
+   * Loader cache scope. Defaults to `session`, which bakes the owning DO id
+   * into the loader key so stateful facets cannot leak bindings or globals
+   * across sessions. Use `global` only for stateless compute modules that do
+   * not receive a Supervisor binding and do not retain user state.
+   */
+  cacheScope?: 'session' | 'global';
+  /**
+   * Override the `doId` baked into the auto-injected SUPERVISOR binding.
+   * Default: `ctx.id.toString()` (the DO that constructs the pool).
+   *
+   * Used by FanoutPool's peer-DO branch (peer-DO fanout): peer DOs
+   * construct their per-task LoaderPool from inside
+   * `_rpcFanoutExecute`, where `ctx` is the PEER DO's ctx. Without this
+   * override the peer's auto-injected SUPERVISOR routes back to the
+   * peer DO itself — so writes (e.g. install-batch-facet's
+   * writeBatchStream) land in the peer's VFS instead of the
+   * COORDINATOR's. The user's terminal session is on the coordinator;
+   * writes-to-peer are invisible. See INSTALL-HONESTY-retro.md.
+   *
+   * When set, the auto-injected supervisor binding uses this string as the
+   * `props.doId`, routing all SUPERVISOR.* calls back to the
+   * coordinator. Effective only when `omitSupervisor !== true`.
+   */
+  supervisorDoIdOverride?: string;
+  /**
+   * Process pid baked into the auto-injected SUPERVISOR binding's props.
+   * The supervisor derives the write credential from this pid
+   * (`SupervisorRPC._pid()` → `processes.cred(pid)`), so any facet that
+   * calls a filesystem RPC (`writeBatchStream`, `writeFile`, …) must be
+   * dispatched with the invoking process's real pid — otherwise the RPC
+   * throws "missing or invalid process pid in props". npm install threads
+   * the shell command's `ctx.pid` here so package files land as the user.
+   * Left 0 (default) for pools whose facets touch only cache/registry RPCs
+   * (npm resolve, pre-bundle), which never call `_pid()`.
+   */
+  supervisorPid?: number;
+  /**
+   * Raw JavaScript source prepended to every generated worker module.
+   * Lets callers inject bundled helpers, such as a tar parser. The user
+   * function can reference top-level names declared in the preamble as if
+   * they were in lexical scope.
+   *
+   * Example: `preamble: 'export const parse = ...; const helper = ...;'`
+   * — the preamble runs at module-load time; any side effects happen
+   * inside the facet isolate.
+   *
+   * Preamble text is bytes-stable for a given pool — it's part of the
+   * loader-cache key (fnHash), so changing the preamble invalidates all
+   * warm slots.
+   */
+  preamble?: string;
+  /**
+   * WebAssembly modules to ship into the facet via the LOADER's
+   * `modules` map. Map keys are module specifier paths (e.g.
+   * `'esbuild.wasm'`); values are the raw bytes.
+   *
+   * Workerd registers each entry as `{ wasm: ArrayBuffer }` in the
+   * worker's modules map. The pool prepends a static
+   * `import __NIMBUS_WASM_<id> from './<key>';` to the generated
+   * worker.js so workerd compiles each at module-load (startup phase,
+   * where wasm code generation is permitted). The compiled Modules
+   * are exposed via `globalThis.__NIMBUS_WASM[<key>]` for the user
+   * function to read at request time.
+   *
+   * Why this works when other paths don't:
+   *   - request-time `WebAssembly.compile()` — disallowed by workerd
+   *     in this deploy.
+   *   - request-time RPC of a pre-compiled Module — workerd
+   *     structured-clone refuses ("Unable to deserialize cloned data").
+   *   - inlining bytes in the preamble — 16 MiB string per dispatch
+   *     OOMs the supervisor at module-source allocation time.
+   *   - LOADER modules-map (this) — bytes ride INSIDE the worker code
+   *     blob; workerd compiles wasm during its own startup pipeline,
+   *     never crossing structured-clone, never executing JS eval.
+   *
+   * The bytes ARE part of the loader-cache key (workerd hashes the
+   * whole WorkerCode), so changing the wasm bytes invalidates warm
+   * slots — desirable when the bundled wasm version changes.
+   */
+  wasmModules?: Record<string, ArrayBuffer>;
+}
+
+/** Per-call override (merged with pool defaults). */
+export interface LoaderCallOptions {
+  timeoutMs?: number;
+  retries?: number;
+  /**
+   * Per-call WebAssembly modules. Merged with the pool's
+   * constructor-time `wasmModules` at dispatch time and shipped via
+   * the LOADER's modules map (same `{ wasm: ArrayBuffer }` shape).
+   *
+   * Shipping path validated empirically against prod (see
+   * `WebAssembly.instantiate(bytes)` is blocked at request-time but
+   * the LOADER-modules path compiles bytes during the inner
+   * worker's module-load phase, where wasm code generation IS
+   * permitted. The bytes ride INSIDE the worker code blob; workerd
+   * never crosses structured-clone, never executes user-eval.
+   *
+   * Cache key impact: per-call bytes are fingerprinted (length +
+   * first/last byte per module) and folded into the loader cache
+   * key. Identical bytes on the same slot → warm reuse; different
+   * bytes → fresh isolate. The pool's existing `wasmHash` field
+   * captures CONSTRUCTOR-time bytes only; per-call bytes get an
+   * independent fingerprint mixed into the slot id at dispatch.
+   *
+   * Naming collision rule: a per-call key MUST NOT collide with a
+   * constructor-time key (after identifier sanitisation). The
+   * dispatch path throws BindingError if it does — silently
+   * shadowing the constructor's wasm would break the cache-key
+   * invariant downstream callers rely on.
+   *
+   * Used by the `wasm-runner` shell command in src/runtime/
+   * wasm-runner.ts to ship user-supplied .wasm bytes from VFS into
+   * a fresh facet isolate per invocation.
+   */
+  wasmModules?: Record<string, ArrayBuffer>;
+}
+
+/** Per-map override. Adds onError strategy for partial failures. */
+export interface LoaderMapOptions extends LoaderCallOptions {
+  /** Concurrency override for this call. Defaults to pool's concurrency. */
+  concurrency?: number;
+  /**
+   * What to do when an individual item fails:
+   *   - 'throw' (default): reject whole map on first failure.
+   *   - 'null': replace failed items with null in the result array.
+   *   - 'skip': omit failed items from the result array.
+   * We default to 'throw' — install-time failures are not silently ignored.
+   */
+  onError?: 'throw' | 'null' | 'skip';
+}
+
+interface ResolvedResilience {
+  timeoutMs: number;
+  retries: number;
+}
+
+/**
+ * esbuild runtime helpers re-declared at the top of every generated facet
+ * module. esbuild emits `__name(fn, "fn")` wrappers around every named
+ * function or arrow-with-binding-name; `fn.toString()` yields a body that
+ * references `__name` by bare identifier. The supervisor bundle declares
+ * `__name` at its own top level, but the binding does NOT cross isolate
+ * boundaries — the facet's worker.js must re-declare it.
+ *
+ * The shim is bytes-stable so it doesn't perturb the loader-cache key;
+ * if esbuild ever emits a new helper we'll see a "<name> is not defined"
+ * error in the facet, add it here, and every slot rebuilds.
+ */
+const ESBUILD_RUNTIME_SHIM = [
+  'const __defProp = Object.defineProperty;',
+  'const __name = (target, value) => __defProp(target, "name", { value, configurable: true });',
+  'const __nimbusDisposeRpcResult = (value) => {',
+  '  if ((typeof value !== "object" && typeof value !== "function") || value === null) return;',
+  '  const dispose = value[Symbol.dispose];',
+  '  if (typeof dispose === "function") { try { dispose.call(value); } catch {} }',
+  '};',
+  'const __nimbusUseRpcResult = async (promise, use) => {',
+  '  const value = await promise;',
+  '  try { return await use(value); }',
+  '  finally { __nimbusDisposeRpcResult(value); }',
+  '};',
+].join('\n');
+
+export interface LoaderWorkerModuleSourceOptions {
+  fnSource: string;
+  preamble?: string;
+  wasmEntries?: ReadonlyArray<{ name: string; id: string }>;
+  hasBindings: boolean;
+}
+
+/** Assemble the exact JavaScript module parsed by a dynamic loader worker. */
+export function assembleLoaderWorkerModuleSource(
+  options: LoaderWorkerModuleSourceOptions,
+): string {
+  const lines: string[] = [
+    'import { WorkerEntrypoint } from "cloudflare:workers";',
+  ];
+  const wasmEntries = options.wasmEntries ?? [];
+
+  if (wasmEntries.length > 0) {
+    lines.push('');
+    lines.push('// ── Pool-injected WebAssembly modules ─────────────────────');
+    for (const entry of wasmEntries) {
+      lines.push(`import __NIMBUS_WASM_${entry.id} from './${entry.name}';`);
+    }
+    lines.push('globalThis.__NIMBUS_WASM = globalThis.__NIMBUS_WASM || {};');
+    for (const entry of wasmEntries) {
+      lines.push(
+        `globalThis.__NIMBUS_WASM[${JSON.stringify(entry.name)}] = __NIMBUS_WASM_${entry.id};`,
+      );
+    }
+    lines.push('// ── End pool-injected WebAssembly modules ─────────────────');
+  }
+
+  lines.push(
+    '',
+    '// ── esbuild runtime shim ──────────────────────────────────',
+    '// When Nimbus is bundled by wrangler/esbuild, our facet function',
+    '// is transformed into `__name(async function …, "…")` at emit',
+    '// time. `fn.toString()` then yields the wrapped function body,',
+    '// but `__name` and its helpers are module-local in the SUPERVISOR',
+    '// bundle and do NOT cross into the facet isolate. Redeclare them',
+    '// here so facet bodies survive the toString() round-trip.',
+    ESBUILD_RUNTIME_SHIM,
+    '// ── End esbuild runtime shim ──────────────────────────────',
+    '',
+  );
+  if (options.preamble) {
+    lines.push(
+      '// ── Preamble (pool-level helpers) ─────────────────────────',
+      options.preamble,
+      '// ── End preamble ──────────────────────────────────────────',
+      '',
+    );
+  }
+  lines.push(`const __fn__ = ${options.fnSource};`);
+  lines.push('');
+  const callExpr = options.hasBindings
+    ? '__fn__(...args, this.env)'
+    : '__fn__(...args)';
+  lines.push(
+    'export default class extends WorkerEntrypoint {',
+    '  execute(...args) {',
+    `    const result = ${callExpr};`,
+    '    if (result instanceof Promise) return result;',
+    '    return result;',
+    '  }',
+    '}',
+  );
+  return lines.join('\n');
+}
+
+/**
+ * Nimbus-scoped parallel dispatch over `env.LOADER`. Tasks are pure
+ * functions whose last argument is an `env` object containing the
+ * forwarded bindings (default: `{ SUPERVISOR }`).
+ *
+ * Typical use:
+ *
+ *   const pool = new LoaderPool(env, ctx, {
+ *     concurrency: 4,
+ *     tag: 'npm-install',
+ *   });
+ *   const results = await pool.map(
+ *     async (pkg, env) => env.SUPERVISOR.writeBatch(buildPayload(pkg)),
+ *     toFetch,
+ *   );
+ */
+export class LoaderPool {
+  private readonly loader: WorkerLoader;
+  /** The hosting actor, as the loader-ledger's per-DO key. */
+  private readonly ctx: DurableObjectState;
+  private readonly concurrency: number;
+  private readonly defaultTimeoutMs: number;
+  private readonly defaultRetries: number;
+  private readonly tag: string;
+  private readonly slotGenerations = new Map<number, number>();
+  private bindings: Record<string, unknown> | undefined;
+
+  private readonly preamble: string | undefined;
+  private readonly preambleHash: string;
+  /**
+   * WASM modules to ship in the LOADER `modules` map. See
+   * LoaderPoolOptions.wasmModules for the rationale. Stored in
+   * insertion order so the per-import preamble we generate matches
+   * across pool dispatches (cache-key stability).
+   */
+  private readonly wasmModules: Array<{
+    /** Specifier path the worker imports from (e.g. 'esbuild.wasm'). */
+    name: string;
+    /** Identifier used inside the generated worker for both the static
+     *  import binding and the globalThis exposure. Sanitised from `name`. */
+    id: string;
+    bytes: ArrayBuffer;
+  }>;
+  /** Hash of (name + byte length + first/last bytes) of every wasm
+   *  module, folded into the loader cache key so changes invalidate
+   *  warm slots. Hashing the FULL bytes would be O(20+ MiB) per dispatch
+   *  and is unnecessary — wasm bytes are pinned at deploy time, the
+   *  length+endpoints are a strong-enough fingerprint. */
+  private readonly wasmHash: string;
+  /**
+   * Short prefix of the owning DO's id, baked into the loader.get()
+   * cache key so warm isolates are scoped to ONE session. Without this,
+   * session A's pool and session B's pool (same `tag` + `fnHash`) share
+   * an isolate — which means B's writeBatch RPCs routed through A's
+   * env.SUPERVISOR binding (minted with A's doId at construction
+   * time). B's install reports success but the writes land in A's VFS,
+   * leaving B with only the git-clone seed files (~119 instead of ~1491).
+   * 12 chars is enough entropy for DO ids to collide-free per process.
+   */
+  private readonly doIdShort: string;
+
+  constructor(
+    env: unknown,
+    ctx: DurableObjectState,
+    opts?: LoaderPoolOptions,
+  ) {
+    // A host hands its whole env over; the binding is claimed here and the
+    // claim is checked on the next line.
+    const loader = (env as LoaderPoolEnv | null | undefined)?.LOADER;
+    if (!loader || typeof loader.get !== 'function') {
+      throw new BindingError(
+        'LoaderPool: env.LOADER binding missing or invalid. ' +
+          'Add a [[worker_loaders]] entry to wrangler.jsonc.',
+      );
+    }
+    this.loader = loader;
+    this.ctx = ctx;
+    this.concurrency = Math.max(1, opts?.concurrency ?? 4);
+    this.defaultTimeoutMs = opts?.timeoutMs ?? 60_000;
+    this.defaultRetries = Math.max(0, opts?.retries ?? 0);
+    this.tag = opts?.tag ?? 'facet';
+    this.preamble = opts?.preamble;
+    // Include preamble in the cache-bucket key so changes to bundled helpers
+    // invalidate warm slots. Empty preamble → '0' suffix (stable).
+    this.preambleHash = this.preamble ? hashSource(this.preamble) : '0';
+    this.doIdShort = opts?.cacheScope === 'global'
+      ? 'global'
+      : ctx.id.toString().slice(0, 12);
+
+    // Materialise the wasm-modules table. Sanitise each name into a
+    // valid JS identifier for the static import binding; key collisions
+    // (e.g. 'esbuild.wasm' and 'esbuild_wasm' both sanitise to
+    // 'esbuild_wasm') are rejected loudly because the generated worker
+    // would otherwise have duplicate imports. Order is preserved.
+    const wasmEntries: Array<{ name: string; id: string; bytes: ArrayBuffer }> = [];
+    const seenIds = new Set<string>();
+    if (opts?.wasmModules) {
+      for (const [name, bytes] of Object.entries(opts.wasmModules)) {
+        if (!(bytes instanceof ArrayBuffer)) {
+          // Reached only when a caller broke the declared option type, so the
+          // value is whatever it really was rather than the ArrayBuffer here.
+          const got = (bytes as { constructor?: { name?: string } } | null | undefined)?.constructor?.name;
+          throw new BindingError(
+            `LoaderPool: wasmModules['${name}'] must be ArrayBuffer ` +
+            `(got ${got || typeof bytes}).`,
+          );
+        }
+        const id = name.replace(/[^A-Za-z0-9_]/g, '_').replace(/^[^A-Za-z_]/, '_');
+        if (seenIds.has(id)) {
+          throw new BindingError(
+            `LoaderPool: wasmModules key '${name}' collides with another after ` +
+            `identifier-sanitisation (id='${id}'). Pick distinct module names.`,
+          );
+        }
+        seenIds.add(id);
+        wasmEntries.push({ name, id, bytes });
+      }
+    }
+    this.wasmModules = wasmEntries;
+    // Fingerprint: name + length + first/last byte of each module.
+    // Hashing 20+ MiB of wasm per dispatch would be wasteful; this
+    // fingerprint is bytes-stable for a given deployed bundle and only
+    // changes when the wasm itself changes (deploy-time event).
+    if (wasmEntries.length === 0) {
+      this.wasmHash = '0';
+    } else {
+      const fp = wasmEntries
+        .map((w) => {
+          const u = new Uint8Array(w.bytes);
+          const len = u.byteLength;
+          const first = len > 0 ? u[0] : 0;
+          const last  = len > 0 ? u[len - 1] : 0;
+          return `${w.name}:${len}:${first}:${last}`;
+        })
+        .join('|');
+      this.wasmHash = hashSource(fp);
+    }
+
+    const bindings: Record<string, unknown> = { ...(opts?.extraBindings ?? {}) };
+    if (!opts?.omitSupervisor) {
+      const supervisorRpc = supervisorEntrypoint();
+      if (supervisorRpc) {
+        // INSTALL-HONESTY: peer-DO branch supplies coordinator's doId
+        // via supervisorDoIdOverride so SUPERVISOR.* RPCs route back
+        // to the user's session DO, not the peer DO. Default to the
+        // local ctx.id (single-DO callers and the in-DO in-DO fanout path).
+        const supDoId = opts?.supervisorDoIdOverride ?? ctx.id.toString();
+        bindings.SUPERVISOR = supervisorRpc({
+          props: { doId: supDoId, pid: opts?.supervisorPid ?? 0 },
+        });
+      } else {
+        // Supervisor entrypoint unavailable — running without ctx.exports
+        // (e.g. unit-test harness, or LOADER.load contexts where the
+        // bindings.SUPERVISOR auto-wire isn't set up). We still construct
+        // the pool but the facet will get env.SUPERVISOR === undefined.
+        // Callers that need SUPERVISOR should check availability before
+        // dispatch. (A facet that tries to call env.SUPERVISOR.writeBatch
+        // will throw a plain TypeError; that's the clearest failure mode.)
+      }
+    }
+    this.bindings = Object.keys(bindings).length > 0 ? bindings : undefined;
+  }
+
+  /** Effective concurrency used when no per-call override is supplied. */
+  get defaultConcurrency(): number {
+    return this.concurrency;
+  }
+
+  #resolve(opts?: LoaderCallOptions): ResolvedResilience {
+    return {
+      timeoutMs: Math.max(0, opts?.timeoutMs ?? this.defaultTimeoutMs),
+      retries: Math.max(0, opts?.retries ?? this.defaultRetries),
+    };
+  }
+
+  /**
+   * Materialise per-call wasm bytes into the {name,id,bytes} shape
+   * the import-build path expects. Mirrors the constructor's logic
+   * (identifier sanitisation + collision check) but ALSO rejects
+   * collisions with constructor-time entries.
+   *
+   * Returns [] when there are no per-call entries — that's the hot
+   * path (every existing pool dispatch).
+   */
+  #materialisePerCallWasm(
+    perCall: Record<string, ArrayBuffer> | undefined,
+  ): Array<{ name: string; id: string; bytes: ArrayBuffer }> {
+    if (!perCall) return [];
+    const out: Array<{ name: string; id: string; bytes: ArrayBuffer }> = [];
+    const ctorIds = new Set(this.wasmModules.map((w) => w.id));
+    const seen = new Set<string>();
+    for (const [name, bytes] of Object.entries(perCall)) {
+      if (!(bytes instanceof ArrayBuffer)) {
+        const got = (bytes as { constructor?: { name?: string } } | null | undefined)?.constructor?.name;
+        throw new BindingError(
+          `LoaderPool: per-call wasmModules['${name}'] must be ` +
+            `ArrayBuffer (got ${got || typeof bytes}).`,
+        );
+      }
+      const id = name.replace(/[^A-Za-z0-9_]/g, '_').replace(/^[^A-Za-z_]/, '_');
+      if (ctorIds.has(id)) {
+        throw new BindingError(
+          `LoaderPool: per-call wasmModules key '${name}' (sanitised ` +
+            `id='${id}') collides with a constructor-time wasm module. ` +
+            `Per-call modules cannot shadow pool-defaults. Pick a distinct name.`,
+        );
+      }
+      if (seen.has(id)) {
+        throw new BindingError(
+          `LoaderPool: per-call wasmModules key '${name}' (sanitised ` +
+            `id='${id}') collides with another per-call key. Pick distinct names.`,
+        );
+      }
+      seen.add(id);
+      out.push({ name, id, bytes });
+    }
+    return out;
+  }
+
+  /**
+   * Compute a stable fingerprint of a list of {name,bytes} entries.
+   * Returns '0' for the empty list (cache-key bytes-stable for the
+   * common no-per-call-wasm case). Same fingerprinting strategy as
+   * the constructor's `wasmHash`: name + length + first/last byte
+   * per module. Hashing all bytes would be O(20+ MiB) per dispatch
+   * for nothing — the length+endpoints fingerprint is a strong-
+   * enough discriminator and identical bytes produce identical
+   * fingerprints (warm reuse).
+   *
+   * Per-call wasm fingerprinting MUST be content-sensitive: when a user
+   * compiles two .c files (e.g. `clang a.c -o a` then `clang b.c -o b`)
+   * the resulting .wasm binaries may differ by only a few bytes deep
+   * inside the code section. The old fingerprint
+   * `name + len + first + last` collided for such cases, returning the
+   * same cache key and forcing a warm-isolate reuse that served the
+   * FIRST binary's WebAssembly.Module on the second dispatch (verified
+   * in prod: ./a and ./b were both 7572 bytes with identical first/last
+   * bytes, differing only at offset 651 — clang-state-fix wave repro).
+   *
+   * Fix: hash the ACTUAL bytes via djb2 over the full content. Per-call
+   * wasm is typically the user's compiled binary (KBs to a few MiB);
+   * djb2 of a few MiB takes microseconds on the supervisor side and
+   * runs once per dispatch (not per request — warm-reuse-on-match still
+   * works for the legitimate "same bytes" case). The savings of NOT
+   * hashing the wasm were marginal; the correctness cost was severe.
+   */
+  #fingerprintWasm(
+    entries: Array<{ name: string; bytes: ArrayBuffer }>,
+  ): string {
+    if (entries.length === 0) return '0';
+    const parts: string[] = [];
+    for (const w of entries) {
+      const u = new Uint8Array(w.bytes);
+      const len = u.byteLength;
+      // djb2 over the bytes. Faster than crypto.subtle.digest at small
+      // sizes, deterministic, and good enough for cache-key
+      // disambiguation (NOT cryptographic — the loader cache doesn't
+      // protect against malicious inputs).
+      let h = 5381;
+      for (let i = 0; i < len; i++) {
+        h = ((h << 5) + h + u[i]) | 0;
+      }
+      parts.push(`${w.name}:${len}:${(h >>> 0).toString(36)}`);
+    }
+    return hashSource(parts.join('|'));
+  }
+
+  /**
+   * Build the WorkerCode blob that the loader callback will return.
+   * Same bytes every time for a given function and slot, allowing workerd
+   * to reuse the isolate.
+   *
+   * Always prepends the ESBUILD_RUNTIME_SHIM so stringified functions that
+   * reference esbuild-emitted helpers (__name, __defProp, etc.) don't
+   * crash the facet with "__name is not defined". User preambles are
+   * appended below the shim.
+   */
+  #buildCode(
+    fnSource: string,
+    perCallWasmEntries?: Array<{ name: string; id: string; bytes: ArrayBuffer }>,
+  ) {
+    const workerOpts = {
+      compatibilityDate: CF_COMPAT_DATE,
+      compatibilityFlags: ['nodejs_compat'],
+      // Inherit parent network so the facet can reach registry.npmjs.org.
+      globalOutbound: undefined,
+      env: this.bindings,
+    };
+
+    // ── WASM module imports ───────────────────────────────────────────
+    // Each entry in `wasmModules` (constructor-time + per-call) is
+    // registered in the LOADER's modules map (below) as
+    // `{ wasm: ArrayBuffer }`. workerd compiles each during the
+    // worker's module-load phase (eval permitted there) and the
+    // standard ESM import binding receives the resulting
+    // WebAssembly.Module. We expose them on `globalThis.__NIMBUS_WASM`
+    // so the user fn can read them at request time without having to
+    // re-import (the user fn is serialized via fn.toString and doesn't
+    // carry import statements).
+    //
+    // Per-call entries (passed via LoaderCallOptions.wasmModules
+    // — used by the wasm-runner shell command) are appended to the same
+    // table. Naming collision with constructor entries is rejected
+    // upstream in #materialisePerCallWasm so the import block here
+    // doesn't have to deduplicate.
+    const allWasmEntries = [
+      ...this.wasmModules,
+      ...(perCallWasmEntries ?? []),
+    ];
+    const moduleSource = assembleLoaderWorkerModuleSource({
+      fnSource,
+      preamble: this.preamble,
+      wasmEntries: allWasmEntries,
+      hasBindings: this.bindings !== undefined,
+    });
+
+    // Modules map: the entry worker.js source plus any wasm modules the
+    // pool was constructed with. Workerd parses the modules map at
+    // worker-load time and resolves the static `import` statements
+    // we generated above against this map. The wasm-shape entry
+    // (`{ wasm: ArrayBuffer }`) tells workerd to compile during the
+    // module-load phase — the only phase where wasm code generation
+    // is permitted in this deploy.
+    //
+    // Per-call entries are appended after constructor entries so the
+    // map order matches the import order in the generated worker.js
+    // (matters only for human-readable diffs; workerd doesn't care).
+    const modules: Record<string, ModuleContent> = { 'worker.js': moduleSource };
+    for (const w of allWasmEntries) {
+      modules[w.name] = { wasm: w.bytes };
+    }
+    assertModuleMapWithinCodeLimit(modules);
+
+    return {
+      compatibilityDate: workerOpts.compatibilityDate,
+      compatibilityFlags: workerOpts.compatibilityFlags,
+      mainModule: 'worker.js',
+      modules,
+      env: workerOpts.env,
+      // globalOutbound: undefined = inherit parent network; omitting the key
+      // from the returned object has the same effect (codegen treats
+      // absence as inherit when the key is explicitly stated; here we keep
+      // it absent to match the cloudflare-parallel semantics).
+    } as const;
+  }
+
+  /**
+   * Dispatch a single task to the slot isolate. `slotIndex` picks which
+   * warm isolate services the call; callers round-robin slots themselves.
+   */
+  async #dispatchSlot(
+    fnSource: string,
+    fnHash: string,
+    slotIndex: number,
+    args: unknown[],
+    resilience: ResolvedResilience,
+    perCallWasm?: Record<string, ArrayBuffer>,
+  ): Promise<unknown> {
+    // Per-call wasm fingerprint. Mixed into the cache key so two calls
+    // with different bytes hit different slots (no cache poisoning).
+    // For the common case (no per-call wasm) the fingerprint is '0',
+    // which is bytes-stable so warm reuse is unaffected.
+    const perCallWasmEntries = this.#materialisePerCallWasm(perCallWasm);
+    const perCallWasmHash = this.#fingerprintWasm(perCallWasmEntries);
+
+    // Cache key includes the short DO id so warm isolates are scoped to
+    // ONE session. See the doIdShort field comment for why — without it,
+    // a later session's pool reuses the warm worker from a previous
+    // session (which still carries the old session's env.SUPERVISOR
+    // binding), and writeBatch RPCs land in the wrong DO's VFS.
+    const buildId = (generation: number): string =>
+      `nfp:${this.tag}:${this.doIdShort}:${fnHash}:${this.preambleHash}:${this.wasmHash}:${perCallWasmHash}:slot-${slotIndex}:g${generation}`;
+    let id = buildId(this.slotGenerations.get(slotIndex) ?? 0);
+    const code = this.#buildCode(fnSource, perCallWasmEntries);
+
+    // W5 Lever 5: record the dispatch so /api/_diag/memory shows the
+    // last-facet-id even on a hang or silent kill. Bounded — single
+    // slot updated on every dispatch.
+    try { setLastFacetId(id, slotIndex); } catch { /* best-effort */ }
+
+    const runOnce = async (): Promise<unknown> => {
+      // loader.get() is synchronous from the caller's POV; the callback
+      // is only invoked on cache miss. We wrap the callback tightly so a
+      // retry doesn't rebuild workerCode — that's already stable here.
+      //
+      // The returned `stub` is the cached worker reference. We deliberately
+      // do NOT dispose it: loader.get() is designed for warm-slot reuse
+      // across dispatches (same `id` returns the same cached worker),
+      // and disposing would invalidate that cache.
+      //
+      // We USED to also dispose the per-dispatch `entrypoint` stub in a
+      // finally block here (added in 3c47b44 to prevent QueueState::ACTIVE
+      // during cold-start install). Empirically that broke dispatch on
+      // every slot after the first: once the finally ran
+      // `entrypoint[Symbol.dispose]()`, subsequent dispatches on the
+      // same cached slot hung — SupervisorRPC.writeBatch logged
+      // 'canceled', the DO-side _rpcWriteBatch completed OK, but the
+      // pool never saw the result and every task stalled to the 60s
+      // per-task timeout at 0/13 packages. Removing the per-dispatch
+      // dispose restored 13/13 in ~2.3s in dev.
+      //
+      // The pool-level dispose() method below (also from 3c47b44) is
+      // fine and stays — it only tears down the long-lived SUPERVISOR
+      // binding stub once the whole pool is done, which does NOT
+      // invalidate any in-flight slot's entrypoint reference.
+      const stub = this.loader.get(id, async () => code);
+      recordLoaderId(this.ctx, id);
+      const entrypoint = stub.getEntrypoint();
+      // Direct property call, awaited by this frame — bracketed, never
+      // wrapped. See beginLoaderFetch for the measured DO-poisoning hazard.
+      const endFetch = beginLoaderFetch(this.ctx);
+      try {
+        const out = await entrypoint.execute(...args);
+        return out;
+      } catch (err) {
+        if (err instanceof Error) {
+          throw new ExecutionError(err.message, err.stack);
+        }
+        throw new ExecutionError(String(err));
+      } finally {
+        endFetch();
+      }
+    };
+
+    const maxAttempts = 1 + resilience.retries;
+    let lastError: Error | undefined;
+    let retriedCloneRefusal = false;
+    let attempt = 0;
+    while (attempt < maxAttempts) {
+      try {
+        if (resilience.timeoutMs > 0) {
+          // Race runOnce() against a settable timer. CRITICAL: clear
+          // the timer in a finally so the timer's reject closure (which
+          // transitively roots `args` — i.e. the per-task payload sent
+          // to the slot, including 28 MiB pre-bundle slices) doesn't
+          // hold its references for the full timeoutMs after the race
+          // settles.
+          //
+          // Before this fix: a facet OOM at t=0 left the slice rooted
+          // for the remaining timeoutMs (default 60s for pre-bundle).
+          // With concurrency=2, two consecutive OOMs could pin
+          // ~56 MiB of slice memory in the supervisor heap for a full
+          // minute — alongside an in-flight cirrus-real boot, that's
+          // enough to push a shared isolate over the 128 MiB cap.
+          // See plan in close-plan-2026-04-28.
+          let timerId: ReturnType<typeof setTimeout> | undefined;
+          try {
+            return await Promise.race([
+              runOnce(),
+              new Promise<never>((_, reject) => {
+                timerId = setTimeout(
+                  () => reject(new TimeoutError(resilience.timeoutMs)),
+                  resilience.timeoutMs,
+                );
+              }),
+            ]);
+          } finally {
+            if (timerId !== undefined) clearTimeout(timerId);
+          }
+        }
+        return await runOnce();
+      } catch (err) {
+        lastError = err instanceof Error ? err : new Error(String(err));
+        const cause = classifyError(lastError);
+        // W5 Lever 5: classify + record. We push on EVERY failed
+        // attempt (not just the final retry-exhausted throw) so the
+        // ring captures transient SQLITE_NOMEM / clone-refused
+        // patterns that still ultimately succeed. Ring is bounded
+        // (50 entries) so noise is self-limiting.
+        try {
+          recordFailure({
+            at: Date.now(),
+            phase: 'rpc',
+            cause,
+            rssEstimateBytes: 0, heapUsedBytes: 0,
+            lruBytes: 0, inFlightBytes: 0,
+            lastRpcFrame: getLastRpcFrame(),
+            lastFacetId: { codeId: id, slotIndex, atMs: Date.now() },
+            message: lastError.message,
+          });
+        } catch { /* fail-soft */ }
+        if (cause === 'clone_refused' && !retriedCloneRefusal) {
+          retriedCloneRefusal = true;
+          const generation = (this.slotGenerations.get(slotIndex) ?? 0) + 1;
+          this.slotGenerations.set(slotIndex, generation);
+          id = buildId(generation);
+          try { setLastFacetId(id, slotIndex); } catch { /* best-effort */ }
+          // If the supervisor DO is stale, a newer loader still cannot
+          // deserialize back into it; only recycling that DO heals the
+          // reverse direction. This refresh targets the stale-loader case.
+          continue;
+        }
+        if (attempt < maxAttempts - 1) {
+          // 100 * 2^attempt, capped at 2s so retries don't compound waiting.
+          const delay = Math.min(2000, 100 * Math.pow(2, attempt));
+          await new Promise((r) => setTimeout(r, delay));
+        }
+        attempt++;
+      }
+    }
+    // On the way out only, so retries do not stack the annotation: a cap hit
+    // carries the ledger — which ids hold slots, and that a keyed id never
+    // gives one back — instead of the platform's bare message.
+    const named = withDynamicWorkerCapNamed(this.ctx, lastError!);
+    if (maxAttempts > 1) {
+      throw new RetryExhaustedError(maxAttempts, named);
+    }
+    throw named;
+  }
+
+  #prepare(fn: Function): { fnSource: string; fnHash: string } {
+    const fnSource = serializeFunction(fn);
+    const fnHash = hashSource(fnSource);
+    return { fnSource, fnHash };
+  }
+
+  /**
+   * Run `fn` once with `arg` on a slot isolate. Returns the result or
+   * throws TimeoutError / RetryExhaustedError / ExecutionError.
+   */
+  async submit<T, R>(
+    fn: FacetTaskFn<T, R>,
+    arg: T,
+    opts?: LoaderCallOptions,
+  ): Promise<Awaited<R>> {
+    const { fnSource, fnHash } = this.#prepare(fn);
+    const resilience = this.#resolve(opts);
+    return (await this.#dispatchSlot(
+      fnSource,
+      fnHash,
+      0,
+      [arg],
+      resilience,
+      opts?.wasmModules,
+    )) as Awaited<R>;
+  }
+
+  /**
+   * Run `fn` on every item in `items`, at most `concurrency` at a time,
+   * pinned to stable slots so warm isolates are reused.
+   *
+   * Results are returned in input order. Failure handling per `onError`.
+   */
+  async map<T, R>(
+    fn: FacetTaskFn<T, R>,
+    items: T[],
+    opts?: LoaderMapOptions,
+  ): Promise<Array<Awaited<R> | null>> {
+    if (items.length === 0) return [];
+
+    const { fnSource, fnHash } = this.#prepare(fn);
+    return this.#mapInternal<T, R>(fnSource, fnHash, items, opts);
+  }
+
+  /**
+   * Same shape as `map`, but accepts a pre-serialized function source
+   * string instead of a live function reference. Used by
+   * `FanoutPool`'s peer-DO leg, where the function was already
+   * serialized on the coordinator side and forwarded over RPC.
+   *
+   * The fnSource MUST be the output of `serializeFunction(fn)`
+   * (typically forwarded directly from a coordinator RPC). Bytes-
+   * stable invariants:
+   *   - `fnHash = hashSource(fnSource)` must be deterministic so
+   *     warm slots are correctly keyed.
+   *   - `fnSource` must NOT reference `this` — same rule as
+   *     `serializeFunction`.
+   *
+   * No fn-validation runs here (it already ran on the coordinator);
+   * the peer trusts the caller to forward a valid serialization.
+   */
+  async mapSource<T, R>(
+    fnSource: string,
+    items: T[],
+    opts?: LoaderMapOptions,
+  ): Promise<Array<Awaited<R> | null>> {
+    if (items.length === 0) return [];
+    const fnHash = hashSource(fnSource);
+    return this.#mapInternal<T, R>(fnSource, fnHash, items, opts);
+  }
+
+  async #mapInternal<T, R>(
+    fnSource: string,
+    fnHash: string,
+    items: T[],
+    opts?: LoaderMapOptions,
+  ): Promise<Array<Awaited<R> | null>> {
+    const resilience = this.#resolve(opts);
+    const concurrency = Math.max(
+      1,
+      Math.min(opts?.concurrency ?? this.concurrency, items.length),
+    );
+    const onError: 'throw' | 'null' | 'skip' = opts?.onError ?? 'throw';
+
+    type Settled =
+      | { ok: true; value: Awaited<R> }
+      | { ok: false; error: Error };
+    const settled: Settled[] = new Array(items.length);
+    let cursor = 0;
+
+    const runSlot = async (slotIndex: number): Promise<void> => {
+      while (true) {
+        const idx = cursor++;
+        if (idx >= items.length) return;
+        try {
+          const value = (await this.#dispatchSlot(
+            fnSource,
+            fnHash,
+            slotIndex,
+            [items[idx]],
+            resilience,
+            opts?.wasmModules,
+          )) as Awaited<R>;
+          settled[idx] = { ok: true, value };
+        } catch (err) {
+          const error = err instanceof Error ? err : new Error(String(err));
+          if (onError === 'throw') throw error;
+          settled[idx] = { ok: false, error };
+        }
+      }
+    };
+
+    await Promise.all(
+      Array.from({ length: concurrency }, (_, slotIndex) => runSlot(slotIndex)),
+    );
+
+    if (onError === 'null') {
+      return settled.map((s) => (s.ok ? s.value : null));
+    }
+    if (onError === 'skip') {
+      return settled
+        .filter((s): s is { ok: true; value: Awaited<R> } => s.ok)
+        .map((s) => s.value);
+    }
+    // onError === 'throw' — all slots succeeded.
+    return settled.map((s) => (s as { ok: true; value: Awaited<R> }).value);
+  }
+
+  /**
+   * Release any RPC stubs held by the pool. Call this once the caller
+   * is done with the pool (post-`map`/`submit`) so the underlying
+   * stubs don't linger in workerd's deferred-destruction queue.
+   *
+   * Primary target: the SUPERVISOR binding stub we minted at
+   * construction time (via the registered supervisor entrypoint). It's
+   * a cross-isolate RPC stub — without explicit disposal it stays
+   * referenced until the parent isolate's event-handler context
+   * finishes, which during npm install means "until the whole install
+   * completes" — long enough to accumulate alongside other leaked
+   * stubs and trip the QueueState::ACTIVE fatal.
+   *
+   * Safe to call more than once; idempotent.
+   */
+  dispose(): void {
+    if (!this.bindings) return;
+    for (const key of Object.keys(this.bindings)) {
+      disposeRpcResource(this.bindings[key]);
+    }
+    // Prevent double-dispose from re-running the loop.
+    this.bindings = undefined;
+  }
+}