@nimbus-sh/fabric 0.4.0 → 0.6.0

package/src/isolate-pool.ts

@@ -93,6 +93,15 @@ export interface IsolatePoolOptions {
    * not receive a Supervisor binding and do not retain user state.
    */
   cacheScope?: 'session' | 'global';
+  /**
+   * Baked into the loader id. Two pools sharing tag, preamble, wasm and
+   * supervisor but differing in `scope` never reuse each other's warm
+   * workers: an interpreter scope that ENDS (REPL close, Ctrl-C reset)
+   * must not resurrect the loader-cached heap for the next owner.
+   * Default '' preserves the existing id bytes — only scoped pools get
+   * the extra segment.
+   */
+  scope?: string;
   /**
    * Override the `doId` baked into the auto-injected SUPERVISOR binding.
    * Default: `ctx.id.toString()` (the DO that constructs the pool).
@@ -308,6 +317,9 @@ export function assembleLoaderWorkerModuleSource(
   const callExpr = options.hasBindings
     ? '__fn__(...args, this.env)'
     : '__fn__(...args)';
+  const requestCallExpr = options.hasBindings
+    ? '__fn__(request, this.env)'
+    : '__fn__(request)';
   lines.push(
     'export default class extends WorkerEntrypoint {',
     '  execute(...args) {',
@@ -315,6 +327,26 @@ export function assembleLoaderWorkerModuleSource(
     '    if (result instanceof Promise) return result;',
     '    return result;',
     '  }',
+    '',
+    '  // Request/Response transport: the ONE dispatch path whose abort is',
+    '  // real. workerd cancels the inner execution context when the request',
+    '  // signal aborts while the fn is suspended on I/O; RPC execute() above',
+    '  // has no equivalent (measured: Symbol.dispose on a pending RPC does',
+    '  // not cancel). The fn is request-shaped — it encodes and decodes its',
+    '  // own payload; nothing generic is serialized here.',
+    '  async fetch(request) {',
+    '    try {',
+    `      const result = await ${requestCallExpr};`,
+    '      if (result instanceof Response) return result;',
+    '      return Response.json(result ?? null);',
+    '    } catch (err) {',
+    '      // Fail loud inside the response so an abort of the transport is',
+    '      // not conflated with a guest exception: the caller sees 500 only',
+    '      // for a real fn failure, and aborts arrive as fetch rejections.',
+    '      const message = err instanceof Error && err.message ? err.message : String(err);',
+    '      return Response.json({ __nimbusFacetError: message }, { status: 500 });',
+    '    }',
+    '  }',
     '}',
   );
   return lines.join('\n');
@@ -345,6 +377,18 @@ export class IsolatePool {
   private readonly defaultRetries: number;
   private readonly tag: string;
   private readonly slotGenerations = new Map<number, number>();
+  /**
+   * Per-slot execution ownership: the tail of each slot's in-flight
+   * dispatch chain. Two dispatches on the same warm isolate at once
+   * interleave on its QueueState — map() callers used to trust the
+   * caller's slot round-robin, which could not prevent submit() (slot
+   * 0) or a second map() landing on a slot a task still occupied. Every
+   * dispatch now waits for the slot's previous execution to settle
+   * before touching it.
+   */
+  private readonly slotTails = new Map<number, Promise<void>>();
+  /** Set by dispose() — queued dispatches reject instead of running. */
+  private disposed = false;
   private bindings: Record<string, unknown> | undefined;
 
   private readonly preamble: string | undefined;
@@ -380,7 +424,16 @@ export class IsolatePool {
    * 12 chars is enough entropy for DO ids to collide-free per process.
    */
   private readonly doIdShort: string;
+  /**
+   * The supervisor identity the minted worker's env.SUPERVISOR binding
+   * bakes (doId short-form + pid), folded into the loader cache key. 's-none'
+   * when no SUPERVISOR binding was minted, so a supervisor-less pool keeps
+   * the old key shape and its warm slots stay shared.
+   */
+  private readonly supervisorKey: string;
 
+  /** Extra loader-id segment from options.scope — see IsolatePoolOptions. */
+  private readonly scope: string;
   constructor(
     env: unknown,
     ctx: DurableObjectState,
@@ -408,6 +461,7 @@ export class IsolatePool {
     this.doIdShort = opts?.cacheScope === 'global'
       ? 'global'
       : ctx.id.toString().slice(0, 12);
+    this.scope = opts?.scope ?? '';
 
     // Materialise the wasm-modules table. Sanitise each name into a
     // valid JS identifier for the static import binding; key collisions
@@ -459,6 +513,7 @@ export class IsolatePool {
     }
 
     const bindings: Record<string, unknown> = { ...(opts?.extraBindings ?? {}) };
+    this.supervisorKey = 's-none';
     if (!opts?.omitSupervisor) {
       const supervisorRpc = supervisorEntrypoint();
       if (supervisorRpc) {
@@ -467,9 +522,19 @@ export class IsolatePool {
         // to the user's session DO, not the peer DO. Default to the
         // local ctx.id (single-DO callers and the in-DO in-DO fanout path).
         const supDoId = opts?.supervisorDoIdOverride ?? ctx.id.toString();
+        const supPid = opts?.supervisorPid ?? 0;
         bindings.SUPERVISOR = supervisorRpc({
-          props: { doId: supDoId, pid: opts?.supervisorPid ?? 0 },
+          props: { doId: supDoId, pid: supPid },
         });
+        // Whatever the minted worker's env carries must be in its loader
+        // cache key — workerd's loader cache survives a DO hibernation
+        // wake while generation-strided pids (1000001 → 2000001) do not:
+        // a warm slot keyed without the supervisor identity returns in
+        // the new generation still credentialed to the dead pid, and
+        // every pid-authorized RPC from it fails "process pid … does
+        // not exist". doIdShort alone cannot cover this — it changes
+        // across sessions, not across wakes of the same session.
+        this.supervisorKey = `s${supDoId.slice(0, 12)}-${supPid}`;
       } else {
         // Supervisor entrypoint unavailable — running without ctx.exports
         // (e.g. unit-test harness, or LOADER.load contexts where the
@@ -669,15 +734,59 @@ export class IsolatePool {
   /**
    * Dispatch a single task to the slot isolate. `slotIndex` picks which
    * warm isolate services the call; callers round-robin slots themselves.
+   * Serialized per slot: this call waits for the slot's previous
+   * execution to settle before touching the warm isolate.
+   *
+   * `invoke` is the single-attempt call against the slot's entrypoint
+   * stub. `execute` dispatches call `entrypoint.execute(...args)`;
+   * `fetch` dispatches call `entrypoint.fetch(<per-attempt Request>)` —
+   * the attempt index lets a fetch invoke mint a fresh clone for retries,
+   * since a Request's body is consumed once.
    */
-  async #dispatchSlot(
+  async #dispatchSlot<T>(
     fnSource: string,
     fnHash: string,
     slotIndex: number,
-    args: unknown[],
+    invoke: (entrypoint: { execute(...args: unknown[]): Promise<unknown>; fetch(input: RequestInfo, init?: RequestInit): Promise<Response> }, attempt: number) => Promise<T>,
     resilience: ResolvedResilience,
     perCallWasm?: Record<string, ArrayBuffer>,
-  ): Promise<unknown> {
+    wasAborted?: () => boolean,
+  ): Promise<T> {
+    // A warm slot executes one dispatch at a time: queue behind the
+    // previous owner, then record this dispatch as the new tail. The
+    // tail outlives the caller's outcome: a timeout rejects the
+    // Promise.race while runOnce's RPC is still live on the Worker, so
+    // release waits for every execution the owned body started to
+    // settle — never the caller's settle alone.
+    const previous = this.slotTails.get(slotIndex) ?? Promise.resolve();
+    let release: () => void;
+    this.slotTails.set(slotIndex, new Promise<void>((resolve) => { release = resolve; }));
+    await previous;
+    if (this.disposed) {
+      release!();
+      throw new BindingError(`IsolatePool(${this.tag}) is disposed`);
+    }
+    const inFlight: Promise<unknown>[] = [];
+    try {
+      return await this.#dispatchSlotOwned(fnSource, fnHash, slotIndex, invoke, resilience, perCallWasm, inFlight, wasAborted);
+    } finally {
+      // Do not delay the caller's own outcome — the tail releases when
+      // the RPCs the body launched have actually settled.
+      void Promise.allSettled(inFlight).then(() => release!());
+    }
+  }
+
+
+  async #dispatchSlotOwned<T>(
+    fnSource: string,
+    fnHash: string,
+    slotIndex: number,
+    invoke: (entrypoint: { execute(...args: unknown[]): Promise<unknown>; fetch(input: RequestInfo, init?: RequestInit): Promise<Response> }, attempt: number) => Promise<T>,
+    resilience: ResolvedResilience,
+    perCallWasm: Record<string, ArrayBuffer> | undefined,
+    inFlight: Promise<unknown>[],
+    wasAborted?: () => boolean,
+  ): Promise<T> {
     // Per-call wasm fingerprint. Mixed into the cache key so two calls
     // with different bytes hit different slots (no cache poisoning).
     // For the common case (no per-call wasm) the fingerprint is '0',
@@ -686,12 +795,12 @@ export class IsolatePool {
     const perCallWasmHash = this.#fingerprintWasm(perCallWasmEntries);
 
     // Cache key includes the short DO id so warm isolates are scoped to
-    // ONE session. See the doIdShort field comment for why — without it,
-    // a later session's pool reuses the warm worker from a previous
-    // session (which still carries the old session's env.SUPERVISOR
-    // binding), and writeBatch RPCs land in the wrong DO's VFS.
+    // ONE session (see the doIdShort field comment), and the supervisor
+    // identity so a wake of that same session can never reuse a warm
+    // worker whose SUPERVISOR binding still names the dead generation's
+    // pid. See the supervisorKey field comment for the failure mode.
     const buildId = (generation: number): string =>
-      `nfp:${this.tag}:${this.doIdShort}:${fnHash}:${this.preambleHash}:${this.wasmHash}:${perCallWasmHash}:slot-${slotIndex}:g${generation}`;
+      `nfp:${this.tag}:${this.doIdShort}:${fnHash}:${this.preambleHash}:${this.wasmHash}:${perCallWasmHash}:${this.supervisorKey}:slot-${slotIndex}:g${generation}${this.scope ? `:${this.scope}` : ''}`;
     let id = buildId(this.slotGenerations.get(slotIndex) ?? 0);
     const code = this.#buildCode(fnSource, perCallWasmEntries);
 
@@ -700,7 +809,7 @@ export class IsolatePool {
     // slot updated on every dispatch.
     try { setLastFacetId(id, slotIndex); } catch { /* best-effort */ }
 
-    const runOnce = async (): Promise<unknown> => {
+    const runOnce = async (): Promise<T> => {
       // loader.get() is synchronous from the caller's POV; the callback
       // is only invoked on cache miss. We wrap the callback tightly so a
       // retry doesn't rebuild workerCode — that's already stable here.
@@ -732,8 +841,7 @@ export class IsolatePool {
       // wrapped. See beginLoaderFetch for the measured DO-poisoning hazard.
       const endFetch = beginLoaderFetch(this.ctx);
       try {
-        const out = await entrypoint.execute(...args);
-        return out;
+        return await invoke(entrypoint, attempt);
       } catch (err) {
         if (err instanceof Error) {
           throw new ExecutionError(err.message, err.stack);
@@ -767,8 +875,10 @@ export class IsolatePool {
           // See plan in close-plan-2026-04-28.
           let timerId: ReturnType<typeof setTimeout> | undefined;
           try {
+            const attemptPromise = runOnce();
+            inFlight.push(attemptPromise);
             return await Promise.race([
-              runOnce(),
+              attemptPromise,
               new Promise<never>((_, reject) => {
                 timerId = setTimeout(
                   () => reject(new TimeoutError(resilience.timeoutMs)),
@@ -780,7 +890,9 @@ export class IsolatePool {
             if (timerId !== undefined) clearTimeout(timerId);
           }
         }
-        return await runOnce();
+        const attemptPromise = runOnce();
+        inFlight.push(attemptPromise);
+        return await attemptPromise;
       } catch (err) {
         lastError = err instanceof Error ? err : new Error(String(err));
         const cause = classifyError(lastError);
@@ -801,6 +913,18 @@ export class IsolatePool {
             message: lastError.message,
           });
         } catch { /* fail-soft */ }
+        if (wasAborted?.()) {
+          // The caller aborted this dispatch's Request. workerd cancelled
+          // the isolate's execution context wherever it was suspended —
+          // mid-syscall, mid-stream — so the interpreter's heap may hold
+          // half-applied state. Bump the slot generation: the next
+          // dispatch lands on a FRESH worker under a new id rather than
+          // reusing the abandoned one. Retrying is wrong — the user
+          // interrupted; surface the abort.
+          const generation = (this.slotGenerations.get(slotIndex) ?? 0) + 1;
+          this.slotGenerations.set(slotIndex, generation);
+          throw lastError;
+        }
         if (cause === 'clone_refused' && !retriedCloneRefusal) {
           retriedCloneRefusal = true;
           const generation = (this.slotGenerations.get(slotIndex) ?? 0) + 1;
@@ -851,12 +975,49 @@ export class IsolatePool {
       fnSource,
       fnHash,
       0,
-      [arg],
+      (entrypoint) => entrypoint.execute(arg) as Promise<Awaited<R>>,
       resilience,
       opts?.wasmModules,
-    )) as Awaited<R>;
+    ));
   }
 
+  /**
+   * Dispatch `fn` through the fetch transport — the pool's only
+   * cancellable path: aborting `request.signal` cancels the inner
+   * execution context at its next I/O suspension (a synchronous CPU
+   * section cannot be preempted — the platform's honest limit), the
+   * call rejects, and the slot generation bumps so the next dispatch
+   * gets a fresh interpreter. `fn` is request-shaped: it encodes and
+   * decodes its own payload.
+   */
+  async submitRequest(
+    fn: (request: Request, env: FacetBindings) => Response | Promise<Response>,
+    request: Request,
+    opts?: IsolateCallOptions,
+  ): Promise<Response> {
+    if (request.bodyUsed) {
+      throw new BindingError(
+        'IsolatePool.submitRequest: request body is already consumed — ' +
+          'pass an unconsumed Request so retries can re-issue it.',
+      );
+    }
+    const { fnSource, fnHash } = this.#prepare(fn);
+    const resilience = this.#resolve(opts);
+    // Every attempt fetches a clone: a Request's body is consumed once,
+    // so the caller's original stays unspent and retries get a fresh
+    // body that follows the same signal.
+    return this.#dispatchSlot(
+      fnSource,
+      fnHash,
+      0,
+      (entrypoint) => entrypoint.fetch(request.clone()),
+      resilience,
+      opts?.wasmModules,
+      () => request.signal.aborted,
+    );
+  }
+
+
   /**
    * Run `fn` on every item in `items`, at most `concurrency` at a time,
    * pinned to stable slots so warm isolates are reused.
@@ -929,7 +1090,7 @@ export class IsolatePool {
             fnSource,
             fnHash,
             slotIndex,
-            [items[idx]],
+            (entrypoint) => entrypoint.execute(items[idx]),
             resilience,
             opts?.wasmModules,
           )) as Awaited<R>;
@@ -974,6 +1135,7 @@ export class IsolatePool {
    * Safe to call more than once; idempotent.
    */
   dispose(): void {
+    this.disposed = true;
     if (!this.bindings) return;
     for (const key of Object.keys(this.bindings)) {
       disposeRpcResource(this.bindings[key]);

package/src/process-host.ts

@@ -67,6 +67,8 @@
 import { disposeRpcResource } from '@nimbus-sh/platform/rpc-dispose.js';
 import { isTransientDoReset } from '@nimbus-sh/platform/oom-classify.js';
 import { PEER_RETRY_BACKOFF_MS, PEER_TRANSIENT_RESET_RETRIES } from './fanout.js';
+import { hostNamespaceBinding, hostOpDispatch, type HostNamespaceBinding } from './host-dispatch.js';
+import { z } from 'zod/v4';
 import {
   type HostedProcess,
   type OneShotParams,
@@ -219,20 +221,6 @@ export interface HostedHttpResponse {
   body: ReadableStream | null;
 }
 
-/**
- * Every method a hosting sibling must answer. Checked as a set at first
- * contact: a stub that has the probe but not the router is a deployment skew,
- * and finding that out mid-process is finding it out too late.
- */
-const PROCESS_PEER_METHODS = [
-  '_rpcProcessHostProbe',
-  '_rpcHostProcess',
-  '_rpcAwaitHostedOpen',
-  '_rpcAwaitHostedBoot',
-  '_rpcRouteHostedHttp',
-  '_rpcCancelHostProcess',
-] as const;
-
 /** The host-process surface a sibling session DO exposes. */
 interface ProcessPeerStub {
   _rpcProcessHostProbe(): Promise<{ isolateToken: string }>;
@@ -257,41 +245,70 @@ interface ProcessPeerStub {
 export const HOSTED_WEBSOCKET_KEY_HEADER = 'x-nimbus-hosted-websocket';
 export const HOSTED_WEBSOCKET_CAPABILITY_HEADER = 'x-nimbus-hosted-websocket-capability';
 
-interface PeerNamespace {
-  idFromName(name: string): unknown;
-  get(id: unknown): unknown;
+/**
+ * Peer stubs forward one supervisorOp entrypoint: every method the host-
+ * process surface needs is an envelope op, not a private _rpc* method.
+ */
+function peerNamespace(env: object): HostNamespaceBinding {
+  try {
+    return hostNamespaceBinding(env, 'ProcessFabric');
+  } catch (e) {
+    if (e instanceof BindingError) {
+      throw new BindingError(
+        e.message
+          + " NIMBUS_PROCESS_HOST='peer' hosts every resident process on a sibling "
+          + "Durable Object; name the host's own binding with composeFabric({ hostNamespace }).",
+      );
+    }
+    throw e;
+  }
 }
 
-function peerNamespace(env: unknown): PeerNamespace {
-  const ns = (typeof env === 'object' || typeof env === 'function') && env !== null
-    ? Reflect.get(env, 'NIMBUS_SESSION')
+const PeerProbeResult = z.object({ isolateToken: z.string() });
+const PeerOpenResult = z.object({ ok: z.boolean() });
+const PeerBootResult = z.object({ payload: z.unknown() });
+const PeerCancelResult = z.object({ cancelled: z.boolean() });
+const PeerHttpResult = z.object({
+  status: z.number().int(),
+  statusText: z.string(),
+  headers: z.array(z.tuple([z.string(), z.string()])),
+  body: z.instanceof(ReadableStream).nullable(),
+});
+
+function processPeerStub(value: object, peerName: string): ProcessPeerStub {
+  const dispatch = hostOpDispatch(value, `ProcessFabric peer '${peerName}'`);
+  const fetchFn = (typeof value === 'object' || typeof value === 'function') && value !== null
+    ? Reflect.get(value, 'fetch')
     : undefined;
-  if ((typeof ns !== 'object' && typeof ns !== 'function') || ns === null
-    || typeof Reflect.get(ns, 'idFromName') !== 'function'
-    || typeof Reflect.get(ns, 'get') !== 'function') {
+  if (typeof fetchFn !== 'function') {
     throw new BindingError(
-      'ProcessFabric: env.NIMBUS_SESSION binding missing or invalid. '
-        + "NIMBUS_PROCESS_HOST='peer' hosts every resident process on a sibling "
-        + 'Durable Object; add the binding via durable_objects.bindings in wrangler.jsonc.',
+      `ProcessFabric: peer '${peerName}' exposes no fetch() — the hosted-websocket `
+        + 'upgrade leg still travels as a service-binding fetch.',
     );
   }
-  return ns as unknown as PeerNamespace;
-}
-
-function processPeerStub(value: unknown, peerName: string): ProcessPeerStub {
-  if ((typeof value !== 'object' && typeof value !== 'function') || value === null) {
-    throw new BindingError(`ProcessFabric: NIMBUS_SESSION.get() returned no stub for peer '${peerName}'.`);
-  }
-  for (const method of PROCESS_PEER_METHODS) {
-    if (typeof Reflect.get(value, method) !== 'function') {
-      throw new BindingError(
-        `ProcessFabric: peer '${peerName}' does not expose ${method}(). `
-          + 'A sibling that answers some of the host-process surface and not the rest '
-          + 'would fail somewhere inside a running process instead of here.',
-      );
-    }
+  const adapter: ProcessPeerStub = {
+    _rpcProcessHostProbe: async () => PeerProbeResult.parse(await dispatch({ op: 'processHostProbe', args: [] })),
+    _rpcHostProcess: async (boot, opts) => PeerOpenResult.parse(await dispatch({ op: 'hostProcess', args: [boot, opts] })),
+    _rpcAwaitHostedOpen: async (key) => PeerOpenResult.parse(await dispatch({ op: 'awaitHostedOpen', args: [key] })),
+    _rpcAwaitHostedBoot: async (key) => {
+      const result = PeerBootResult.parse(await dispatch({ op: 'awaitHostedBoot', args: [key] }));
+      return { payload: result.payload };
+    },
+    _rpcRouteHostedHttp: async (key, request) => PeerHttpResult.parse(await dispatch({ op: 'routeHostedHttp', args: [key, request] })),
+    _rpcCancelHostProcess: async (key) => PeerCancelResult.parse(await dispatch({ op: 'cancelHostProcess', args: [key] })),
+    fetch: async (request) => {
+      const response = await Reflect.apply(fetchFn, value, [request]);
+      if (!(response instanceof Response)) throw new BindingError(`ProcessFabric: peer '${peerName}' returned no Response`);
+      return response;
+    },
+  };
+  // The adapter wraps the raw stub; releasing the adapter must release it —
+  // a held RPC stub pins the peer DO for the session's life.
+  const disposerKey = Reflect.get(Symbol, 'dispose');
+  if (typeof disposerKey === 'symbol') {
+    Object.defineProperty(adapter, disposerKey, { value: () => disposeRpcResource(value) });
   }
-  return value as unknown as ProcessPeerStub;
+  return adapter;
 }
 
 interface PeerPlacement {
@@ -312,13 +329,16 @@ class PeerProcessHost implements ProcessHost {
     storageSharedWithSession: false,
   };
 
-  private readonly ns: PeerNamespace;
+  private readonly ns: HostNamespaceBinding;
   private readonly env: ResidentFacetEnv;
   private readonly coordDoId: string;
   /** pid → the isolate token of the peer currently hosting that process. */
   private readonly tokensInUse = new Map<number, string>();
 
   constructor(private readonly ctx: DurableObjectState, env: unknown) {
+    if (env === null || (typeof env !== 'object' && typeof env !== 'function')) {
+      throw new BindingError('ProcessFabric: a peer host requires environment bindings');
+    }
     this.ns = peerNamespace(env);
     this.env = (env ?? {}) as ResidentFacetEnv;
     this.coordDoId = ctx.id.toString();
@@ -445,7 +465,7 @@ class PeerProcessHost implements ProcessHost {
   /** One sibling name, resolved and probed. Leaks nothing on failure. */
   private async _probePlacement(pid: number, attempt: number): Promise<PeerPlacement> {
     const peerName = `${this.coordDoId}:proc:${pid}:${attempt}`;
-    let resource: unknown;
+    let resource: DurableObjectStub | null = null;
     try {
       resource = this.ns.get(this.ns.idFromName(peerName));
       const stub = processPeerStub(resource, peerName);

package/src/turn-budget.ts

@@ -63,10 +63,10 @@ export const TURN_CHUNK_MAX_BYTES = 2_000_000;
  * spent.
  *
  * Callers report the work they are about to do or have just done and await
- * the result; a pacer that is not yielding returns without suspending, so the
- * one-shot exec path — which passes no pacer at all — keeps its exact
- * behaviour and cost. Nothing here decides WHAT the launch does, only where it
- * is allowed to stop.
+ * the result; a pacer that is not yielding returns without suspending, so a
+ * build small enough to fit in one chunk — most one-shot execs, every cache
+ * hit — keeps its exact behaviour and cost. Nothing here decides WHAT the
+ * launch does, only where it is allowed to stop.
  */
 export class TurnBudget {
   /** Turn handoffs this launch has taken. Reported with the launch. */