@nimbus-sh/fabric 0.7.1 → 0.9.0

package/src/fanout.ts

@@ -1,25 +1,30 @@
 /**
  * Two-tier fan-out primitive for work that must execute in Worker Loader
- * facets without tripping workerd's per-DO dynamic-worker ceiling.
+ * isolates within the Durable Object's Dynamic Worker budget.
  *
- * A single Durable Object method can drive at most four concurrent
- * Worker Loader fetches before extra dispatches serialize or fail. Small
- * batches therefore run in the coordinator DO through IsolatePool.
- * Wider batches are sharded across sibling NimbusSession DOs, each of
- * which owns its own four-loader budget.
+ * A Durable Object may have `DO_DYNAMIC_WORKER_LIMIT` distinct Dynamic
+ * Workers with in-flight requests, shared across every concurrent request to
+ * it (budgets.ts). A batch the coordinator's remaining headroom can hold —
+ * the limit less the workers it already has in flight (resident processes,
+ * the esbuild facet, a git network op) and other fan-outs' claims — runs in
+ * the coordinator through IsolatePool, one Dynamic Worker per task. Only a
+ * batch wider than that headroom is sharded across sibling NimbusSession
+ * DOs, each of which spends its own budget.
  *
- * Routing is deterministic: each task has a stable key, and the key maps
- * to a sibling DO shard. There is no silent fallback to width-1 execution;
- * missing LOADER or NIMBUS_SESSION bindings fail loudly so install and
- * runtime operations do not appear successful after partial dispatch.
+ * Routing is deterministic for a given headroom: each task has a stable key,
+ * and the key maps to a sibling DO shard. There is no silent fallback to
+ * width-1 execution; missing LOADER or NIMBUS_SESSION bindings fail loudly so
+ * install and runtime operations do not appear successful after partial
+ * dispatch.
  */
 
 import { serializeFunction } from './vendor/serialize.js';
 import { BindingError } from './vendor/errors.js';
 import { IsolatePool, type FacetTaskFn } from './isolate-pool.js';
+import { claimDynamicWorkers, dynamicWorkerHeadroom } from './budgets.js';
 import { hostRoute } from './composition.js';
 import { disposeRpcResource } from '@nimbus-sh/platform/rpc-dispose.js';
-import { describeError, isDoOverloaded, isTransientDoReset } from '@nimbus-sh/platform/oom-classify.js';
+import { classifyDoCall, describeError, isRetryableDoCall } from '@nimbus-sh/platform/oom-classify.js';
 import type { WorkerLoader } from './vendor/types.js';
 import {
   hostNamespaceBinding,
@@ -35,14 +40,17 @@ export interface FanoutEnv {
   NIMBUS_SESSION?: unknown;
 }
 
-/**
- * Threshold at which routing switches from coordinator-local loaders to
- * sibling Durable Objects.
- *
- * Set to **5** so the in-DO path stays below the V8 4-loaders-per-method
- * cap by construction. width < 5 stays local; width >= 5 uses sibling DOs.
- */
-export const IN_DO_THRESHOLD = 5;
+/** Where one `submitMany` ran. */
+export type FanoutTopology = 'in-do' | 'peer-do';
+
+/** The routing decision for one `submitMany`, as its caller may log it. */
+export interface FanoutRoute {
+  topology: FanoutTopology;
+  /** Tasks in the batch — the Dynamic Workers an in-DO run spends. */
+  tasks: number;
+  /** The coordinator's Dynamic Worker headroom when the batch was routed. */
+  headroom: number;
+}
 
 /**
  * Hard cap on concurrent peer DOs per single submitMany call. Throughput stays
@@ -153,12 +161,14 @@ export interface FanoutOptions {
    * Not called on the in-DO path, which has no phases.
    */
   onDispatchPhase?: (width: number, elapsedMs: number) => void;
+  /** Called once per non-empty submitMany with the route it took. */
+  onRoute?: (route: FanoutRoute) => void;
   /**
    * Cap on peer DOs this pool will spread one submitMany across. Defaults to
    * MAX_PEER_FANOUT. Tasks beyond the cap bucket into the peers that exist and
    * run through their in-peer pool, so lowering it trades peers for barriers
-   * without lowering total concurrency: each peer runs its bucket at
-   * concurrency 4, so N peers still resolve 4N tasks at once.
+   * without lowering total concurrency: each peer runs its bucket as wide as
+   * its own Dynamic Worker headroom allows.
    *
    * A caller sets this when its per-task work is small enough that a peer per
    * task buys nothing but round-trips — one task per peer costs ⌈tasks/
@@ -178,7 +188,7 @@ function isPeerResult<R>(value: Awaited<ReturnType<HostOpDispatch>>): value is F
 
 /**
  * Two-tier fan-out pool. Constructed by the supervisor DO; routes
- * each `submitMany` call automatically based on width.
+ * each `submitMany` call on the coordinator's live Dynamic Worker headroom.
  *
  * Lifetime: cheap to construct (no async init). Multiple submitMany
  * calls share NO state — each is dispatched fresh. The class
@@ -216,16 +226,16 @@ export class Fanout {
    * Dispatch `tasks` across the appropriate topology and return
    * results in input order.
    *
-   * Routing:
-   *   tasks.length < 5   -> coordinator-local IsolatePool
-   *   tasks.length >= 5  -> sibling NimbusSession DOs
+   * Routing, against the coordinator's Dynamic Worker headroom at call time:
+   *   tasks.length <= headroom -> coordinator-local IsolatePool, one Dynamic
+   *                               Worker per task, the width claimed on the
+   *                               ledger until the batch settles
+   *   tasks.length >  headroom -> sibling NimbusSession DOs
    *
-   * Backpressure: if `tasks.length > MAX_PEER_FANOUT (32)`, tasks
-   * are sharded modulo `MAX_PEER_FANOUT` and each shard's bucket
-   * runs serially inside its assigned peer DO via the in-peer
-   * IsolatePool's concurrency (capped at 4 there too). A
-   * single submitMany call returns when ALL tasks complete (or any
-   * throws).
+   * Peer shards: tasks hash onto min(tasks, maxPeers ?? MAX_PEER_FANOUT)
+   * peers and each peer runs its bucket through its own IsolatePool, as
+   * wide as that peer's headroom allows. A single submitMany call returns
+   * when ALL tasks complete (or any throws).
    *
    * `fn` is the user function executed per task. It runs INSIDE a
    * Worker Loader isolate (in the in-DO path) or inside a peer DO's
@@ -239,16 +249,21 @@ export class Fanout {
   ): Promise<R[]> {
     if (tasks.length === 0) return [];
 
-    if (tasks.length < IN_DO_THRESHOLD) {
-      return this._dispatchInDo<A, R>(tasks, fn);
+    const headroom = dynamicWorkerHeadroom(this.ctx);
+    const claim = claimDynamicWorkers(this.ctx, tasks.length);
+    this.opts.onRoute?.({ topology: claim ? 'in-do' : 'peer-do', tasks: tasks.length, headroom });
+    if (!claim) return this._dispatchPeerDo<A, R>(tasks, fn);
+    try {
+      return await this._dispatchInDo<A, R>(tasks, fn);
+    } finally {
+      claim.release();
     }
-    return this._dispatchPeerDo<A, R>(tasks, fn);
   }
 
-  /** Report which topology a task count uses without dispatching. */
-  topologyFor(taskCount: number): 'in-do' | 'peer-do' | 'empty' {
+  /** The topology a task count would take against the headroom right now. */
+  topologyFor(taskCount: number): FanoutTopology | 'empty' {
     if (taskCount === 0) return 'empty';
-    return taskCount < IN_DO_THRESHOLD ? 'in-do' : 'peer-do';
+    return taskCount <= dynamicWorkerHeadroom(this.ctx) ? 'in-do' : 'peer-do';
   }
 
   /**
@@ -269,13 +284,10 @@ export class Fanout {
     tasks: FanoutTask<A>[],
     fn: FacetTaskFn<A, R>,
   ): Promise<R[]> {
-    // Use the existing IsolatePool. Concurrency = task count
-    // (capped at 4 by constructor — tasks.length is already < 5
-    // here, so the cap won't bite). Each task = one pool.submit;
-    // pool.map runs them with stable-slot reuse.
-    const concurrency = Math.min(tasks.length, IN_DO_THRESHOLD - 1);
+    // One slot — one Dynamic Worker — per task; submitMany has claimed
+    // that width on the ledger.
     const pool = new IsolatePool(this.env, this.ctx, {
-      concurrency,
+      concurrency: tasks.length,
       timeoutMs: this.opts.timeoutMs,
       tag: this.opts.tag,
       preamble: this.opts.preamble,
@@ -285,9 +297,6 @@ export class Fanout {
       supervisorPid: this.opts.supervisorPid,
     });
     try {
-      // pool.map runs the function over `items` with concurrency-bounded
-      // slot reuse. Each slot is one warm loader isolate; we get exactly
-      // `concurrency` loader isolates total — well under the 4-cap.
       const items = tasks.map((t) => t.args);
       const results = await pool.map<A, R>(fn, items);
       // pool.map returns Array<R | null> (null on per-item failure with
@@ -316,10 +325,9 @@ export class Fanout {
     // identical fns.
     const fnSource = serializeFunction(fn);
 
-    // Cap peer count at MAX_PEER_FANOUT. Tasks beyond N=32 are
-    // bucketed into existing shards — each shard's peer DO then
-    // runs its bucket through its in-DO IsolatePool.map
-    // (concurrency capped at 4 there).
+    // Cap peer count at maxPeers (default MAX_PEER_FANOUT). Tasks beyond
+    // it bucket into existing shards; each peer runs its bucket through
+    // its own IsolatePool, as wide as its own headroom allows.
     const peerCount = Math.min(tasks.length, this.opts.maxPeers ?? MAX_PEER_FANOUT);
     // Group tasks by deterministic shard. Same key → same shard, so
     // tests can predict which peer handles which task.
@@ -360,10 +368,9 @@ export class Fanout {
           const peerStub = ns.get(id);
           try {
             const dispatch = hostOpDispatch(peerStub, `Fanout peer ${siblingName}`);
-            // Each peer DO RPC call uses ONE LOADER worker on its side.
-            // Supervisor → peer DO is a stub.fetch / RPC method call,
-            // NOT an env.LOADER.get(); that's the cap-sidestep that
-            // makes peer-DO fanout work.
+            // A peer DO call is a Durable Object RPC, not a Dynamic Worker:
+            // it spends none of this coordinator's budget, and the peer
+            // spends its own.
             const rpcResp = await dispatch({
               op: 'fanoutExecute',
               args: [
@@ -414,8 +421,9 @@ export class Fanout {
             }
             return;
           } catch (err) {
-            const schedule = isTransientDoReset(err) ? PEER_RETRY_BACKOFF_MS
-              : isDoOverloaded(err) ? PEER_OVERLOAD_BACKOFF_MS
+            const cls = classifyDoCall(err);
+            const schedule = cls === 'overloaded' ? PEER_OVERLOAD_BACKOFF_MS
+              : isRetryableDoCall(cls) ? PEER_RETRY_BACKOFF_MS
               : null;
             if (schedule && attempt < PEER_TRANSIENT_RESET_RETRIES) {
               const backoff = schedule[Math.min(attempt, schedule.length - 1)];

package/src/fenced-work.ts

@@ -56,9 +56,9 @@ export const FENCED_WORK_MAX_ATTEMPT = 1;
  * (staging, 2026-08-13): every observed reset struck seconds AFTER the launch
  * settled — the platform kills the object while the resident runs, which is
  * when a launch-scoped row had already been deleted and recovery had nothing
- * to find. A resident's facet cannot outlive its session instance (the
- * process host's held-open leg dies with it), so a row from a previous
- * generation always names a process that is genuinely gone.
+ * to find. A resident's facet can outlive its session instance (a pending
+ * timer keeps it running), but nothing routes to it from the next one, and a
+ * spawn that takes its facet name ends it first.
  */
 export interface FencedWorkRecord {
   pid: number;

package/src/host-wasm.ts

@@ -0,0 +1,41 @@
+/**
+ * host-wasm.ts — what the fabric knows about a WebAssembly.Module a host
+ * hands a dynamic worker already compiled.
+ *
+ * workerd accepts a compiled module in a Worker Loader module map and lets
+ * the dynamic worker share its compiled code (src/workerd/api/
+ * worker-loader.c++, extractWasmModuleContent), so a host that bundles a
+ * fixed wasm (esbuild's, say) hands that module over instead of fetching a
+ * second copy of the bytes and having the guest compile them again. Two
+ * things the fabric needs about a member are unreadable from JS for a
+ * Module:
+ *   - its size: the module's wire bytes still count toward the 64 MiB
+ *     dynamic-worker code limit (worker-loader.c++ sums every member), and
+ *     assertModuleMapWithinCodeLimit must count them;
+ *   - its identity: a pool folds its wasm into the loader cache key, and a
+ *     Module has no bytes to fingerprint.
+ * The host that owns the module states both once, here.
+ */
+
+export interface HostWasmIdentity {
+  /** A stable name for the module's content, e.g. `esbuild@0.24.2`. */
+  readonly id: string;
+  /** The module's wire bytes: what it costs the dynamic-worker code budget. */
+  readonly bytes: number;
+}
+
+const described = new WeakMap<WebAssembly.Module, HostWasmIdentity>();
+
+/** Record what `module` is, and return it. */
+export function describeHostWasm(module: WebAssembly.Module, identity: HostWasmIdentity): WebAssembly.Module {
+  if (!Number.isSafeInteger(identity.bytes) || identity.bytes <= 0) {
+    throw new RangeError(`describeHostWasm: '${identity.id}' needs its wire size in bytes, got ${identity.bytes}`);
+  }
+  described.set(module, identity);
+  return module;
+}
+
+/** What a host said `module` is, or undefined for a module nobody described. */
+export function hostWasmIdentity(module: WebAssembly.Module): HostWasmIdentity | undefined {
+  return described.get(module);
+}

package/src/image-store.ts

@@ -122,7 +122,7 @@ export class ImageStore {
    */
   async materialize(
     pid: number,
-    images: AsyncIterable<readonly [string, string]> | Iterable<readonly [string, string]>,
+    images: AsyncIterable<readonly [string, string | readonly string[]]> | Iterable<readonly [string, string | readonly string[]]>,
     pacer: TurnBudget,
   ): Promise<Record<string, string>> {
     const fs = this.blobs();
@@ -138,12 +138,12 @@ export class ImageStore {
     fs.mkdirp(FACET_IMAGE_DIR);
     let count = 0;
     for await (const [moduleName, source] of images) {
-      const path = facetImagePath(await facetImageDigest(source));
+      const bytes = typeof source === 'string' ? new TextEncoder().encode(source) : encodeParts(source);
+      const path = facetImagePath(await facetImageDigest(bytes));
       paths[moduleName] = path;
       rooted.push(path);
       count++;
       const stored = path.replace(/^\/+/, '');
-      const bytes = new TextEncoder().encode(source);
       console.log(
         '[image-store] pid=' + pid + ' image ' + count + ' ' + moduleName + ' → '
         + path.slice(-12) + ' ' + bytes.byteLength + ' bytes, slice=' + FACET_IMAGE_WRITE_SLICE_BYTES
@@ -207,3 +207,30 @@ export class ImageStore {
     }
   }
 }
+
+/**
+ * UTF-8 bytes of an image given as ordered parts, encoded into one buffer a
+ * part at a time: joining the parts first would hold the whole image twice as
+ * text.
+ */
+function encodeParts(parts: readonly string[]): Uint8Array {
+  let length = 0;
+  for (const part of parts) {
+    for (let i = 0; i < part.length; i++) {
+      const code = part.charCodeAt(i);
+      if (code < 0x80) length += 1;
+      else if (code < 0x800) length += 2;
+      else if (code >= 0xd800 && code <= 0xdbff && (part.charCodeAt(i + 1) & 0xfc00) === 0xdc00) { length += 4; i++; }
+      else length += 3;
+    }
+  }
+  const bytes = new Uint8Array(length);
+  const encoder = new TextEncoder();
+  let offset = 0;
+  for (const part of parts) {
+    const encoded = encoder.encode(part);
+    bytes.set(encoded, offset);
+    offset += encoded.byteLength;
+  }
+  return bytes;
+}

package/src/inner-do-registry.ts

@@ -56,3 +56,34 @@ export function clearInnerDoClasses(supervisorDoId: string): void {
     if (k.startsWith(prefix)) _NIMBUS_INNER_DO_CLASSES.delete(k);
   }
 }
+
+/** Inner-DO facet names each incarnation has opened, by binding; keyed off ctx, so a new incarnation starts empty. */
+const openedInnerDoFacets = new WeakMap<DurableObjectState, Map<string, Set<string>>>();
+
+/**
+ * Record that this incarnation opens `facetName` for `bindingName`. True on the
+ * name's first open since the incarnation began or the binding's facets were
+ * last aborted: whatever still runs under it has a class this build no longer
+ * uses, and a get() with the new class on it resets the whole object.
+ */
+export function noteInnerDoFacetOpened(ctx: DurableObjectState, bindingName: string, facetName: string): boolean {
+  let byBinding = openedInnerDoFacets.get(ctx);
+  if (!byBinding) openedInnerDoFacets.set(ctx, byBinding = new Map());
+  let names = byBinding.get(bindingName);
+  if (!names) byBinding.set(bindingName, names = new Set());
+  if (names.has(facetName)) return false;
+  names.add(facetName);
+  return true;
+}
+
+/** Abort the facets this incarnation opened for `bindingNames`, keeping their storage, so each next request starts the class registered then. */
+export function abortInnerDoFacets(ctx: DurableObjectState, bindingNames: Iterable<string>, reason: Error): void {
+  const byBinding = openedInnerDoFacets.get(ctx);
+  if (!byBinding) return;
+  for (const bindingName of bindingNames) {
+    for (const facetName of byBinding.get(bindingName) ?? []) {
+      try { ctx.facets.abort(facetName, reason); } catch { /* already gone */ }
+    }
+    byBinding.delete(bindingName);
+  }
+}