@nimbus-sh/fabric 0.7.1 → 0.9.0

package/dist/do-calls.js

@@ -3,12 +3,14 @@
  * the one property that decides whether a retry is safe.
  *
  * Both consumers asked for this. Proteus hand-wrote the retry
- * (`cf-backend/src/lib/do-rpc.ts`) with the rule its header states:
+ * (originally `cf-backend/src/lib/do-rpc.ts`) with the rule its header states:
  * "An operation that appends, sends, charges or mints is never wrapped: a
  * dropped call there may already have run, so a retry is a correctness bug
  * wearing resilience as a costume." agent-core has no retry machinery at all
  * and its backlog calls the gap "the most production-proven gap in the
  * corpus". Here the rule is a type: `idempotent` retries, `mutating` cannot.
+ * A mutation earns a retry only by carrying an identity its callee applies
+ * at most once; it is then `idempotent` by construction (see `mutating`).
  *
  * What the platform contract requires, and this keeps:
  *   - a FRESH stub per attempt. Cloudflare documents that many exceptions
@@ -20,6 +22,11 @@
  *     object is what overloaded it.
  *   - attempts and backoff are the consumer-proven bounds: 3 attempts total,
  *     full-jitter delays in [0, 2**attempt * 60ms).
+ *   - an `idempotent` call may also be HEDGED (`hedgeAfterMs`): an attempt
+ *     that has not answered by then is joined by the same call on a fresh
+ *     stub, both left running, the first success taken. Hedges count
+ *     against the attempts, and a callee that joins a repeat to the call it
+ *     is already serving makes one that did arrive cost nothing.
  *
  * The resolver MINTS a stub per call and the verb disposes each one it
  * minted — that ownership is what makes the fresh-stub retry real.
@@ -34,6 +41,7 @@
  */
 import { classifyDoCall, isRetryableDoCall } from '@nimbus-sh/platform/oom-classify.js';
 import { disposeRpcResource } from '@nimbus-sh/platform/rpc-dispose.js';
+import { untraced } from '@nimbus-sh/platform/tracing.js';
 /** Total attempts. Two retries is what a dropped connection or a deploy
  *  bounce needs; beyond that the object is not coming back inside this
  *  request (the consumer's measured bound). */
@@ -64,38 +72,169 @@ export class DoCallError extends Error {
 }
 /**
  * Call another Durable Object with an operation that is safe to repeat: a
- * read, or a converge-to-a-value write. Transient failures retry on a fresh
- * stub with full-jitter backoff; overloaded and permanent failures surface
- * unchanged, as does the last error at exhaustion.
+ * read, a converge-to-a-value write, or a mutation carrying an identity its
+ * callee applies at most once (see {@link mutating}). Transient failures
+ * retry on a fresh stub with full-jitter backoff; overloaded and permanent
+ * failures surface unchanged, as does the last error once no attempt may be
+ * repeated — attempts spent, or the policy's retry window closed. With
+ * `hedgeAfterMs`, an attempt still unanswered by then is joined by another
+ * on a fresh stub, and the first answer is taken: a success, or the
+ * callee's own error. A transient or overloaded failure then ends the call
+ * only once no attempt is left in flight.
+ *
+ * A failure of the resolver or of `onRetry` is the caller's own, and fails
+ * the call with it at once.
  */
-export async function idempotent(operation, stub, call, policy = {}) {
+export function idempotent(operation, stub, call, policy = {}) {
     const maxAttempts = policy.maxAttempts ?? MAX_ATTEMPTS;
     const baseDelayMs = policy.baseDelayMs ?? BASE_DELAY_MS;
-    for (let attempt = 1;; attempt++) {
-        const minted = await stub();
-        try {
-            const result = await call(minted);
-            disposeRpcResource(minted);
-            return result;
-        }
-        catch (error) {
-            // A stub that threw may be permanently broken; it is never reused.
-            disposeRpcResource(minted);
+    const { hedgeAfterMs, retryWindowMs, span = untraced } = policy;
+    const startedAt = Date.now();
+    // The executor form: fabric's library target predates Promise.withResolvers.
+    return new Promise((resolve, reject) => {
+        const hedges = new Set();
+        let started = 0;
+        // Attempts in flight, or backing off before their retry: while one is,
+        // a failure is not the call's answer.
+        let live = 0;
+        // An attempt was shed as overloaded: nothing is repeated after it.
+        let refused = false;
+        let settled = false;
+        // Attempts a hedge started rather than a retry or the first send.
+        let hedged = 0;
+        const settle = (outcome, answeredBy, answer) => {
+            if (settled)
+                return;
+            settled = true;
+            for (const timer of hedges)
+                clearTimeout(timer);
+            hedges.clear();
+            answer();
+            span.set({
+                'do_call.attempts': started,
+                'do_call.hedges': hedged,
+                'do_call.answered_by': answeredBy,
+                'do_call.outcome': outcome,
+            });
+        };
+        /** May another attempt start at `at`? */
+        const canRepeat = (at) => !settled && !refused && started < maxAttempts
+            && (retryWindowMs === undefined || at - startedAt <= retryWindowMs);
+        /** `error` ended an attempt that will not be repeated: the call's answer, once nothing else is live. */
+        const exhausted = (error) => {
+            if (live === 0)
+                settle(refused ? 'overloaded' : 'exhausted', undefined, () => reject(error));
+        };
+        /** A failed attempt, numbered: repeat it after its backoff, or let it stand. */
+        const failed = async (number, error) => {
+            if (settled)
+                return;
             const classification = classifyDoCall(error);
-            if (!isRetryableDoCall(classification) || attempt >= maxAttempts)
-                throw error;
-            policy.onRetry?.({ operation, classification, attempt, maxAttempts, error });
-            await new Promise((resolve) => {
-                setTimeout(resolve, Math.floor(Math.random() * 2 ** attempt * baseDelayMs));
+            if (!isRetryableDoCall(classification) && classification !== 'overloaded') {
+                // The call ran and its answer is this error — ENOENT is a read's answer as much as bytes are.
+                settle('callee_error', number, () => reject(error));
+                return;
+            }
+            // A lost attempt: the call's span says which, and why.
+            span.exception(error, classification, `attempt ${number} of ${maxAttempts}: `);
+            if (classification === 'overloaded') {
+                // A shed call is no answer: nothing more is sent, and an attempt
+                // still in flight may yet answer.
+                refused = true;
+                exhausted(error);
+                return;
+            }
+            const delayMs = Math.floor(Math.random() * 2 ** number * baseDelayMs);
+            if (!canRepeat(Date.now() + delayMs)) {
+                exhausted(error);
+                return;
+            }
+            live++;
+            await new Promise((wake) => {
+                setTimeout(wake, delayMs);
             });
-        }
-    }
+            live--;
+            // A hedge may have taken the last attempt, or answered, meanwhile.
+            if (!canRepeat(Date.now())) {
+                exhausted(error);
+                return;
+            }
+            policy.onRetry?.({ operation, classification, attempt: number, maxAttempts, error });
+            attempt();
+        };
+        const run = async () => {
+            const number = ++started;
+            live++;
+            const minted = await stub();
+            if (settled) {
+                live--;
+                disposeRpcResource(minted);
+                return;
+            }
+            const hedge = hedgeAfterMs === undefined ? undefined : setTimeout(() => {
+                if (hedge !== undefined)
+                    hedges.delete(hedge);
+                if (canRepeat(Date.now())) {
+                    hedged++;
+                    attempt();
+                }
+            }, hedgeAfterMs);
+            if (hedge !== undefined)
+                hedges.add(hedge);
+            /** This attempt has its answer: it hedges no more, and its stub goes. */
+            const answered = () => {
+                if (hedge !== undefined) {
+                    clearTimeout(hedge);
+                    hedges.delete(hedge);
+                }
+                live--;
+                // A stub that threw may be permanently broken; none is ever reused.
+                disposeRpcResource(minted);
+            };
+            let result;
+            try {
+                result = await call(minted);
+            }
+            catch (error) {
+                answered();
+                await failed(number, error);
+                return;
+            }
+            answered();
+            if (settled) {
+                // Another attempt answered first: this answer is dropped, so nothing of it is kept.
+                disposeRpcResource(result);
+                return;
+            }
+            settle('answered', number, () => resolve(result));
+        };
+        /** Start an attempt. Whatever it throws outside the call itself fails the call. */
+        const attempt = () => {
+            run().catch((error) => settle('caller_error', undefined, () => reject(error)));
+        };
+        attempt();
+    });
 }
 /**
  * Call another Durable Object with an operation that appends, sends, charges
  * or mints. NEVER retried — a dropped call may already have run. Failure
  * surfaces as a {@link DoCallError} carrying the classification, so the
  * caller can tell a refusal from an indeterminate drop.
+ *
+ * The rule is about the call as sent, not the operation's kind. A mutation
+ * the callee applies at most once per identity the call carries is
+ * repeatable by construction: a repeat of one that already ran is answered
+ * from the callee's record and applies nothing. Nimbus has two:
+ *   - delivered filesystem mutations (@nimbus-sh/core supervisor-delivery):
+ *     a delivery id plus the callee INSTANCE's incarnation. The record lives
+ *     in that instance's memory, and any other instance — or a callee that
+ *     predates delivery — refuses the call permanently rather than apply it
+ *     without one;
+ *   - appends: writer, module incarnation and operation sequence, recorded
+ *     durably until acknowledged.
+ * Such a call goes through {@link idempotent}, re-sending the same identity
+ * on every attempt, with a `retryWindowMs` inside the callee's retention of
+ * that record. Without such an identity, a mutation stays here.
  */
 export async function mutating(operation, stub, call) {
     const minted = await stub();

package/dist/facet-pool.d.ts

@@ -41,6 +41,7 @@
  * whichever acts second aborts or retires facets the other still tracks,
  * and the facet-id ledger here counts only the names this pool minted.
  */
+import type { SqlDatabase } from '@nimbus-sh/core/runtime/os-contracts.js';
 /** `ctx.facets`, as the pool drives it — same surface the facet host uses. */
 export interface FacetPoolContainer {
     get(name: string, start: () => Promise<{
@@ -56,6 +57,8 @@ export interface FacetPoolContext {
     storage: {
         get(key: string): Promise<unknown> | unknown;
         put(key: string, value: unknown): Promise<void>;
+        /** The session's SQL, where the storage ledger (N18) records facet databases. */
+        sql?: SqlDatabase;
     };
 }
 /**

package/dist/facet-pool.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"facet-pool.d.ts","sourceRoot":"","sources":["../src/facet-pool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AASH,8EAA8E;AAC9E,MAAM,WAAW,kBAAkB;IACjC,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;QAAE,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC;IACrE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAC5C,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED;wCACwC;AACxC,MAAM,WAAW,gBAAgB;IAC/B,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,OAAO,EAAE;QACP,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;QAC7C,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;KACjD,CAAC;CACH;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU,CAAC,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IACjB,gEAAgE;IAChE,MAAM,IAAI,IAAI,CAAC;IACf,sEAAsE;IACtE,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxC;AAKD,4EAA4E;AAC5E,wBAAgB,SAAS,CAAC,GAAG,EAAE,gBAAgB,GAAG,SAAS,CAE1D;AAED,qBAAa,SAAS;IACR,OAAO,CAAC,QAAQ,CAAC,GAAG;gBAAH,GAAG,EAAE,gBAAgB;IAElD;;;;OAIG;IACG,OAAO,CAAC,CAAC,GAAG,OAAO,EACvB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,OAAO,CAAC;QAAE,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,GACvC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;CAiD1B"}
+{"version":3,"file":"facet-pool.d.ts","sourceRoot":"","sources":["../src/facet-pool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,yCAAyC,CAAC;AAQ3E,8EAA8E;AAC9E,MAAM,WAAW,kBAAkB;IACjC,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;QAAE,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC;IACrE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAC5C,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED;wCACwC;AACxC,MAAM,WAAW,gBAAgB;IAC/B,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,OAAO,EAAE;QACP,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;QAC7C,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAChD,iFAAiF;QACjF,GAAG,CAAC,EAAE,WAAW,CAAC;KACnB,CAAC;CACH;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU,CAAC,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IACjB,gEAAgE;IAChE,MAAM,IAAI,IAAI,CAAC;IACf,sEAAsE;IACtE,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxC;AAKD,4EAA4E;AAC5E,wBAAgB,SAAS,CAAC,GAAG,EAAE,gBAAgB,GAAG,SAAS,CAE1D;AAED,qBAAa,SAAS;IACR,OAAO,CAAC,QAAQ,CAAC,GAAG;gBAAH,GAAG,EAAE,gBAAgB;IAElD;;;;OAIG;IACG,OAAO,CAAC,CAAC,GAAG,OAAO,EACvB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,OAAO,CAAC;QAAE,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,GACvC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;CAkD1B"}

package/dist/facet-pool.js

@@ -41,6 +41,7 @@
  * whichever acts second aborts or retires facets the other still tracks,
  * and the facet-id ledger here counts only the names this pool minted.
  */
+import { forgetFacetStorage } from '@nimbus-sh/core/runtime/storage-ledger.js';
 import { FACET_ID_LIFETIME_BUDGET, facetNameCount, recordFacetNameMinted, withFacetBudgetNamed, } from './budgets.js';
 /** Names this ctx's pool has already charged to the lifetime ledger. */
 const chargedNames = new WeakMap();
@@ -93,6 +94,8 @@ export class FacetPool {
                 return;
             try {
                 facets.delete(name);
+                if (this.ctx.storage.sql)
+                    forgetFacetStorage(this.ctx.storage.sql, name);
             }
             catch (e) {
                 throw new Error(`fabric: facet '${name}' was evicted but its storage was not reclaimed — `

package/dist/fanout.d.ts

@@ -1,17 +1,21 @@
 /**
  * Two-tier fan-out primitive for work that must execute in Worker Loader
- * facets without tripping workerd's per-DO dynamic-worker ceiling.
+ * isolates within the Durable Object's Dynamic Worker budget.
  *
- * A single Durable Object method can drive at most four concurrent
- * Worker Loader fetches before extra dispatches serialize or fail. Small
- * batches therefore run in the coordinator DO through IsolatePool.
- * Wider batches are sharded across sibling NimbusSession DOs, each of
- * which owns its own four-loader budget.
+ * A Durable Object may have `DO_DYNAMIC_WORKER_LIMIT` distinct Dynamic
+ * Workers with in-flight requests, shared across every concurrent request to
+ * it (budgets.ts). A batch the coordinator's remaining headroom can hold —
+ * the limit less the workers it already has in flight (resident processes,
+ * the esbuild facet, a git network op) and other fan-outs' claims — runs in
+ * the coordinator through IsolatePool, one Dynamic Worker per task. Only a
+ * batch wider than that headroom is sharded across sibling NimbusSession
+ * DOs, each of which spends its own budget.
  *
- * Routing is deterministic: each task has a stable key, and the key maps
- * to a sibling DO shard. There is no silent fallback to width-1 execution;
- * missing LOADER or NIMBUS_SESSION bindings fail loudly so install and
- * runtime operations do not appear successful after partial dispatch.
+ * Routing is deterministic for a given headroom: each task has a stable key,
+ * and the key maps to a sibling DO shard. There is no silent fallback to
+ * width-1 execution; missing LOADER or NIMBUS_SESSION bindings fail loudly so
+ * install and runtime operations do not appear successful after partial
+ * dispatch.
  */
 import { type FacetTaskFn } from './isolate-pool.js';
 import type { WorkerLoader } from './vendor/types.js';
@@ -22,14 +26,16 @@ export interface FanoutEnv {
     LOADER?: WorkerLoader;
     NIMBUS_SESSION?: unknown;
 }
-/**
- * Threshold at which routing switches from coordinator-local loaders to
- * sibling Durable Objects.
- *
- * Set to **5** so the in-DO path stays below the V8 4-loaders-per-method
- * cap by construction. width < 5 stays local; width >= 5 uses sibling DOs.
- */
-export declare const IN_DO_THRESHOLD = 5;
+/** Where one `submitMany` ran. */
+export type FanoutTopology = 'in-do' | 'peer-do';
+/** The routing decision for one `submitMany`, as its caller may log it. */
+export interface FanoutRoute {
+    topology: FanoutTopology;
+    /** Tasks in the batch — the Dynamic Workers an in-DO run spends. */
+    tasks: number;
+    /** The coordinator's Dynamic Worker headroom when the batch was routed. */
+    headroom: number;
+}
 /**
  * Hard cap on concurrent peer DOs per single submitMany call. Throughput stays
  * flat through this width while keeping per-request scheduler pressure bounded.
@@ -134,12 +140,14 @@ export interface FanoutOptions {
      * Not called on the in-DO path, which has no phases.
      */
     onDispatchPhase?: (width: number, elapsedMs: number) => void;
+    /** Called once per non-empty submitMany with the route it took. */
+    onRoute?: (route: FanoutRoute) => void;
     /**
      * Cap on peer DOs this pool will spread one submitMany across. Defaults to
      * MAX_PEER_FANOUT. Tasks beyond the cap bucket into the peers that exist and
      * run through their in-peer pool, so lowering it trades peers for barriers
-     * without lowering total concurrency: each peer runs its bucket at
-     * concurrency 4, so N peers still resolve 4N tasks at once.
+     * without lowering total concurrency: each peer runs its bucket as wide as
+     * its own Dynamic Worker headroom allows.
      *
      * A caller sets this when its per-task work is small enough that a peer per
      * task buys nothing but round-trips — one task per peer costs ⌈tasks/
@@ -149,7 +157,7 @@ export interface FanoutOptions {
 }
 /**
  * Two-tier fan-out pool. Constructed by the supervisor DO; routes
- * each `submitMany` call automatically based on width.
+ * each `submitMany` call on the coordinator's live Dynamic Worker headroom.
  *
  * Lifetime: cheap to construct (no async init). Multiple submitMany
  * calls share NO state — each is dispatched fresh. The class
@@ -167,16 +175,16 @@ export declare class Fanout {
      * Dispatch `tasks` across the appropriate topology and return
      * results in input order.
      *
-     * Routing:
-     *   tasks.length < 5   -> coordinator-local IsolatePool
-     *   tasks.length >= 5  -> sibling NimbusSession DOs
+     * Routing, against the coordinator's Dynamic Worker headroom at call time:
+     *   tasks.length <= headroom -> coordinator-local IsolatePool, one Dynamic
+     *                               Worker per task, the width claimed on the
+     *                               ledger until the batch settles
+     *   tasks.length >  headroom -> sibling NimbusSession DOs
      *
-     * Backpressure: if `tasks.length > MAX_PEER_FANOUT (32)`, tasks
-     * are sharded modulo `MAX_PEER_FANOUT` and each shard's bucket
-     * runs serially inside its assigned peer DO via the in-peer
-     * IsolatePool's concurrency (capped at 4 there too). A
-     * single submitMany call returns when ALL tasks complete (or any
-     * throws).
+     * Peer shards: tasks hash onto min(tasks, maxPeers ?? MAX_PEER_FANOUT)
+     * peers and each peer runs its bucket through its own IsolatePool, as
+     * wide as that peer's headroom allows. A single submitMany call returns
+     * when ALL tasks complete (or any throws).
      *
      * `fn` is the user function executed per task. It runs INSIDE a
      * Worker Loader isolate (in the in-DO path) or inside a peer DO's
@@ -185,8 +193,8 @@ export declare class Fanout {
      * the vendored serializeFunction (same as IsolatePool#prepare).
      */
     submitMany<A, R>(tasks: FanoutTask<A>[], fn: FacetTaskFn<A, R>): Promise<R[]>;
-    /** Report which topology a task count uses without dispatching. */
-    topologyFor(taskCount: number): 'in-do' | 'peer-do' | 'empty';
+    /** The topology a task count would take against the headroom right now. */
+    topologyFor(taskCount: number): FanoutTopology | 'empty';
     /**
      * Compute the deterministic peer-DO id for a task key and peer count.
      *

package/dist/fanout.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fanout.d.ts","sourceRoot":"","sources":["../src/fanout.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAIH,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIlE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAOtD;;+EAE+E;AAC/E,MAAM,WAAW,SAAS;IACxB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,IAAI,CAAC;AAEjC;;;GAGG;AACH,eAAO,MAAM,eAAe,KAAK,CAAC;AAElC;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,4BAA4B,IAAI,CAAC;AAC9C,eAAO,MAAM,qBAAqB,UAAmB,CAAC;AAEtD;;;;;GAKG;AACH,eAAO,MAAM,wBAAwB,UAAqB,CAAC;AAE3D;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC,uCAAuC;AACvC,MAAM,WAAW,UAAU,CAAC,CAAC;IAC3B;;;OAGG;IACH,GAAG,EAAE,MAAM,CAAC;IACZ,sCAAsC;IACtC,IAAI,EAAE,CAAC,CAAC;CACT;AAED,8CAA8C;AAC9C,MAAM,WAAW,aAAa;IAC5B;;;;;OAKG;IACH,GAAG,EAAE,MAAM,CAAC;IACZ;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAC1C;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,KAAK,IAAI,CAAC;IAC7D;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAWD;;;;;;;;GAQG;AACH,qBAAa,MAAM;IACjB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAY;IAChC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAqB;IACzC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAgB;IACrC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;gBAE5B,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,kBAAkB,EAAE,IAAI,EAAE,aAAa;IAoBzE;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,UAAU,CAAC,CAAC,EAAE,CAAC,EACnB,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,EACtB,EAAE,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,GACpB,OAAO,CAAC,CAAC,EAAE,CAAC;IASf,mEAAmE;IACnE,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,GAAG,OAAO;IAK7D;;;;;;OAMG;IACH,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM;YAOvC,aAAa;YAoCb,eAAe;CA4J9B;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAUrE"}
+{"version":3,"file":"fanout.d.ts","sourceRoot":"","sources":["../src/fanout.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAIH,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAKlE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAOtD;;+EAE+E;AAC/E,MAAM,WAAW,SAAS;IACxB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,kCAAkC;AAClC,MAAM,MAAM,cAAc,GAAG,OAAO,GAAG,SAAS,CAAC;AAEjD,2EAA2E;AAC3E,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,cAAc,CAAC;IACzB,oEAAoE;IACpE,KAAK,EAAE,MAAM,CAAC;IACd,2EAA2E;IAC3E,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,eAAO,MAAM,eAAe,KAAK,CAAC;AAElC;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,4BAA4B,IAAI,CAAC;AAC9C,eAAO,MAAM,qBAAqB,UAAmB,CAAC;AAEtD;;;;;GAKG;AACH,eAAO,MAAM,wBAAwB,UAAqB,CAAC;AAE3D;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC,uCAAuC;AACvC,MAAM,WAAW,UAAU,CAAC,CAAC;IAC3B;;;OAGG;IACH,GAAG,EAAE,MAAM,CAAC;IACZ,sCAAsC;IACtC,IAAI,EAAE,CAAC,CAAC;CACT;AAED,8CAA8C;AAC9C,MAAM,WAAW,aAAa;IAC5B;;;;;OAKG;IACH,GAAG,EAAE,MAAM,CAAC;IACZ;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAC1C;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,KAAK,IAAI,CAAC;IAC7D,mEAAmE;IACnE,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC;IACvC;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAWD;;;;;;;;GAQG;AACH,qBAAa,MAAM;IACjB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAY;IAChC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAqB;IACzC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAgB;IACrC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;gBAE5B,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,kBAAkB,EAAE,IAAI,EAAE,aAAa;IAoBzE;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,UAAU,CAAC,CAAC,EAAE,CAAC,EACnB,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,EACtB,EAAE,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,GACpB,OAAO,CAAC,CAAC,EAAE,CAAC;IAcf,2EAA2E;IAC3E,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,cAAc,GAAG,OAAO;IAKxD;;;;;;OAMG;IACH,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM;YAOvC,aAAa;YA8Bb,eAAe;CA2J9B;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAUrE"}

package/dist/fanout.js

@@ -1,33 +1,30 @@
 /**
  * Two-tier fan-out primitive for work that must execute in Worker Loader
- * facets without tripping workerd's per-DO dynamic-worker ceiling.
+ * isolates within the Durable Object's Dynamic Worker budget.
  *
- * A single Durable Object method can drive at most four concurrent
- * Worker Loader fetches before extra dispatches serialize or fail. Small
- * batches therefore run in the coordinator DO through IsolatePool.
- * Wider batches are sharded across sibling NimbusSession DOs, each of
- * which owns its own four-loader budget.
+ * A Durable Object may have `DO_DYNAMIC_WORKER_LIMIT` distinct Dynamic
+ * Workers with in-flight requests, shared across every concurrent request to
+ * it (budgets.ts). A batch the coordinator's remaining headroom can hold —
+ * the limit less the workers it already has in flight (resident processes,
+ * the esbuild facet, a git network op) and other fan-outs' claims — runs in
+ * the coordinator through IsolatePool, one Dynamic Worker per task. Only a
+ * batch wider than that headroom is sharded across sibling NimbusSession
+ * DOs, each of which spends its own budget.
  *
- * Routing is deterministic: each task has a stable key, and the key maps
- * to a sibling DO shard. There is no silent fallback to width-1 execution;
- * missing LOADER or NIMBUS_SESSION bindings fail loudly so install and
- * runtime operations do not appear successful after partial dispatch.
+ * Routing is deterministic for a given headroom: each task has a stable key,
+ * and the key maps to a sibling DO shard. There is no silent fallback to
+ * width-1 execution; missing LOADER or NIMBUS_SESSION bindings fail loudly so
+ * install and runtime operations do not appear successful after partial
+ * dispatch.
  */
 import { serializeFunction } from './vendor/serialize.js';
 import { BindingError } from './vendor/errors.js';
 import { IsolatePool } from './isolate-pool.js';
+import { claimDynamicWorkers, dynamicWorkerHeadroom } from './budgets.js';
 import { hostRoute } from './composition.js';
 import { disposeRpcResource } from '@nimbus-sh/platform/rpc-dispose.js';
-import { describeError, isDoOverloaded, isTransientDoReset } from '@nimbus-sh/platform/oom-classify.js';
+import { classifyDoCall, describeError, isRetryableDoCall } from '@nimbus-sh/platform/oom-classify.js';
 import { hostNamespaceBinding, hostOpDispatch, } from './host-dispatch.js';
-/**
- * Threshold at which routing switches from coordinator-local loaders to
- * sibling Durable Objects.
- *
- * Set to **5** so the in-DO path stays below the V8 4-loaders-per-method
- * cap by construction. width < 5 stays local; width >= 5 uses sibling DOs.
- */
-export const IN_DO_THRESHOLD = 5;
 /**
  * Hard cap on concurrent peer DOs per single submitMany call. Throughput stays
  * flat through this width while keeping per-request scheduler pressure bounded.
@@ -78,7 +75,7 @@ function isPeerResult(value) {
 }
 /**
  * Two-tier fan-out pool. Constructed by the supervisor DO; routes
- * each `submitMany` call automatically based on width.
+ * each `submitMany` call on the coordinator's live Dynamic Worker headroom.
  *
  * Lifetime: cheap to construct (no async init). Multiple submitMany
  * calls share NO state — each is dispatched fresh. The class
@@ -112,16 +109,16 @@ export class Fanout {
      * Dispatch `tasks` across the appropriate topology and return
      * results in input order.
      *
-     * Routing:
-     *   tasks.length < 5   -> coordinator-local IsolatePool
-     *   tasks.length >= 5  -> sibling NimbusSession DOs
+     * Routing, against the coordinator's Dynamic Worker headroom at call time:
+     *   tasks.length <= headroom -> coordinator-local IsolatePool, one Dynamic
+     *                               Worker per task, the width claimed on the
+     *                               ledger until the batch settles
+     *   tasks.length >  headroom -> sibling NimbusSession DOs
      *
-     * Backpressure: if `tasks.length > MAX_PEER_FANOUT (32)`, tasks
-     * are sharded modulo `MAX_PEER_FANOUT` and each shard's bucket
-     * runs serially inside its assigned peer DO via the in-peer
-     * IsolatePool's concurrency (capped at 4 there too). A
-     * single submitMany call returns when ALL tasks complete (or any
-     * throws).
+     * Peer shards: tasks hash onto min(tasks, maxPeers ?? MAX_PEER_FANOUT)
+     * peers and each peer runs its bucket through its own IsolatePool, as
+     * wide as that peer's headroom allows. A single submitMany call returns
+     * when ALL tasks complete (or any throws).
      *
      * `fn` is the user function executed per task. It runs INSIDE a
      * Worker Loader isolate (in the in-DO path) or inside a peer DO's
@@ -132,16 +129,23 @@ export class Fanout {
     async submitMany(tasks, fn) {
         if (tasks.length === 0)
             return [];
-        if (tasks.length < IN_DO_THRESHOLD) {
-            return this._dispatchInDo(tasks, fn);
+        const headroom = dynamicWorkerHeadroom(this.ctx);
+        const claim = claimDynamicWorkers(this.ctx, tasks.length);
+        this.opts.onRoute?.({ topology: claim ? 'in-do' : 'peer-do', tasks: tasks.length, headroom });
+        if (!claim)
+            return this._dispatchPeerDo(tasks, fn);
+        try {
+            return await this._dispatchInDo(tasks, fn);
+        }
+        finally {
+            claim.release();
         }
-        return this._dispatchPeerDo(tasks, fn);
     }
-    /** Report which topology a task count uses without dispatching. */
+    /** The topology a task count would take against the headroom right now. */
     topologyFor(taskCount) {
         if (taskCount === 0)
             return 'empty';
-        return taskCount < IN_DO_THRESHOLD ? 'in-do' : 'peer-do';
+        return taskCount <= dynamicWorkerHeadroom(this.ctx) ? 'in-do' : 'peer-do';
     }
     /**
      * Compute the deterministic peer-DO id for a task key and peer count.
@@ -156,13 +160,10 @@ export class Fanout {
     }
     // ── Private: in-DO dispatch (in-DO fanout) ──────────────────────────────
     async _dispatchInDo(tasks, fn) {
-        // Use the existing IsolatePool. Concurrency = task count
-        // (capped at 4 by constructor — tasks.length is already < 5
-        // here, so the cap won't bite). Each task = one pool.submit;
-        // pool.map runs them with stable-slot reuse.
-        const concurrency = Math.min(tasks.length, IN_DO_THRESHOLD - 1);
+        // One slot — one Dynamic Worker — per task; submitMany has claimed
+        // that width on the ledger.
         const pool = new IsolatePool(this.env, this.ctx, {
-            concurrency,
+            concurrency: tasks.length,
             timeoutMs: this.opts.timeoutMs,
             tag: this.opts.tag,
             preamble: this.opts.preamble,
@@ -172,9 +173,6 @@ export class Fanout {
             supervisorPid: this.opts.supervisorPid,
         });
         try {
-            // pool.map runs the function over `items` with concurrency-bounded
-            // slot reuse. Each slot is one warm loader isolate; we get exactly
-            // `concurrency` loader isolates total — well under the 4-cap.
             const items = tasks.map((t) => t.args);
             const results = await pool.map(fn, items);
             // pool.map returns Array<R | null> (null on per-item failure with
@@ -200,10 +198,9 @@ export class Fanout {
         // loader isolates (keyed on fnHash) reuse across calls with
         // identical fns.
         const fnSource = serializeFunction(fn);
-        // Cap peer count at MAX_PEER_FANOUT. Tasks beyond N=32 are
-        // bucketed into existing shards — each shard's peer DO then
-        // runs its bucket through its in-DO IsolatePool.map
-        // (concurrency capped at 4 there).
+        // Cap peer count at maxPeers (default MAX_PEER_FANOUT). Tasks beyond
+        // it bucket into existing shards; each peer runs its bucket through
+        // its own IsolatePool, as wide as its own headroom allows.
         const peerCount = Math.min(tasks.length, this.opts.maxPeers ?? MAX_PEER_FANOUT);
         // Group tasks by deterministic shard. Same key → same shard, so
         // tests can predict which peer handles which task.
@@ -242,10 +239,9 @@ export class Fanout {
                     const peerStub = ns.get(id);
                     try {
                         const dispatch = hostOpDispatch(peerStub, `Fanout peer ${siblingName}`);
-                        // Each peer DO RPC call uses ONE LOADER worker on its side.
-                        // Supervisor → peer DO is a stub.fetch / RPC method call,
-                        // NOT an env.LOADER.get(); that's the cap-sidestep that
-                        // makes peer-DO fanout work.
+                        // A peer DO call is a Durable Object RPC, not a Dynamic Worker:
+                        // it spends none of this coordinator's budget, and the peer
+                        // spends its own.
                         const rpcResp = await dispatch({
                             op: 'fanoutExecute',
                             args: [
@@ -297,8 +293,9 @@ export class Fanout {
                         return;
                     }
                     catch (err) {
-                        const schedule = isTransientDoReset(err) ? PEER_RETRY_BACKOFF_MS
-                            : isDoOverloaded(err) ? PEER_OVERLOAD_BACKOFF_MS
+                        const cls = classifyDoCall(err);
+                        const schedule = cls === 'overloaded' ? PEER_OVERLOAD_BACKOFF_MS
+                            : isRetryableDoCall(cls) ? PEER_RETRY_BACKOFF_MS
                                 : null;
                         if (schedule && attempt < PEER_TRANSIENT_RESET_RETRIES) {
                             const backoff = schedule[Math.min(attempt, schedule.length - 1)];

package/dist/fenced-work.d.ts

@@ -53,9 +53,9 @@ export declare const FENCED_WORK_MAX_ATTEMPT = 1;
  * (staging, 2026-08-13): every observed reset struck seconds AFTER the launch
  * settled — the platform kills the object while the resident runs, which is
  * when a launch-scoped row had already been deleted and recovery had nothing
- * to find. A resident's facet cannot outlive its session instance (the
- * process host's held-open leg dies with it), so a row from a previous
- * generation always names a process that is genuinely gone.
+ * to find. A resident's facet can outlive its session instance (a pending
+ * timer keeps it running), but nothing routes to it from the next one, and a
+ * spawn that takes its facet name ends it first.
  */
 export interface FencedWorkRecord {
     pid: number;

package/dist/host-wasm.d.ts

@@ -0,0 +1,29 @@
+/**
+ * host-wasm.ts — what the fabric knows about a WebAssembly.Module a host
+ * hands a dynamic worker already compiled.
+ *
+ * workerd accepts a compiled module in a Worker Loader module map and lets
+ * the dynamic worker share its compiled code (src/workerd/api/
+ * worker-loader.c++, extractWasmModuleContent), so a host that bundles a
+ * fixed wasm (esbuild's, say) hands that module over instead of fetching a
+ * second copy of the bytes and having the guest compile them again. Two
+ * things the fabric needs about a member are unreadable from JS for a
+ * Module:
+ *   - its size: the module's wire bytes still count toward the 64 MiB
+ *     dynamic-worker code limit (worker-loader.c++ sums every member), and
+ *     assertModuleMapWithinCodeLimit must count them;
+ *   - its identity: a pool folds its wasm into the loader cache key, and a
+ *     Module has no bytes to fingerprint.
+ * The host that owns the module states both once, here.
+ */
+export interface HostWasmIdentity {
+    /** A stable name for the module's content, e.g. `esbuild@0.24.2`. */
+    readonly id: string;
+    /** The module's wire bytes: what it costs the dynamic-worker code budget. */
+    readonly bytes: number;
+}
+/** Record what `module` is, and return it. */
+export declare function describeHostWasm(module: WebAssembly.Module, identity: HostWasmIdentity): WebAssembly.Module;
+/** What a host said `module` is, or undefined for a module nobody described. */
+export declare function hostWasmIdentity(module: WebAssembly.Module): HostWasmIdentity | undefined;
+//# sourceMappingURL=host-wasm.d.ts.map

package/dist/host-wasm.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"host-wasm.d.ts","sourceRoot":"","sources":["../src/host-wasm.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,MAAM,WAAW,gBAAgB;IAC/B,qEAAqE;IACrE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,6EAA6E;IAC7E,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAID,8CAA8C;AAC9C,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,QAAQ,EAAE,gBAAgB,GAAG,WAAW,CAAC,MAAM,CAM3G;AAED,gFAAgF;AAChF,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,GAAG,gBAAgB,GAAG,SAAS,CAEzF"}