@nimbus-sh/fabric 0.8.0 → 0.9.0

package/dist/workerd-facet-host.js

@@ -15,7 +15,7 @@
 import { disposeRpcResource } from '@nimbus-sh/platform/rpc-dispose.js';
 import { StorageLedger, forgetFacetStorage } from '@nimbus-sh/core/runtime/storage-ledger.js';
 import { getCtxExports, stagedBootAssembler, supervisorEntrypoint, supervisorEntrypointName, } from './composition.js';
-import { assertModuleMapWithinCodeLimit, beginLoaderFetch, facetNameCount, facetNameCountDurable, recordFacetNameMinted, recordLoaderId, withDynamicWorkerCapNamed, withFacetBudgetNamed, } from './budgets.js';
+import { assertModuleMapWithinCodeLimit, beginLoaderFetch, facetNameCount, facetNameCountDurable, recordFacetNameMinted, withDynamicWorkerCapNamed, withFacetBudgetNamed, } from './budgets.js';
 import { RESIDENT_PROCESS_CLASS, residentLoaderConfig, } from './process-fabric.js';
 import { supervisorLoaderKey } from './supervisor-props.js';
 export function getNimbusCtxExports() {
@@ -87,7 +87,7 @@ export async function cloneStorage(ctx, clone) {
     const facets = facetContainer(ctx);
     if (typeof facets.clone !== 'function') {
         throw new Error('Nimbus: ctx.facets.clone is unavailable in this runtime; the reflink image '
-            + 'path needs deployed Cloudflare workerd (local workerd <= 1.20260603.1 lacks it)');
+            + 'path needs workerd 1.20260926.1 or later, or deployed Cloudflare workerd');
     }
     const { src, dst } = clone;
     if (!(await clone.populated(src))) {
@@ -292,10 +292,13 @@ function spawnResident(ctx, env, disk, supervisor, params) {
                 + 'it is not restarted');
         }
         evaluated = true;
-        return { class: residentProcessClass(ctx, env, disk, supervisor, params) };
+        return { class: residentProcessClass(env, disk, supervisor, params, loaderKey) };
     };
     const book = slotBook(ctx);
     const ledger = sessionLedger(ctx);
+    // A warm worker keeps the SUPERVISOR binding it was built with, and the
+    // loader outlives this instance.
+    const loaderKey = supervisorLoaderKey(params.workerKey, supervisor);
     let facet;
     try {
         // N18: the fill is admitted, and recorded under the facet's name, before
@@ -315,6 +318,12 @@ function spawnResident(ctx, env, disk, supervisor, params) {
     }
     if (explicit)
         book.live.add(name);
+    // The facet's worker is one Dynamic Worker in flight for as long as the
+    // process is resident, not only while a call is open: its WebSockets and
+    // streamed responses outlive the calls the ledger could bracket, and a
+    // request can reach it at any moment. Held from here to `release`, so no
+    // fan-out spends the slot a running process needs.
+    const endResidency = beginLoaderFetch(ctx, loaderKey);
     facetOfPid(ctx).set(params.pid, name);
     let disposed = false;
     const release = async () => {
@@ -323,6 +332,7 @@ function spawnResident(ctx, env, disk, supervisor, params) {
         disposed = true;
         released = true;
         facetOfPid(ctx).delete(params.pid);
+        endResidency();
         try {
             facets.abort(name, new Error('Nimbus: resident process released'));
         }
@@ -396,25 +406,15 @@ function spawnResident(ctx, env, disk, supervisor, params) {
  * assembles its module map at most once and the bytes never stay resident in
  * the hosting DO's heap.
  */
-function residentProcessClass(ctx, env, disk, supervisor, params) {
+function residentProcessClass(env, disk, supervisor, params, loaderKey) {
     const loader = env.LOADER;
     if (!loader || typeof loader.get !== 'function') {
         throw new Error('Nimbus: env.LOADER binding missing or invalid. Resident processes require '
             + 'the Worker Loader binding; add it via worker_loaders in wrangler.jsonc.');
     }
-    // A warm worker keeps the SUPERVISOR binding it was built with, and the
-    // loader outlives this instance.
-    const loaderKey = supervisorLoaderKey(params.workerKey, supervisor);
-    try {
-        const worker = loader
-            .get(loaderKey, () => residentWorkerConfig(env, disk, supervisor, params.boot))
-            .getDurableObjectClass(RESIDENT_PROCESS_CLASS);
-        recordLoaderId(ctx, loaderKey);
-        return worker;
-    }
-    catch (error) {
-        throw withDynamicWorkerCapNamed(ctx, error);
-    }
+    return loader
+        .get(loaderKey, () => residentWorkerConfig(env, disk, supervisor, params.boot))
+        .getDurableObjectClass(RESIDENT_PROCESS_CLASS);
 }
 async function runOneShot(ctx, env, supervisor, params, consume) {
     const loader = env.LOADER;
@@ -455,24 +455,24 @@ async function runOneShot(ctx, env, supervisor, params, consume) {
             throw new Error('Nimbus: one-shot runtime entrypoint has no fetch method');
         }
         params.onLoaded?.();
-        // The unkeyed worker is a live dynamic worker for exactly this call, so
-        // the run is a Loader fetch on the hosting actor's ledger — bracketed,
-        // never wrapped: see beginLoaderFetch for the measured DO-poisoning
-        // hazard, and the pipelined-`fetch.call` note above for its sibling.
-        const endFetch = beginLoaderFetch(ctx);
-        let response;
+        // The unkeyed worker is one distinct dynamic worker in flight until its
+        // response is consumed (the body streams from it), keyed by this run's
+        // writer id — bracketed, never wrapped: see beginLoaderFetch for the
+        // measured DO-poisoning hazard, and the pipelined-`fetch.call` note above
+        // for its sibling.
+        const endFetch = beginLoaderFetch(ctx, `one-shot:${params.writerId}`);
         try {
-            response = await ep.fetch(params.request);
+            const response = await ep.fetch(params.request);
+            try {
+                return await consume(response);
+            }
+            finally {
+                disposeRpcResource(response);
+            }
         }
         finally {
             endFetch();
         }
-        try {
-            return await consume(response);
-        }
-        finally {
-            disposeRpcResource(response);
-        }
     }
     catch (error) {
         throw withDynamicWorkerCapNamed(ctx, error);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nimbus-sh/fabric",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "description": "The Cloudflare half of Nimbus \u2014 Durable Object facet hosting, dynamic-worker loader pools, the resident-process fabric, and DO alarm/hibernation machinery.",
   "keywords": [
     "cloudflare",
@@ -50,12 +50,12 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@nimbus-sh/core": "^0.13.0",
-    "@nimbus-sh/platform": "^0.6.0",
+    "@nimbus-sh/core": "^0.14.0",
+    "@nimbus-sh/platform": "^0.7.0",
     "zod": "^4.4.3"
   },
   "devDependencies": {
-    "@cloudflare/workers-types": "^4.20250327.0",
+    "@cloudflare/workers-types": "^5.20260928.1",
     "typescript": "^5.7.0"
   },
   "publishConfig": {

package/src/budgets.ts

@@ -1,39 +1,46 @@
 /**
  * budgets.ts — per-DO accounting for the platform budgets the fabric spends:
- * the Worker Loader's two caps, the facet-ID lifetime budget, and the
- * dynamic-worker module-map ceiling.
+ * the Durable Object's Dynamic Worker concurrency limit, the facet-ID
+ * lifetime budget, and the dynamic-worker module-map ceiling.
  *
- * Measured on production workerd: a Durable Object admits ~5–6 concurrent
- * dynamic workers before the platform refuses with "Too many concurrent
- * dynamic workers", one DO method can drive at most 4 concurrent Loader
- * fetches, and loader-cache entries are never released — every DISTINCT
- * `loader.get(id)` permanently consumes one of the dynamic-worker slots for
- * the object's lifetime. Nimbus stays under the caps by construction
- * (`IN_DO_THRESHOLD` = 5 in the fanout pool), which until now meant the slots
- * were counted in prose. This ledger counts them at the fabric's loader call
- * sites instead — the loader pool's slots, a resident process's keyed worker,
- * a one-shot's load — so proximity is measurable and a cap failure can name
- * the ids actually holding slots.
+ * The Dynamic Worker model is Cloudflare's documented one
+ * ({@link DO_DYNAMIC_WORKER_LIMIT}): a Durable Object may have a fixed number
+ * of DISTINCT Dynamic Workers with in-flight requests at once, shared across
+ * every concurrent request to that object (one I/O context), and any number
+ * of in-flight requests to the same Dynamic Worker count as one. Only
+ * in-flight requests count: a loader id with nothing in flight holds nothing.
  *
- * Measurement only: no admission control. The caps are the platform's, they
- * are approximate ("~5–6"), and a gate on an approximate number would refuse
- * work the platform would have run.
+ * The ledger counts, per hosting actor, the distinct workers that are in
+ * flight right now, keyed by loader id (a fresh key per unkeyed `load`), plus
+ * the width fan-outs have claimed and not yet released. A fan-out spends only
+ * the {@link dynamicWorkerHeadroom} that leaves, so work a Durable Object
+ * already has in flight — a resident process, the esbuild facet, a git
+ * network op, another fan-out — keeps its slots.
  *
  * Keyed weakly off the hosting actor's `ctx`, like the facet slot books: the
- * caps are per Durable Object, and dynamic workers die with the isolate that
+ * limit is per Durable Object, and dynamic workers die with the isolate that
  * loaded them, so a ledger that goes away with its host describes nothing
  * that still exists.
  */
 
 import { classifyError } from '@nimbus-sh/platform/oom-classify.js';
+import { hostWasmIdentity } from './host-wasm.js';
+
+/**
+ * Distinct Dynamic Workers one Durable Object may have with in-flight
+ * requests at once, shared across all concurrent requests to that object;
+ * multiple in-flight requests to one Dynamic Worker count once.
+ * https://developers.cloudflare.com/changelog/post/2026-08-28-durable-objects-dynamic-workers-limit/
+ */
+export const DO_DYNAMIC_WORKER_LIMIT = 10;
 
 interface LoaderLedger {
-  /** Distinct loader ids ever gotten — each one a permanently consumed slot. */
-  ids: Set<string>;
-  /** In-flight calls into dynamic workers, right now. */
-  liveFetches: number;
-  /** The most that were ever in flight at once. */
-  peakLiveFetches: number;
+  /** Loader id → open holds on it. A key is present only while held. */
+  inFlight: Map<string, number>;
+  /** Width claimed by fan-outs that have not released it. */
+  claimed: number;
+  /** The most distinct workers (holds plus claims) ever counted at once. */
+  peak: number;
 }
 
 const ledgers = new WeakMap<object, LoaderLedger>();
@@ -41,20 +48,21 @@ const ledgers = new WeakMap<object, LoaderLedger>();
 function ledger(ctx: object): LoaderLedger {
   let entry = ledgers.get(ctx);
   if (!entry) {
-    entry = { ids: new Set(), liveFetches: 0, peakLiveFetches: 0 };
+    entry = { inFlight: new Map(), claimed: 0, peak: 0 };
     ledgers.set(ctx, entry);
   }
   return entry;
 }
 
-/** Record a keyed `loader.get(id)` — a permanent slot if the id is new. */
-export function recordLoaderId(ctx: object, id: string): void {
-  ledger(ctx).ids.add(id);
+function inUse(entry: LoaderLedger): number {
+  return entry.inFlight.size + entry.claimed;
 }
 
 /**
- * Count one call into a dynamic worker as a live Loader fetch; the returned
- * function ends it (idempotently), from the caller's own `finally`.
+ * Hold the Dynamic Worker `workerKey` in flight on this actor's ledger; the
+ * returned function ends the hold (idempotently), from the caller's own
+ * `finally`. Holds on one key nest: the worker counts once until the last
+ * one ends, as the platform counts it.
  *
  * A begin/end pair rather than a wrapper on purpose, and the shape is
  * load-bearing: wrapping the stub call in a ledger-owned async frame
@@ -67,48 +75,82 @@ export function recordLoaderId(ctx: object, id: string): void {
  * workers: an RPC stub call must stay a direct property call awaited by the
  * frame that made it, so the ledger only brackets it.
  */
-export function beginLoaderFetch(ctx: object): () => void {
+export function beginLoaderFetch(ctx: object, workerKey: string): () => void {
   const entry = ledger(ctx);
-  entry.liveFetches++;
-  entry.peakLiveFetches = Math.max(entry.peakLiveFetches, entry.liveFetches);
+  entry.inFlight.set(workerKey, (entry.inFlight.get(workerKey) ?? 0) + 1);
+  entry.peak = Math.max(entry.peak, inUse(entry));
   let ended = false;
   return () => {
     if (ended) return;
     ended = true;
-    entry.liveFetches--;
+    const open = (entry.inFlight.get(workerKey) ?? 1) - 1;
+    if (open > 0) entry.inFlight.set(workerKey, open);
+    else entry.inFlight.delete(workerKey);
+  };
+}
+
+/**
+ * Distinct Dynamic Workers this actor may still put in flight: the limit
+ * less what is held and claimed right now. Never negative.
+ */
+export function dynamicWorkerHeadroom(ctx: object): number {
+  return Math.max(0, DO_DYNAMIC_WORKER_LIMIT - inUse(ledger(ctx)));
+}
+
+/**
+ * Claim `width` distinct Dynamic Workers for one fan-out, or null when the
+ * headroom cannot hold it. The claim counts until `release` (idempotent), so
+ * a second fan-out sizing itself meanwhile sees it; the claimant's own
+ * dispatches are held as well while they run, which only ever over-counts
+ * toward sending that second fan-out elsewhere.
+ */
+export function claimDynamicWorkers(ctx: object, width: number): { release(): void } | null {
+  const entry = ledger(ctx);
+  if (width < 1 || width > DO_DYNAMIC_WORKER_LIMIT - inUse(entry)) return null;
+  entry.claimed += width;
+  entry.peak = Math.max(entry.peak, inUse(entry));
+  let released = false;
+  return {
+    release() {
+      if (released) return;
+      released = true;
+      entry.claimed -= width;
+    },
   };
 }
 
 /** Snapshot for the diag surface. Pure read; no I/O. */
 export function loaderLedgerStats(ctx: object): {
-  idsEverGotten: string[];
-  liveFetches: number;
-  peakLiveFetches: number;
+  limit: number;
+  inFlightWorkers: string[];
+  claimed: number;
+  headroom: number;
+  peak: number;
 } {
   const entry = ledger(ctx);
   return {
-    idsEverGotten: [...entry.ids],
-    liveFetches: entry.liveFetches,
-    peakLiveFetches: entry.peakLiveFetches,
+    limit: DO_DYNAMIC_WORKER_LIMIT,
+    inFlightWorkers: [...entry.inFlight.keys()],
+    claimed: entry.claimed,
+    headroom: dynamicWorkerHeadroom(ctx),
+    peak: entry.peak,
   };
 }
 
 /**
- * Name the per-DO accounting on a "Too many concurrent dynamic workers"
+ * Name the per-DO accounting on a "Dynamic worker concurrency limit exceeded"
  * failure; hand every other error back untouched. The platform's message
- * says only that the cap was hit — which ids hold the slots, and that a
- * keyed id can never give one back, is what the operator needs to know to
- * shrink anything.
+ * says only that the limit was hit — which workers were in flight, and what
+ * fan-outs had claimed, is what the operator needs to know to shrink anything.
  */
 export function withDynamicWorkerCapNamed<E>(ctx: object, error: E): E | Error {
   if (classifyError(error) !== 'dynamic_worker_cap') return error;
   const entry = ledger(ctx);
   const platform = error instanceof Error ? error.message : String(error);
   return new Error(
-    `${platform} — this Durable Object has ${entry.ids.size} loader id(s) permanently `
-      + `holding dynamic-worker slots (a loader.get id is never released): `
-      + `${[...entry.ids].join(', ') || '(none recorded)'}; live Loader fetches ${entry.liveFetches}, `
-      + `peak ${entry.peakLiveFetches}`,
+    `${platform} — this Durable Object had ${entry.inFlight.size} distinct dynamic worker(s) in flight `
+      + `(${[...entry.inFlight.keys()].join(', ') || 'none recorded'}) and ${entry.claimed} claimed by fan-outs, `
+      + `against a limit of ${DO_DYNAMIC_WORKER_LIMIT}; peak ${entry.peak}`,
     { cause: error },
   );
 }
@@ -163,18 +205,23 @@ export function assertModuleMapWithinCodeLimit(modules: Record<string, unknown>)
 
 /**
  * Bytes one module-map member carries, across the loader's content kinds
- * (plain string, `{ js | cjs | py | text }`, `{ wasm | data }`). With an
- * encoder, text is measured exactly; without one, by code-unit length.
+ * (plain string, `{ js | cjs | py | text }`, `{ wasm | data }`, a bare
+ * WebAssembly.Module). With an encoder, text is measured exactly; without
+ * one, by code-unit length. A compiled module counts the wire size its host
+ * described (host-wasm.ts); one nobody described counts nothing here and is
+ * left to the platform's own refusal, as the text undercount is.
  */
 function memberBytes(content: unknown, encoder: TextEncoder | null): number {
   const textBytes = (text: string): number =>
     encoder ? encoder.encode(text).byteLength : text.length;
   if (typeof content === 'string') return textBytes(content);
+  if (content instanceof WebAssembly.Module) return hostWasmIdentity(content)?.bytes ?? 0;
   if (content !== null && typeof content === 'object') {
     for (const value of Object.values(content)) {
       if (typeof value === 'string') return textBytes(value);
       if (value instanceof ArrayBuffer) return value.byteLength;
       if (ArrayBuffer.isView(value)) return value.byteLength;
+      if (value instanceof WebAssembly.Module) return hostWasmIdentity(value)?.bytes ?? 0;
     }
   }
   return 0;

package/src/do-calls.ts

@@ -42,6 +42,7 @@
 
 import { classifyDoCall, isRetryableDoCall, type DoCallClass } from '@nimbus-sh/platform/oom-classify.js';
 import { disposeRpcResource } from '@nimbus-sh/platform/rpc-dispose.js';
+import { untraced, type SpanRecorder } from '@nimbus-sh/platform/tracing.js';
 
 /** Total attempts. Two retries is what a dropped connection or a deploy
  *  bounce needs; beyond that the object is not coming back inside this
@@ -88,8 +89,28 @@ export interface DoCallRetryPolicy {
    * its error.
    */
   onRetry?(info: DoCallRetryInfo): void;
+  /**
+   * Where the call's telemetry goes: its span's recorder. Each attempt lost
+   * to a transient or overloaded failure is recorded as an exception whose
+   * `code` is the failure's class, and once the call has settled it gets
+   * `do_call.attempts` (started), `do_call.hedges` (started by a hedge),
+   * `do_call.answered_by` (the attempt whose answer the call took, absent
+   * when none answered) and `do_call.outcome` ({@link DoCallOutcome}).
+   * Nothing recorded can change the call's answer. Records nothing when
+   * absent.
+   */
+  span?: SpanRecorder;
 }
 
+/**
+ * How an `idempotent` call ended: `answered` (an attempt succeeded),
+ * `callee_error` (the callee's own failure, which is an answer),
+ * `exhausted` (the last transient failure, no repeat left), `overloaded`
+ * (shed, nothing repeated after it), or `caller_error` (the resolver or
+ * `onRetry` threw).
+ */
+export type DoCallOutcome = 'answered' | 'callee_error' | 'exhausted' | 'overloaded' | 'caller_error';
+
 /** What one retry is answering: which call, which platform class, which
  *  attempt just failed out of how many. */
 export interface DoCallRetryInfo {
@@ -161,7 +182,7 @@ export function idempotent<S, T>(
 ): Promise<T> {
   const maxAttempts = policy.maxAttempts ?? MAX_ATTEMPTS;
   const baseDelayMs = policy.baseDelayMs ?? BASE_DELAY_MS;
-  const { hedgeAfterMs, retryWindowMs } = policy;
+  const { hedgeAfterMs, retryWindowMs, span = untraced } = policy;
   const startedAt = Date.now();
   // The executor form: fabric's library target predates Promise.withResolvers.
   return new Promise<T>((resolve, reject) => {
@@ -173,13 +194,21 @@ export function idempotent<S, T>(
     // An attempt was shed as overloaded: nothing is repeated after it.
     let refused = false;
     let settled = false;
+    // Attempts a hedge started rather than a retry or the first send.
+    let hedged = 0;
 
-    const settle = (answer: () => void): void => {
+    const settle = (outcome: DoCallOutcome, answeredBy: number | undefined, answer: () => void): void => {
       if (settled) return;
       settled = true;
       for (const timer of hedges) clearTimeout(timer);
       hedges.clear();
       answer();
+      span.set({
+        'do_call.attempts': started,
+        'do_call.hedges': hedged,
+        'do_call.answered_by': answeredBy,
+        'do_call.outcome': outcome,
+      });
     };
     /** May another attempt start at `at`? */
     const canRepeat = (at: number): boolean =>
@@ -187,13 +216,20 @@ export function idempotent<S, T>(
       && (retryWindowMs === undefined || at - startedAt <= retryWindowMs);
     /** `error` ended an attempt that will not be repeated: the call's answer, once nothing else is live. */
     const exhausted = <E>(error: E): void => {
-      if (live === 0) settle(() => reject(error));
+      if (live === 0) settle(refused ? 'overloaded' : 'exhausted', undefined, () => reject(error));
     };
 
     /** A failed attempt, numbered: repeat it after its backoff, or let it stand. */
     const failed = async <E>(number: number, error: E): Promise<void> => {
       if (settled) return;
       const classification = classifyDoCall(error);
+      if (!isRetryableDoCall(classification) && classification !== 'overloaded') {
+        // The call ran and its answer is this error — ENOENT is a read's answer as much as bytes are.
+        settle('callee_error', number, () => reject(error));
+        return;
+      }
+      // A lost attempt: the call's span says which, and why.
+      span.exception(error, classification, `attempt ${number} of ${maxAttempts}: `);
       if (classification === 'overloaded') {
         // A shed call is no answer: nothing more is sent, and an attempt
         // still in flight may yet answer.
@@ -201,11 +237,6 @@ export function idempotent<S, T>(
         exhausted(error);
         return;
       }
-      if (!isRetryableDoCall(classification)) {
-        // The call ran and its answer is this error — ENOENT is a read's answer as much as bytes are.
-        settle(() => reject(error));
-        return;
-      }
       const delayMs = Math.floor(Math.random() * 2 ** number * baseDelayMs);
       if (!canRepeat(Date.now() + delayMs)) {
         exhausted(error);
@@ -236,7 +267,10 @@ export function idempotent<S, T>(
       }
       const hedge = hedgeAfterMs === undefined ? undefined : setTimeout(() => {
         if (hedge !== undefined) hedges.delete(hedge);
-        if (canRepeat(Date.now())) attempt();
+        if (canRepeat(Date.now())) {
+          hedged++;
+          attempt();
+        }
       }, hedgeAfterMs);
       if (hedge !== undefined) hedges.add(hedge);
       /** This attempt has its answer: it hedges no more, and its stub goes. */
@@ -263,12 +297,12 @@ export function idempotent<S, T>(
         disposeRpcResource(result);
         return;
       }
-      settle(() => resolve(result));
+      settle('answered', number, () => resolve(result));
     };
 
     /** Start an attempt. Whatever it throws outside the call itself fails the call. */
     const attempt = (): void => {
-      run().catch((error) => settle(() => reject(error)));
+      run().catch((error) => settle('caller_error', undefined, () => reject(error)));
     };
 
     attempt();