@nimbus-sh/fabric 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/fanout.ts CHANGED
@@ -1,22 +1,27 @@
1
1
  /**
2
2
  * Two-tier fan-out primitive for work that must execute in Worker Loader
3
- * facets without tripping workerd's per-DO dynamic-worker ceiling.
3
+ * isolates within the Durable Object's Dynamic Worker budget.
4
4
  *
5
- * A single Durable Object method can drive at most four concurrent
6
- * Worker Loader fetches before extra dispatches serialize or fail. Small
7
- * batches therefore run in the coordinator DO through IsolatePool.
8
- * Wider batches are sharded across sibling NimbusSession DOs, each of
9
- * which owns its own four-loader budget.
5
+ * A Durable Object may have `DO_DYNAMIC_WORKER_LIMIT` distinct Dynamic
6
+ * Workers with in-flight requests, shared across every concurrent request to
7
+ * it (budgets.ts). A batch the coordinator's remaining headroom can hold —
8
+ * the limit less the workers it already has in flight (resident processes,
9
+ * the esbuild facet, a git network op) and other fan-outs' claims — runs in
10
+ * the coordinator through IsolatePool, one Dynamic Worker per task. Only a
11
+ * batch wider than that headroom is sharded across sibling NimbusSession
12
+ * DOs, each of which spends its own budget.
10
13
  *
11
- * Routing is deterministic: each task has a stable key, and the key maps
12
- * to a sibling DO shard. There is no silent fallback to width-1 execution;
13
- * missing LOADER or NIMBUS_SESSION bindings fail loudly so install and
14
- * runtime operations do not appear successful after partial dispatch.
14
+ * Routing is deterministic for a given headroom: each task has a stable key,
15
+ * and the key maps to a sibling DO shard. There is no silent fallback to
16
+ * width-1 execution; missing LOADER or NIMBUS_SESSION bindings fail loudly so
17
+ * install and runtime operations do not appear successful after partial
18
+ * dispatch.
15
19
  */
16
20
 
17
21
  import { serializeFunction } from './vendor/serialize.js';
18
22
  import { BindingError } from './vendor/errors.js';
19
23
  import { IsolatePool, type FacetTaskFn } from './isolate-pool.js';
24
+ import { claimDynamicWorkers, dynamicWorkerHeadroom } from './budgets.js';
20
25
  import { hostRoute } from './composition.js';
21
26
  import { disposeRpcResource } from '@nimbus-sh/platform/rpc-dispose.js';
22
27
  import { classifyDoCall, describeError, isRetryableDoCall } from '@nimbus-sh/platform/oom-classify.js';
@@ -35,14 +40,17 @@ export interface FanoutEnv {
35
40
  NIMBUS_SESSION?: unknown;
36
41
  }
37
42
 
38
- /**
39
- * Threshold at which routing switches from coordinator-local loaders to
40
- * sibling Durable Objects.
41
- *
42
- * Set to **5** so the in-DO path stays below the V8 4-loaders-per-method
43
- * cap by construction. width < 5 stays local; width >= 5 uses sibling DOs.
44
- */
45
- export const IN_DO_THRESHOLD = 5;
43
+ /** Where one `submitMany` ran. */
44
+ export type FanoutTopology = 'in-do' | 'peer-do';
45
+
46
+ /** The routing decision for one `submitMany`, as its caller may log it. */
47
+ export interface FanoutRoute {
48
+ topology: FanoutTopology;
49
+ /** Tasks in the batch — the Dynamic Workers an in-DO run spends. */
50
+ tasks: number;
51
+ /** The coordinator's Dynamic Worker headroom when the batch was routed. */
52
+ headroom: number;
53
+ }
46
54
 
47
55
  /**
48
56
  * Hard cap on concurrent peer DOs per single submitMany call. Throughput stays
@@ -153,12 +161,14 @@ export interface FanoutOptions {
153
161
  * Not called on the in-DO path, which has no phases.
154
162
  */
155
163
  onDispatchPhase?: (width: number, elapsedMs: number) => void;
164
+ /** Called once per non-empty submitMany with the route it took. */
165
+ onRoute?: (route: FanoutRoute) => void;
156
166
  /**
157
167
  * Cap on peer DOs this pool will spread one submitMany across. Defaults to
158
168
  * MAX_PEER_FANOUT. Tasks beyond the cap bucket into the peers that exist and
159
169
  * run through their in-peer pool, so lowering it trades peers for barriers
160
- * without lowering total concurrency: each peer runs its bucket at
161
- * concurrency 4, so N peers still resolve 4N tasks at once.
170
+ * without lowering total concurrency: each peer runs its bucket as wide as
171
+ * its own Dynamic Worker headroom allows.
162
172
  *
163
173
  * A caller sets this when its per-task work is small enough that a peer per
164
174
  * task buys nothing but round-trips — one task per peer costs ⌈tasks/
@@ -178,7 +188,7 @@ function isPeerResult<R>(value: Awaited<ReturnType<HostOpDispatch>>): value is F
178
188
 
179
189
  /**
180
190
  * Two-tier fan-out pool. Constructed by the supervisor DO; routes
181
- * each `submitMany` call automatically based on width.
191
+ * each `submitMany` call on the coordinator's live Dynamic Worker headroom.
182
192
  *
183
193
  * Lifetime: cheap to construct (no async init). Multiple submitMany
184
194
  * calls share NO state — each is dispatched fresh. The class
@@ -216,16 +226,16 @@ export class Fanout {
216
226
  * Dispatch `tasks` across the appropriate topology and return
217
227
  * results in input order.
218
228
  *
219
- * Routing:
220
- * tasks.length < 5 -> coordinator-local IsolatePool
221
- * tasks.length >= 5 -> sibling NimbusSession DOs
229
+ * Routing, against the coordinator's Dynamic Worker headroom at call time:
230
+ * tasks.length <= headroom -> coordinator-local IsolatePool, one Dynamic
231
+ * Worker per task, the width claimed on the
232
+ * ledger until the batch settles
233
+ * tasks.length > headroom -> sibling NimbusSession DOs
222
234
  *
223
- * Backpressure: if `tasks.length > MAX_PEER_FANOUT (32)`, tasks
224
- * are sharded modulo `MAX_PEER_FANOUT` and each shard's bucket
225
- * runs serially inside its assigned peer DO via the in-peer
226
- * IsolatePool's concurrency (capped at 4 there too). A
227
- * single submitMany call returns when ALL tasks complete (or any
228
- * throws).
235
+ * Peer shards: tasks hash onto min(tasks, maxPeers ?? MAX_PEER_FANOUT)
236
+ * peers and each peer runs its bucket through its own IsolatePool, as
237
+ * wide as that peer's headroom allows. A single submitMany call returns
238
+ * when ALL tasks complete (or any throws).
229
239
  *
230
240
  * `fn` is the user function executed per task. It runs INSIDE a
231
241
  * Worker Loader isolate (in the in-DO path) or inside a peer DO's
@@ -239,16 +249,21 @@ export class Fanout {
239
249
  ): Promise<R[]> {
240
250
  if (tasks.length === 0) return [];
241
251
 
242
- if (tasks.length < IN_DO_THRESHOLD) {
243
- return this._dispatchInDo<A, R>(tasks, fn);
252
+ const headroom = dynamicWorkerHeadroom(this.ctx);
253
+ const claim = claimDynamicWorkers(this.ctx, tasks.length);
254
+ this.opts.onRoute?.({ topology: claim ? 'in-do' : 'peer-do', tasks: tasks.length, headroom });
255
+ if (!claim) return this._dispatchPeerDo<A, R>(tasks, fn);
256
+ try {
257
+ return await this._dispatchInDo<A, R>(tasks, fn);
258
+ } finally {
259
+ claim.release();
244
260
  }
245
- return this._dispatchPeerDo<A, R>(tasks, fn);
246
261
  }
247
262
 
248
- /** Report which topology a task count uses without dispatching. */
249
- topologyFor(taskCount: number): 'in-do' | 'peer-do' | 'empty' {
263
+ /** The topology a task count would take against the headroom right now. */
264
+ topologyFor(taskCount: number): FanoutTopology | 'empty' {
250
265
  if (taskCount === 0) return 'empty';
251
- return taskCount < IN_DO_THRESHOLD ? 'in-do' : 'peer-do';
266
+ return taskCount <= dynamicWorkerHeadroom(this.ctx) ? 'in-do' : 'peer-do';
252
267
  }
253
268
 
254
269
  /**
@@ -269,13 +284,10 @@ export class Fanout {
269
284
  tasks: FanoutTask<A>[],
270
285
  fn: FacetTaskFn<A, R>,
271
286
  ): Promise<R[]> {
272
- // Use the existing IsolatePool. Concurrency = task count
273
- // (capped at 4 by constructor — tasks.length is already < 5
274
- // here, so the cap won't bite). Each task = one pool.submit;
275
- // pool.map runs them with stable-slot reuse.
276
- const concurrency = Math.min(tasks.length, IN_DO_THRESHOLD - 1);
287
+ // One slot — one Dynamic Worker — per task; submitMany has claimed
288
+ // that width on the ledger.
277
289
  const pool = new IsolatePool(this.env, this.ctx, {
278
- concurrency,
290
+ concurrency: tasks.length,
279
291
  timeoutMs: this.opts.timeoutMs,
280
292
  tag: this.opts.tag,
281
293
  preamble: this.opts.preamble,
@@ -285,9 +297,6 @@ export class Fanout {
285
297
  supervisorPid: this.opts.supervisorPid,
286
298
  });
287
299
  try {
288
- // pool.map runs the function over `items` with concurrency-bounded
289
- // slot reuse. Each slot is one warm loader isolate; we get exactly
290
- // `concurrency` loader isolates total — well under the 4-cap.
291
300
  const items = tasks.map((t) => t.args);
292
301
  const results = await pool.map<A, R>(fn, items);
293
302
  // pool.map returns Array<R | null> (null on per-item failure with
@@ -316,10 +325,9 @@ export class Fanout {
316
325
  // identical fns.
317
326
  const fnSource = serializeFunction(fn);
318
327
 
319
- // Cap peer count at MAX_PEER_FANOUT. Tasks beyond N=32 are
320
- // bucketed into existing shards — each shard's peer DO then
321
- // runs its bucket through its in-DO IsolatePool.map
322
- // (concurrency capped at 4 there).
328
+ // Cap peer count at maxPeers (default MAX_PEER_FANOUT). Tasks beyond
329
+ // it bucket into existing shards; each peer runs its bucket through
330
+ // its own IsolatePool, as wide as its own headroom allows.
323
331
  const peerCount = Math.min(tasks.length, this.opts.maxPeers ?? MAX_PEER_FANOUT);
324
332
  // Group tasks by deterministic shard. Same key → same shard, so
325
333
  // tests can predict which peer handles which task.
@@ -360,10 +368,9 @@ export class Fanout {
360
368
  const peerStub = ns.get(id);
361
369
  try {
362
370
  const dispatch = hostOpDispatch(peerStub, `Fanout peer ${siblingName}`);
363
- // Each peer DO RPC call uses ONE LOADER worker on its side.
364
- // Supervisor → peer DO is a stub.fetch / RPC method call,
365
- // NOT an env.LOADER.get(); that's the cap-sidestep that
366
- // makes peer-DO fanout work.
371
+ // A peer DO call is a Durable Object RPC, not a Dynamic Worker:
372
+ // it spends none of this coordinator's budget, and the peer
373
+ // spends its own.
367
374
  const rpcResp = await dispatch({
368
375
  op: 'fanoutExecute',
369
376
  args: [
@@ -0,0 +1,41 @@
1
+ /**
2
+ * host-wasm.ts — what the fabric knows about a WebAssembly.Module a host
3
+ * hands a dynamic worker already compiled.
4
+ *
5
+ * workerd accepts a compiled module in a Worker Loader module map and lets
6
+ * the dynamic worker share its compiled code (src/workerd/api/
7
+ * worker-loader.c++, extractWasmModuleContent), so a host that bundles a
8
+ * fixed wasm (esbuild's, say) hands that module over instead of fetching a
9
+ * second copy of the bytes and having the guest compile them again. Two
10
+ * things the fabric needs about a member are unreadable from JS for a
11
+ * Module:
12
+ * - its size: the module's wire bytes still count toward the 64 MiB
13
+ * dynamic-worker code limit (worker-loader.c++ sums every member), and
14
+ * assertModuleMapWithinCodeLimit must count them;
15
+ * - its identity: a pool folds its wasm into the loader cache key, and a
16
+ * Module has no bytes to fingerprint.
17
+ * The host that owns the module states both once, here.
18
+ */
19
+
20
+ export interface HostWasmIdentity {
21
+ /** A stable name for the module's content, e.g. `esbuild@0.24.2`. */
22
+ readonly id: string;
23
+ /** The module's wire bytes: what it costs the dynamic-worker code budget. */
24
+ readonly bytes: number;
25
+ }
26
+
27
+ const described = new WeakMap<WebAssembly.Module, HostWasmIdentity>();
28
+
29
+ /** Record what `module` is, and return it. */
30
+ export function describeHostWasm(module: WebAssembly.Module, identity: HostWasmIdentity): WebAssembly.Module {
31
+ if (!Number.isSafeInteger(identity.bytes) || identity.bytes <= 0) {
32
+ throw new RangeError(`describeHostWasm: '${identity.id}' needs its wire size in bytes, got ${identity.bytes}`);
33
+ }
34
+ described.set(module, identity);
35
+ return module;
36
+ }
37
+
38
+ /** What a host said `module` is, or undefined for a module nobody described. */
39
+ export function hostWasmIdentity(module: WebAssembly.Module): HostWasmIdentity | undefined {
40
+ return described.get(module);
41
+ }
@@ -122,7 +122,7 @@ export class ImageStore {
122
122
  */
123
123
  async materialize(
124
124
  pid: number,
125
- images: AsyncIterable<readonly [string, string]> | Iterable<readonly [string, string]>,
125
+ images: AsyncIterable<readonly [string, string | readonly string[]]> | Iterable<readonly [string, string | readonly string[]]>,
126
126
  pacer: TurnBudget,
127
127
  ): Promise<Record<string, string>> {
128
128
  const fs = this.blobs();
@@ -138,7 +138,7 @@ export class ImageStore {
138
138
  fs.mkdirp(FACET_IMAGE_DIR);
139
139
  let count = 0;
140
140
  for await (const [moduleName, source] of images) {
141
- const bytes = new TextEncoder().encode(source);
141
+ const bytes = typeof source === 'string' ? new TextEncoder().encode(source) : encodeParts(source);
142
142
  const path = facetImagePath(await facetImageDigest(bytes));
143
143
  paths[moduleName] = path;
144
144
  rooted.push(path);
@@ -207,3 +207,30 @@ export class ImageStore {
207
207
  }
208
208
  }
209
209
  }
210
+
211
+ /**
212
+ * UTF-8 bytes of an image given as ordered parts, encoded into one buffer a
213
+ * part at a time: joining the parts first would hold the whole image twice as
214
+ * text.
215
+ */
216
+ function encodeParts(parts: readonly string[]): Uint8Array {
217
+ let length = 0;
218
+ for (const part of parts) {
219
+ for (let i = 0; i < part.length; i++) {
220
+ const code = part.charCodeAt(i);
221
+ if (code < 0x80) length += 1;
222
+ else if (code < 0x800) length += 2;
223
+ else if (code >= 0xd800 && code <= 0xdbff && (part.charCodeAt(i + 1) & 0xfc00) === 0xdc00) { length += 4; i++; }
224
+ else length += 3;
225
+ }
226
+ }
227
+ const bytes = new Uint8Array(length);
228
+ const encoder = new TextEncoder();
229
+ let offset = 0;
230
+ for (const part of parts) {
231
+ const encoded = encoder.encode(part);
232
+ bytes.set(encoded, offset);
233
+ offset += encoded.byteLength;
234
+ }
235
+ return bytes;
236
+ }
@@ -7,9 +7,9 @@
7
7
  * running 67 npm tarball extractions (cold-start dominates). We pin
8
8
  * each job to `slot = cursor % concurrency` and use stable loader
9
9
  * IDs `nfp:${fnHash}:slot-${i}:g${generation}`, so a pool of
10
- * concurrency=4 keeps at most 4 warm isolates rather than N fresh ones.
10
+ * concurrency=N keeps at most N warm isolates rather than one per job.
11
11
  * 2. **Nimbus defaults**: compatibilityDate = CF_COMPAT_DATE (matches
12
- * the supervisor worker), compatibilityFlags = ['nodejs_compat'],
12
+ * the supervisor worker), compatibilityFlags = GUEST_COMPAT_FLAGS,
13
13
  * globalOutbound = undefined (inherit parent network so the facet can
14
14
  * reach https://registry.npmjs.org without a proxy binding).
15
15
  * 3. **Supervisor autoinjection**. The pool grabs the embedder's
@@ -24,12 +24,12 @@
24
24
  * and binding types used by this implementation.
25
25
  */
26
26
 
27
- import { CF_COMPAT_DATE } from '@nimbus-sh/core/constants.js';
27
+ import { CF_COMPAT_DATE, GUEST_COMPAT_FLAGS } from '@nimbus-sh/core/constants.js';
28
28
  import { supervisorEntrypoint, type HostRoute } from './composition.js';
29
29
  import { supervisorBindingProps, supervisorLoaderKey } from './supervisor-props.js';
30
30
  import { disposeRpcResource } from '@nimbus-sh/platform/rpc-dispose.js';
31
31
  import { serializeFunction, hashSource } from './vendor/serialize.js';
32
- import { beginLoaderFetch, recordLoaderId, withDynamicWorkerCapNamed } from './budgets.js';
32
+ import { beginLoaderFetch, withDynamicWorkerCapNamed } from './budgets.js';
33
33
  import { assertModuleMapWithinCodeLimit } from './budgets.js';
34
34
  import { recordFailure, setLastFacetId, getLastRpcFrame } from '@nimbus-sh/platform/oom-discriminator.js';
35
35
  import { classifyError } from '@nimbus-sh/platform/oom-classify.js';
@@ -41,6 +41,7 @@ import {
41
41
  } from './vendor/errors.js';
42
42
  import type { FacetBindings } from '@nimbus-sh/core/runtime/facet-host.js';
43
43
  import type { ModuleContent, WorkerLoader } from './vendor/types.js';
44
+ import { hostWasmIdentity } from './host-wasm.js';
44
45
 
45
46
  /**
46
47
  * A function dispatched into a facet isolate, with the bindings that facet was
@@ -62,7 +63,11 @@ export interface IsolatePoolEnv {
62
63
 
63
64
  /** Options handed to IsolatePool's constructor. */
64
65
  export interface IsolatePoolOptions {
65
- /** Maximum concurrent in-flight facets. Default 4. */
66
+ /**
67
+ * Maximum concurrent in-flight facets, each a distinct Dynamic Worker
68
+ * spent from the hosting DO's `DO_DYNAMIC_WORKER_LIMIT`. Default 1; a
69
+ * caller that wants more sizes it against that budget (Fanout does).
70
+ */
66
71
  concurrency?: number;
67
72
  /** Per-task timeout in ms. Default 60_000. */
68
73
  timeoutMs?: number;
@@ -157,15 +162,19 @@ export interface IsolatePoolOptions {
157
162
  /**
158
163
  * WebAssembly modules to ship into the facet via the LOADER's
159
164
  * `modules` map. Map keys are module specifier paths (e.g.
160
- * `'esbuild.wasm'`); values are the raw bytes.
165
+ * `'esbuild.wasm'`); values are the raw bytes, or a module the host
166
+ * already holds compiled.
161
167
  *
162
- * Workerd registers each entry as `{ wasm: ArrayBuffer }` in the
163
- * worker's modules map. The pool prepends a static
168
+ * Workerd registers each entry as `{ wasm }` in the worker's modules
169
+ * map. The pool prepends a static
164
170
  * `import __NIMBUS_WASM_<id> from './<key>';` to the generated
165
- * worker.js so workerd compiles each at module-load (startup phase,
166
- * where wasm code generation is permitted). The compiled Modules
167
- * are exposed via `globalThis.__NIMBUS_WASM[<key>]` for the user
168
- * function to read at request time.
171
+ * worker.js; bytes are compiled at the facet's module-load (startup
172
+ * phase, where wasm code generation is permitted), and a compiled
173
+ * module is shared with the facet as is, its compiled code included
174
+ * (workerd src/workerd/api/worker-loader.c++,
175
+ * extractWasmModuleContent). The Modules are exposed via
176
+ * `globalThis.__NIMBUS_WASM[<key>]` for the user function to read at
177
+ * request time.
169
178
  *
170
179
  * Why this works when other paths don't:
171
180
  * - request-time `WebAssembly.compile()` — disallowed by workerd
@@ -174,15 +183,15 @@ export interface IsolatePoolOptions {
174
183
  * structured-clone refuses ("Unable to deserialize cloned data").
175
184
  * - inlining bytes in the preamble — 16 MiB string per dispatch
176
185
  * OOMs the supervisor at module-source allocation time.
177
- * - LOADER modules-map (this) — bytes ride INSIDE the worker code
178
- * blob; workerd compiles wasm during its own startup pipeline,
179
- * never crossing structured-clone, never executing JS eval.
186
+ * - LOADER modules-map (this) — the module rides INSIDE the worker
187
+ * code blob, never crossing structured-clone, never executing JS
188
+ * eval.
180
189
  *
181
- * The bytes ARE part of the loader-cache key (workerd hashes the
182
- * whole WorkerCode), so changing the wasm bytes invalidates warm
183
- * slots — desirable when the bundled wasm version changes.
190
+ * A compiled module must be described (host-wasm.ts describeHostWasm):
191
+ * its identity keys warm slots, as the bytes' fingerprint does, and its
192
+ * size counts toward the dynamic-worker code limit.
184
193
  */
185
- wasmModules?: Record<string, ArrayBuffer>;
194
+ wasmModules?: Record<string, ArrayBuffer | WebAssembly.Module>;
186
195
  }
187
196
 
188
197
  /** Per-call override (merged with pool defaults). */
@@ -240,6 +249,14 @@ interface ResolvedResilience {
240
249
  retries: number;
241
250
  }
242
251
 
252
+ /**
253
+ * How long one call waits, in all, for the platform to admit it after
254
+ * "Dynamic worker concurrency limit exceeded" (doubling from 50 ms, at most
255
+ * 2 s a wait). A deployed Durable Object admitted the refused batch after a
256
+ * 6 s pause; 15 s bounds a call that would never be admitted.
257
+ */
258
+ const CAP_REFUSAL_WAIT_MS = 15_000;
259
+
243
260
  /**
244
261
  * esbuild runtime helpers re-declared at the top of every generated facet
245
262
  * module. esbuild emits `__name(fn, "fn")` wrappers around every named
@@ -367,7 +384,7 @@ export function assembleLoaderWorkerModuleSource(
367
384
  * Typical use:
368
385
  *
369
386
  * const pool = new IsolatePool(env, ctx, {
370
- * concurrency: 4,
387
+ * concurrency: 2,
371
388
  * tag: 'npm-install',
372
389
  * });
373
390
  * const results = await pool.map(
@@ -412,13 +429,13 @@ export class IsolatePool {
412
429
  /** Identifier used inside the generated worker for both the static
413
430
  * import binding and the globalThis exposure. Sanitised from `name`. */
414
431
  id: string;
415
- bytes: ArrayBuffer;
432
+ wasm: ArrayBuffer | WebAssembly.Module;
416
433
  }>;
417
- /** Hash of (name + byte length + first/last bytes) of every wasm
418
- * module, folded into the loader cache key so changes invalidate
419
- * warm slots. Hashing the FULL bytes would be O(20+ MiB) per dispatch
420
- * and is unnecessary — wasm bytes are pinned at deploy time, the
421
- * length+endpoints are a strong-enough fingerprint. */
434
+ /** Hash of every constructor-time wasm module, folded into the loader
435
+ * cache key so changes invalidate warm slots: a compiled module by the
436
+ * identity its host described, bytes by name + length + first/last
437
+ * byte. Hashing the FULL bytes would be O(20+ MiB) per dispatch and is
438
+ * unnecessary — they are pinned at deploy time. */
422
439
  private readonly wasmHash: string;
423
440
  /**
424
441
  * Short prefix of the owning DO's id, baked into the loader.get()
@@ -457,7 +474,7 @@ export class IsolatePool {
457
474
  }
458
475
  this.loader = loader;
459
476
  this.ctx = ctx;
460
- this.concurrency = Math.max(1, opts?.concurrency ?? 4);
477
+ this.concurrency = Math.max(1, opts?.concurrency ?? 1);
461
478
  this.defaultTimeoutMs = opts?.timeoutMs ?? 60_000;
462
479
  this.defaultRetries = Math.max(0, opts?.retries ?? 0);
463
480
  this.tag = opts?.tag ?? 'facet';
@@ -475,17 +492,36 @@ export class IsolatePool {
475
492
  // (e.g. 'esbuild.wasm' and 'esbuild_wasm' both sanitise to
476
493
  // 'esbuild_wasm') are rejected loudly because the generated worker
477
494
  // would otherwise have duplicate imports. Order is preserved.
478
- const wasmEntries: Array<{ name: string; id: string; bytes: ArrayBuffer }> = [];
495
+ const wasmEntries: Array<{ name: string; id: string; wasm: ArrayBuffer | WebAssembly.Module }> = [];
496
+ const fingerprints: string[] = [];
479
497
  const seenIds = new Set<string>();
480
498
  if (opts?.wasmModules) {
481
- for (const [name, bytes] of Object.entries(opts.wasmModules)) {
482
- if (!(bytes instanceof ArrayBuffer)) {
499
+ for (const [name, wasm] of Object.entries(opts.wasmModules)) {
500
+ let fingerprint: string;
501
+ if (wasm instanceof ArrayBuffer) {
502
+ // Name + length + first/last byte: hashing 20+ MiB of wasm per
503
+ // dispatch would be wasteful, and these bytes change only with
504
+ // the deployed bundle.
505
+ const u = new Uint8Array(wasm);
506
+ const len = u.byteLength;
507
+ fingerprint = `${name}:${len}:${len > 0 ? u[0] : 0}:${len > 0 ? u[len - 1] : 0}`;
508
+ } else if (wasm instanceof WebAssembly.Module) {
509
+ const identity = hostWasmIdentity(wasm);
510
+ if (!identity) {
511
+ throw new BindingError(
512
+ `IsolatePool: wasmModules['${name}'] is a WebAssembly.Module nobody described; ` +
513
+ 'pass it through describeHostWasm (@nimbus-sh/fabric/host-wasm.js) so warm slots ' +
514
+ 'can be keyed by it and the code limit can count it.',
515
+ );
516
+ }
517
+ fingerprint = `${name}:host:${identity.id}:${identity.bytes}`;
518
+ } else {
483
519
  // Reached only when a caller broke the declared option type, so the
484
- // value is whatever it really was rather than the ArrayBuffer here.
485
- const got = (bytes as { constructor?: { name?: string } } | null | undefined)?.constructor?.name;
520
+ // value is whatever it really was rather than the union here.
521
+ const got = (wasm as { constructor?: { name?: string } } | null | undefined)?.constructor?.name;
486
522
  throw new BindingError(
487
- `IsolatePool: wasmModules['${name}'] must be ArrayBuffer ` +
488
- `(got ${got || typeof bytes}).`,
523
+ `IsolatePool: wasmModules['${name}'] must be an ArrayBuffer or a WebAssembly.Module ` +
524
+ `(got ${got || typeof wasm}).`,
489
525
  );
490
526
  }
491
527
  const id = name.replace(/[^A-Za-z0-9_]/g, '_').replace(/^[^A-Za-z_]/, '_');
@@ -496,28 +532,12 @@ export class IsolatePool {
496
532
  );
497
533
  }
498
534
  seenIds.add(id);
499
- wasmEntries.push({ name, id, bytes });
535
+ wasmEntries.push({ name, id, wasm });
536
+ fingerprints.push(fingerprint);
500
537
  }
501
538
  }
502
539
  this.wasmModules = wasmEntries;
503
- // Fingerprint: name + length + first/last byte of each module.
504
- // Hashing 20+ MiB of wasm per dispatch would be wasteful; this
505
- // fingerprint is bytes-stable for a given deployed bundle and only
506
- // changes when the wasm itself changes (deploy-time event).
507
- if (wasmEntries.length === 0) {
508
- this.wasmHash = '0';
509
- } else {
510
- const fp = wasmEntries
511
- .map((w) => {
512
- const u = new Uint8Array(w.bytes);
513
- const len = u.byteLength;
514
- const first = len > 0 ? u[0] : 0;
515
- const last = len > 0 ? u[len - 1] : 0;
516
- return `${w.name}:${len}:${first}:${last}`;
517
- })
518
- .join('|');
519
- this.wasmHash = hashSource(fp);
520
- }
540
+ this.wasmHash = fingerprints.length === 0 ? '0' : hashSource(fingerprints.join('|'));
521
541
 
522
542
  const bindings: Record<string, unknown> = { ...(opts?.extraBindings ?? {}) };
523
543
  this.supervisorKey = 's-none';
@@ -579,9 +599,9 @@ export class IsolatePool {
579
599
  */
580
600
  #materialisePerCallWasm(
581
601
  perCall: Record<string, ArrayBuffer> | undefined,
582
- ): Array<{ name: string; id: string; bytes: ArrayBuffer }> {
602
+ ): Array<{ name: string; id: string; wasm: ArrayBuffer }> {
583
603
  if (!perCall) return [];
584
- const out: Array<{ name: string; id: string; bytes: ArrayBuffer }> = [];
604
+ const out: Array<{ name: string; id: string; wasm: ArrayBuffer }> = [];
585
605
  const ctorIds = new Set(this.wasmModules.map((w) => w.id));
586
606
  const seen = new Set<string>();
587
607
  for (const [name, bytes] of Object.entries(perCall)) {
@@ -607,7 +627,7 @@ export class IsolatePool {
607
627
  );
608
628
  }
609
629
  seen.add(id);
610
- out.push({ name, id, bytes });
630
+ out.push({ name, id, wasm: bytes });
611
631
  }
612
632
  return out;
613
633
  }
@@ -640,12 +660,12 @@ export class IsolatePool {
640
660
  * hashing the wasm were marginal; the correctness cost was severe.
641
661
  */
642
662
  #fingerprintWasm(
643
- entries: Array<{ name: string; bytes: ArrayBuffer }>,
663
+ entries: Array<{ name: string; wasm: ArrayBuffer }>,
644
664
  ): string {
645
665
  if (entries.length === 0) return '0';
646
666
  const parts: string[] = [];
647
667
  for (const w of entries) {
648
- const u = new Uint8Array(w.bytes);
668
+ const u = new Uint8Array(w.wasm);
649
669
  const len = u.byteLength;
650
670
  // djb2 over the bytes. Faster than crypto.subtle.digest at small
651
671
  // sizes, deterministic, and good enough for cache-key
@@ -672,11 +692,11 @@ export class IsolatePool {
672
692
  */
673
693
  #buildCode(
674
694
  fnSource: string,
675
- perCallWasmEntries?: Array<{ name: string; id: string; bytes: ArrayBuffer }>,
695
+ perCallWasmEntries?: Array<{ name: string; id: string; wasm: ArrayBuffer }>,
676
696
  ) {
677
697
  const workerOpts = {
678
698
  compatibilityDate: CF_COMPAT_DATE,
679
- compatibilityFlags: ['nodejs_compat'],
699
+ compatibilityFlags: [...GUEST_COMPAT_FLAGS],
680
700
  // Inherit parent network so the facet can reach registry.npmjs.org.
681
701
  globalOutbound: undefined,
682
702
  env: this.bindings,
@@ -722,7 +742,7 @@ export class IsolatePool {
722
742
  // (matters only for human-readable diffs; workerd doesn't care).
723
743
  const modules: Record<string, ModuleContent> = { 'worker.js': moduleSource };
724
744
  for (const w of allWasmEntries) {
725
- modules[w.name] = { wasm: w.bytes };
745
+ modules[w.name] = { wasm: w.wasm };
726
746
  }
727
747
  assertModuleMapWithinCodeLimit(modules);
728
748
 
@@ -843,11 +863,10 @@ export class IsolatePool {
843
863
  // binding stub once the whole pool is done, which does NOT
844
864
  // invalidate any in-flight slot's entrypoint reference.
845
865
  const stub = this.loader.get(id, async () => code);
846
- recordLoaderId(this.ctx, id);
847
866
  const entrypoint = stub.getEntrypoint();
848
867
  // Direct property call, awaited by this frame — bracketed, never
849
868
  // wrapped. See beginLoaderFetch for the measured DO-poisoning hazard.
850
- const endFetch = beginLoaderFetch(this.ctx);
869
+ const endFetch = beginLoaderFetch(this.ctx, id);
851
870
  try {
852
871
  return await invoke(entrypoint, attempt);
853
872
  } catch (err) {
@@ -863,6 +882,8 @@ export class IsolatePool {
863
882
  const maxAttempts = 1 + resilience.retries;
864
883
  let lastError: Error | undefined;
865
884
  let retriedCloneRefusal = false;
885
+ let capRefusals = 0;
886
+ let capWaitedMs = 0;
866
887
  let attempt = 0;
867
888
  while (attempt < maxAttempts) {
868
889
  try {
@@ -944,6 +965,17 @@ export class IsolatePool {
944
965
  // reverse direction. This refresh targets the stale-loader case.
945
966
  continue;
946
967
  }
968
+ if (cause === 'dynamic_worker_cap' && capWaitedMs < CAP_REFUSAL_WAIT_MS) {
969
+ // The platform refused to start this call: it still counts a
970
+ // worker this Durable Object's ledger has already given back (a
971
+ // fan-out's workers stay counted for a moment after their calls
972
+ // return). Nothing ran, so the call waits, as the platform asks,
973
+ // and is sent again; it does not spend an attempt.
974
+ const delay = Math.min(CAP_REFUSAL_WAIT_MS - capWaitedMs, 50 * 2 ** capRefusals++, 2000);
975
+ capWaitedMs += delay;
976
+ await new Promise<void>((resolve) => setTimeout(resolve, delay));
977
+ continue;
978
+ }
947
979
  if (attempt < maxAttempts - 1) {
948
980
  // 100 * 2^attempt, capped at 2s so retries don't compound waiting.
949
981
  const delay = Math.min(2000, 100 * Math.pow(2, attempt));
@@ -952,9 +984,9 @@ export class IsolatePool {
952
984
  attempt++;
953
985
  }
954
986
  }
955
- // On the way out only, so retries do not stack the annotation: a cap hit
956
- // carries the ledger — which ids hold slots, and that a keyed id never
957
- // gives one back — instead of the platform's bare message.
987
+ // On the way out only, so retries do not stack the annotation: a limit
988
+ // hit carries the ledger — which workers were in flight — instead of the
989
+ // platform's bare message.
958
990
  const named = withDynamicWorkerCapNamed(this.ctx, lastError!);
959
991
  if (maxAttempts > 1) {
960
992
  throw new RetryExhaustedError(maxAttempts, named);