@nimbus-sh/fabric 0.8.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/README.md +98 -11
  2. package/dist/bindings.d.ts +31 -33
  3. package/dist/bindings.d.ts.map +1 -1
  4. package/dist/bindings.js +108 -97
  5. package/dist/budgets.d.ts +102 -29
  6. package/dist/budgets.d.ts.map +1 -1
  7. package/dist/budgets.js +266 -44
  8. package/dist/do-calls.d.ts +20 -0
  9. package/dist/do-calls.d.ts.map +1 -1
  10. package/dist/do-calls.js +24 -11
  11. package/dist/fanout.d.ts +40 -32
  12. package/dist/fanout.d.ts.map +1 -1
  13. package/dist/fanout.js +48 -51
  14. package/dist/fenced-work.d.ts +3 -3
  15. package/dist/fenced-work.js +3 -3
  16. package/dist/host-wasm.d.ts +29 -0
  17. package/dist/host-wasm.d.ts.map +1 -0
  18. package/dist/host-wasm.js +31 -0
  19. package/dist/image-store.d.ts +1 -1
  20. package/dist/image-store.d.ts.map +1 -1
  21. package/dist/image-store.js +33 -1
  22. package/dist/inner-do-env.d.ts +83 -0
  23. package/dist/inner-do-env.d.ts.map +1 -0
  24. package/dist/inner-do-env.js +181 -0
  25. package/dist/isolate-pool.d.ts +40 -23
  26. package/dist/isolate-pool.d.ts.map +1 -1
  27. package/dist/isolate-pool.js +105 -55
  28. package/dist/process-fabric.d.ts +26 -11
  29. package/dist/process-fabric.d.ts.map +1 -1
  30. package/dist/process-fabric.js +44 -0
  31. package/dist/timers.d.ts +12 -0
  32. package/dist/timers.d.ts.map +1 -1
  33. package/dist/timers.js +44 -9
  34. package/dist/vendor/types.d.ts +11 -5
  35. package/dist/vendor/types.d.ts.map +1 -1
  36. package/dist/workerd-facet-host.d.ts.map +1 -1
  37. package/dist/workerd-facet-host.js +35 -30
  38. package/package.json +4 -4
  39. package/src/bindings.ts +121 -98
  40. package/src/budgets.ts +311 -53
  41. package/src/do-calls.ts +45 -11
  42. package/src/fanout.ts +62 -53
  43. package/src/fenced-work.ts +3 -3
  44. package/src/host-wasm.ts +41 -0
  45. package/src/image-store.ts +29 -2
  46. package/src/inner-do-env.ts +213 -0
  47. package/src/isolate-pool.ts +145 -75
  48. package/src/process-fabric.ts +55 -13
  49. package/src/timers.ts +45 -9
  50. package/src/vendor/types.ts +11 -5
  51. package/src/workerd-facet-host.ts +36 -31
package/dist/budgets.d.ts CHANGED
@@ -1,34 +1,59 @@
1
1
  /**
2
2
  * budgets.ts — per-DO accounting for the platform budgets the fabric spends:
3
- * the Worker Loader's two caps, the facet-ID lifetime budget, and the
4
- * dynamic-worker module-map ceiling.
3
+ * the Durable Object's Dynamic Worker concurrency limit, the facet-ID
4
+ * lifetime budget, and the dynamic-worker module-map ceiling.
5
5
  *
6
- * Measured on production workerd: a Durable Object admits ~5–6 concurrent
7
- * dynamic workers before the platform refuses with "Too many concurrent
8
- * dynamic workers", one DO method can drive at most 4 concurrent Loader
9
- * fetches, and loader-cache entries are never released — every DISTINCT
10
- * `loader.get(id)` permanently consumes one of the dynamic-worker slots for
11
- * the object's lifetime. Nimbus stays under the caps by construction
12
- * (`IN_DO_THRESHOLD` = 5 in the fanout pool), which until now meant the slots
13
- * were counted in prose. This ledger counts them at the fabric's loader call
14
- * sites instead — the loader pool's slots, a resident process's keyed worker,
15
- * a one-shot's load — so proximity is measurable and a cap failure can name
16
- * the ids actually holding slots.
6
+ * The Dynamic Worker model is Cloudflare's documented one
7
+ * ({@link DO_DYNAMIC_WORKER_LIMIT}): a Durable Object may have a fixed number
8
+ * of DISTINCT Dynamic Workers with in-flight requests at once, shared across
9
+ * every concurrent request to that object (one I/O context), and any number
10
+ * of in-flight requests to the same Dynamic Worker count as one. Only
11
+ * in-flight requests count: a loader id with nothing in flight holds nothing.
17
12
  *
18
- * Measurement only: no admission control. The caps are the platform's, they
19
- * are approximate ("~5–6"), and a gate on an approximate number would refuse
20
- * work the platform would have run.
13
+ * The ledger counts, per hosting actor, the distinct workers that are in
14
+ * flight right now, keyed by loader id (a fresh key per unkeyed `load`), plus
15
+ * the width fan-outs have claimed and not yet released. A fan-out spends only
16
+ * the {@link dynamicWorkerHeadroom} that leaves, so work a Durable Object
17
+ * already has in flight — a resident process, the esbuild facet, a git
18
+ * network op, another fan-out — keeps its slots. Work that would rather wait
19
+ * than be refused waits on the ledger ({@link beginLoaderFetchWhenFree}) and
20
+ * is let in, in the order it asked, by whichever release makes room.
21
21
  *
22
22
  * Keyed weakly off the hosting actor's `ctx`, like the facet slot books: the
23
- * caps are per Durable Object, and dynamic workers die with the isolate that
23
+ * limit is per Durable Object, and dynamic workers die with the isolate that
24
24
  * loaded them, so a ledger that goes away with its host describes nothing
25
25
  * that still exists.
26
26
  */
27
- /** Record a keyed `loader.get(id)` — a permanent slot if the id is new. */
28
- export declare function recordLoaderId(ctx: object, id: string): void;
29
27
  /**
30
- * Count one call into a dynamic worker as a live Loader fetch; the returned
31
- * function ends it (idempotently), from the caller's own `finally`.
28
+ * Distinct Dynamic Workers one Durable Object may have with in-flight
29
+ * requests at once, shared across all concurrent requests to that object;
30
+ * multiple in-flight requests to one Dynamic Worker count once.
31
+ * https://developers.cloudflare.com/changelog/post/2026-08-28-durable-objects-dynamic-workers-limit/
32
+ */
33
+ export declare const DO_DYNAMIC_WORKER_LIMIT = 10;
34
+ /**
35
+ * Ends one hold, idempotently. Pass the error the call failed with, if it
36
+ * did: a "Dynamic worker concurrency limit exceeded" refusal pauses the
37
+ * ledger's admissions (see {@link beginLoaderFetchWhenFree}); anything else,
38
+ * or nothing, just ends the hold.
39
+ */
40
+ export type EndLoaderFetch = (failure?: unknown) => void;
41
+ /**
42
+ * A width one fan-out reserved with {@link claimDynamicWorkers}. Holds taken
43
+ * under it (`beginLoaderFetch(ctx, key, claim)`) count inside that width, not
44
+ * on top of it, until `release` (idempotent).
45
+ */
46
+ export interface DynamicWorkerClaim {
47
+ release(): void;
48
+ }
49
+ /**
50
+ * Hold the Dynamic Worker `workerKey` in flight on this actor's ledger; the
51
+ * returned function ends the hold (idempotently), from the caller's own
52
+ * `finally`. Holds on one key nest: the worker counts once until the last
53
+ * one ends, as the platform counts it. Under a `claim`, the hold counts
54
+ * inside the claim's width. This never waits: it is for work the actor
55
+ * starts regardless (a resident process); {@link beginLoaderFetchWhenFree}
56
+ * waits for room.
32
57
  *
33
58
  * A begin/end pair rather than a wrapper on purpose, and the shape is
34
59
  * load-bearing: wrapping the stub call in a ledger-owned async frame
@@ -41,19 +66,67 @@ export declare function recordLoaderId(ctx: object, id: string): void;
41
66
  * workers: an RPC stub call must stay a direct property call awaited by the
42
67
  * frame that made it, so the ledger only brackets it.
43
68
  */
44
- export declare function beginLoaderFetch(ctx: object): () => void;
69
+ export declare function beginLoaderFetch(ctx: object, workerKey: string, claim?: DynamicWorkerClaim): EndLoaderFetch;
70
+ /**
71
+ * {@link beginLoaderFetch} once the ledger has room: resolves, holding
72
+ * `workerKey`, as soon as that worker is already in flight (holds on it
73
+ * count once) or a distinct worker more fits — within the `claim`'s width,
74
+ * or the actor's headroom. Waits are let in in the order they asked, by
75
+ * whoever's release makes the room: a hold's end, a claim's release, a
76
+ * pause's end. The hold is taken as the wait is let in, so a freed slot
77
+ * wakes one waiter and no other caller can take it first; once resolved, it
78
+ * is the caller's to end.
79
+ *
80
+ * A call refused with "Dynamic worker concurrency limit exceeded" ends its
81
+ * hold with the refusal (`end(error)`) and waits again: the refusal pauses
82
+ * admission (50 ms, doubling to 2 s while refusals continue), because the
83
+ * platform counts a worker for a moment after its call returns and no
84
+ * release can show that.
85
+ *
86
+ * `signal` abandons the wait: it rejects with the signal's reason and holds
87
+ * nothing. A wait outlives nothing on its own: bound it with a signal when
88
+ * room may never come (a resident process holds its worker for as long as
89
+ * it runs).
90
+ *
91
+ * const end = await beginLoaderFetchWhenFree(ctx, key, { signal });
92
+ * try { return await worker.getEntrypoint().run(); }
93
+ * catch (error) { end(error); throw error; }
94
+ * finally { end(); }
95
+ */
96
+ export declare function beginLoaderFetchWhenFree(ctx: object, workerKey: string, options?: {
97
+ signal?: AbortSignal;
98
+ claim?: DynamicWorkerClaim;
99
+ }): Promise<EndLoaderFetch>;
100
+ /**
101
+ * Distinct Dynamic Workers this actor may still put in flight: the limit
102
+ * less what is held and claimed right now, and none while a limit refusal's
103
+ * pause lasts. Never negative.
104
+ */
105
+ export declare function dynamicWorkerHeadroom(ctx: object): number;
106
+ /**
107
+ * Claim `width` distinct Dynamic Workers for one fan-out, or null when the
108
+ * headroom cannot hold it. The claim counts until `release` (idempotent), so
109
+ * a second fan-out sizing itself meanwhile sees it; the claimant's own
110
+ * dispatches, held under the claim, count inside it.
111
+ */
112
+ export declare function claimDynamicWorkers(ctx: object, width: number): DynamicWorkerClaim | null;
45
113
  /** Snapshot for the diag surface. Pure read; no I/O. */
46
114
  export declare function loaderLedgerStats(ctx: object): {
47
- idsEverGotten: string[];
48
- liveFetches: number;
49
- peakLiveFetches: number;
115
+ limit: number;
116
+ inFlightWorkers: string[];
117
+ claimed: number;
118
+ headroom: number;
119
+ peak: number;
120
+ /** Waits not yet admitted. */
121
+ waiting: number;
122
+ /** Length of the pause a limit refusal started, while it lasts; 0 when admitting. */
123
+ pauseMs: number;
50
124
  };
51
125
  /**
52
- * Name the per-DO accounting on a "Too many concurrent dynamic workers"
126
+ * Name the per-DO accounting on a "Dynamic worker concurrency limit exceeded"
53
127
  * failure; hand every other error back untouched. The platform's message
54
- * says only that the cap was hit — which ids hold the slots, and that a
55
- * keyed id can never give one back, is what the operator needs to know to
56
- * shrink anything.
128
+ * says only that the limit was hit — which workers were in flight, and what
129
+ * fan-outs had claimed, is what the operator needs to know to shrink anything.
57
130
  */
58
131
  export declare function withDynamicWorkerCapNamed<E>(ctx: object, error: E): E | Error;
59
132
  /**
@@ -1 +1 @@
1
- {"version":3,"file":"budgets.d.ts","sourceRoot":"","sources":["../src/budgets.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAwBH,2EAA2E;AAC3E,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,IAAI,CAE5D;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,IAAI,CAUxD;AAED,wDAAwD;AACxD,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG;IAC9C,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;CACzB,CAOA;AAED;;;;;;GAMG;AACH,wBAAgB,yBAAyB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,GAAG,KAAK,CAW7E;AAID;;;;;;;GAOG;AACH,eAAO,MAAM,+BAA+B,WAAa,CAAC;AAE1D;;;;;;;;;;;;;GAaG;AACH,wBAAgB,8BAA8B,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAoBrF;AAuBD;;;;;GAKG;AACH,eAAO,MAAM,wBAAwB,QAAS,CAAC;AAE/C,sEAAsE;AACtE,eAAO,MAAM,yBAAyB,iCAAiC,CAAC;AAExE,mEAAmE;AACnE,UAAU,sBAAsB;IAC9B,OAAO,EAAE;QACP,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;QAC7C,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;KACjD,CAAC;CACH;AAqCD;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,sBAAsB,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAatF;AAED,4EAA4E;AAC5E,wBAAgB,cAAc,CAAC,GAAG,EAAE,sBAAsB,GAAG,MAAM,CAGlE;AAED,4EAA4E;AAC5E,wBAAsB,qBAAqB,CAAC,GAAG,EAAE,sBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC,CAIxF;AAED;;;;;;;;GAQG;AACH,wBAAsB,aAAa,CACjC,GAAG,EAAE,sBAAsB,GAC1B,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAK/C;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAU9E"}
1
+ {"version":3,"file":"budgets.d.ts","sourceRoot":"","sources":["../src/budgets.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAKH;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,KAAK,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,IAAI,CAAC;AAEzD;;;;GAIG;AACH,MAAM,WAAW,kBAAkB;IACjC,OAAO,IAAI,IAAI,CAAC;CACjB;AAuJD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,kBAAkB,GAAG,cAAc,CAK3G;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,wBAAwB,CACtC,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,WAAW,CAAC;IAAC,KAAK,CAAC,EAAE,kBAAkB,CAAA;CAAO,GACjE,OAAO,CAAC,cAAc,CAAC,CA0BzB;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,kBAAkB,GAAG,IAAI,CAczF;AAED,wDAAwD;AACxD,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,8BAA8B;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,qFAAqF;IACrF,OAAO,EAAE,MAAM,CAAC;CACjB,CAWA;AAQD;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,GAAG,KAAK,CAU7E;AAID;;;;;;;GAOG;AACH,eAAO,MAAM,+BAA+B,WAAa,CAAC;AAE1D;;;;;;;;;;;;;GAaG;AACH,wBAAgB,8BAA8B,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAoBrF;AA4BD;;;;;GAKG;AACH,eAAO,MAAM,wBAAwB,QAAS,CAAC;AAE/C,sEAAsE;AACtE,eAAO,MAAM,yBAAyB,iCAAiC,CAAC;AAExE,mEAAmE;AACnE,UAAU,sBAAsB;IAC9B,OAAO,EAAE;QACP,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;QAC7C,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;KACjD,CAAC;CACH;AAqCD;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,sBAAsB,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAatF;AAED,4EAA4E;AAC5E,wBAAgB,cAAc,CAAC,GAAG,EAAE,sBAAsB,GAAG,MAAM,CAGlE;AAED,4EAA4E;AAC5E,wBAAsB,qBAAqB,CAAC,GAAG,EAAE,sBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC,CAIxF;AAED;;;;;;;;GAQG;AACH,wBAAsB,aAAa,CACjC,GAAG,EAAE,sBAAsB,GAC1B,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAK/C;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAU9E"}
package/dist/budgets.js CHANGED
@@ -1,46 +1,174 @@
1
1
  /**
2
2
  * budgets.ts — per-DO accounting for the platform budgets the fabric spends:
3
- * the Worker Loader's two caps, the facet-ID lifetime budget, and the
4
- * dynamic-worker module-map ceiling.
3
+ * the Durable Object's Dynamic Worker concurrency limit, the facet-ID
4
+ * lifetime budget, and the dynamic-worker module-map ceiling.
5
5
  *
6
- * Measured on production workerd: a Durable Object admits ~5–6 concurrent
7
- * dynamic workers before the platform refuses with "Too many concurrent
8
- * dynamic workers", one DO method can drive at most 4 concurrent Loader
9
- * fetches, and loader-cache entries are never released — every DISTINCT
10
- * `loader.get(id)` permanently consumes one of the dynamic-worker slots for
11
- * the object's lifetime. Nimbus stays under the caps by construction
12
- * (`IN_DO_THRESHOLD` = 5 in the fanout pool), which until now meant the slots
13
- * were counted in prose. This ledger counts them at the fabric's loader call
14
- * sites instead — the loader pool's slots, a resident process's keyed worker,
15
- * a one-shot's load — so proximity is measurable and a cap failure can name
16
- * the ids actually holding slots.
6
+ * The Dynamic Worker model is Cloudflare's documented one
7
+ * ({@link DO_DYNAMIC_WORKER_LIMIT}): a Durable Object may have a fixed number
8
+ * of DISTINCT Dynamic Workers with in-flight requests at once, shared across
9
+ * every concurrent request to that object (one I/O context), and any number
10
+ * of in-flight requests to the same Dynamic Worker count as one. Only
11
+ * in-flight requests count: a loader id with nothing in flight holds nothing.
17
12
  *
18
- * Measurement only: no admission control. The caps are the platform's, they
19
- * are approximate ("~5–6"), and a gate on an approximate number would refuse
20
- * work the platform would have run.
13
+ * The ledger counts, per hosting actor, the distinct workers that are in
14
+ * flight right now, keyed by loader id (a fresh key per unkeyed `load`), plus
15
+ * the width fan-outs have claimed and not yet released. A fan-out spends only
16
+ * the {@link dynamicWorkerHeadroom} that leaves, so work a Durable Object
17
+ * already has in flight — a resident process, the esbuild facet, a git
18
+ * network op, another fan-out — keeps its slots. Work that would rather wait
19
+ * than be refused waits on the ledger ({@link beginLoaderFetchWhenFree}) and
20
+ * is let in, in the order it asked, by whichever release makes room.
21
21
  *
22
22
  * Keyed weakly off the hosting actor's `ctx`, like the facet slot books: the
23
- * caps are per Durable Object, and dynamic workers die with the isolate that
23
+ * limit is per Durable Object, and dynamic workers die with the isolate that
24
24
  * loaded them, so a ledger that goes away with its host describes nothing
25
25
  * that still exists.
26
26
  */
27
27
  import { classifyError } from '@nimbus-sh/platform/oom-classify.js';
28
+ import { hostWasmIdentity } from './host-wasm.js';
29
+ /**
30
+ * Distinct Dynamic Workers one Durable Object may have with in-flight
31
+ * requests at once, shared across all concurrent requests to that object;
32
+ * multiple in-flight requests to one Dynamic Worker count once.
33
+ * https://developers.cloudflare.com/changelog/post/2026-08-28-durable-objects-dynamic-workers-limit/
34
+ */
35
+ export const DO_DYNAMIC_WORKER_LIMIT = 10;
36
+ /**
37
+ * The first pause after a limit refusal, doubling while refusals continue,
38
+ * up to {@link REFUSAL_PAUSE_MAX_MS}. A deployed Durable Object admitted a
39
+ * batch it had refused after a 6 s pause.
40
+ */
41
+ const REFUSAL_PAUSE_MS = 50;
42
+ const REFUSAL_PAUSE_MAX_MS = 2_000;
28
43
  const ledgers = new WeakMap();
44
+ const claimEntries = new WeakMap();
29
45
  function ledger(ctx) {
30
46
  let entry = ledgers.get(ctx);
31
47
  if (!entry) {
32
- entry = { ids: new Set(), liveFetches: 0, peakLiveFetches: 0 };
48
+ entry = {
49
+ inFlight: new Map(), claims: new Set(), peak: 0,
50
+ waiters: [], pauseMs: 0, pauseTimer: undefined, epoch: 0, refusals: 0,
51
+ };
33
52
  ledgers.set(ctx, entry);
34
53
  }
35
54
  return entry;
36
55
  }
37
- /** Record a keyed `loader.get(id)` — a permanent slot if the id is new. */
38
- export function recordLoaderId(ctx, id) {
39
- ledger(ctx).ids.add(id);
56
+ /** Distinct workers counted: each claim's width (or more, if its holds exceed it), plus held keys no claim covers. */
57
+ function inUse(entry) {
58
+ let count = 0;
59
+ const covered = new Set();
60
+ for (const claim of entry.claims) {
61
+ count += Math.max(claim.width, claim.keys.size);
62
+ for (const key of claim.keys.keys())
63
+ covered.add(key);
64
+ }
65
+ for (const key of entry.inFlight.keys())
66
+ if (!covered.has(key))
67
+ count++;
68
+ return count;
69
+ }
70
+ function headroom(entry) {
71
+ return entry.pauseMs > 0 ? 0 : Math.max(0, DO_DYNAMIC_WORKER_LIMIT - inUse(entry));
72
+ }
73
+ function claimOf(ctx, claim) {
74
+ if (claim === undefined)
75
+ return undefined;
76
+ const owned = claimEntries.get(claim);
77
+ if (owned === undefined || owned.ledger !== ledger(ctx)) {
78
+ throw new Error('Nimbus: a Dynamic Worker claim is used only on the ledger of the actor that claimed it');
79
+ }
80
+ return owned.ledger.claims.has(owned.entry) ? owned.entry : undefined;
81
+ }
82
+ function count(map, key, by) {
83
+ const open = (map.get(key) ?? 0) + by;
84
+ if (open > 0)
85
+ map.set(key, open);
86
+ else
87
+ map.delete(key);
88
+ }
89
+ /** Take one hold; the caller admits waiters after. */
90
+ function hold(entry, workerKey, claim) {
91
+ count(entry.inFlight, workerKey, 1);
92
+ if (claim)
93
+ count(claim.keys, workerKey, 1);
94
+ entry.peak = Math.max(entry.peak, inUse(entry));
95
+ const epoch = entry.epoch;
96
+ let ended = false;
97
+ return (failure) => {
98
+ if (ended)
99
+ return;
100
+ ended = true;
101
+ count(entry.inFlight, workerKey, -1);
102
+ if (claim)
103
+ count(claim.keys, workerKey, -1);
104
+ if (classifyError(failure) === 'dynamic_worker_cap')
105
+ refused(entry, epoch);
106
+ else if (epoch === entry.epoch && entry.pauseMs === 0)
107
+ entry.refusals = 0;
108
+ admitWaiters(entry);
109
+ };
110
+ }
111
+ /**
112
+ * The platform refused a worker this ledger counted room for: it still
113
+ * counts workers the ledger has released, which no release here can show.
114
+ * So nothing new is admitted until a pause has passed. A refusal of a call
115
+ * that began before the latest pause started or ended is the same lag and
116
+ * changes nothing; one of a call let in after it doubles the next pause.
117
+ */
118
+ function refused(entry, epoch) {
119
+ if (epoch !== entry.epoch)
120
+ return;
121
+ if (entry.pauseTimer !== undefined)
122
+ clearTimeout(entry.pauseTimer);
123
+ entry.pauseMs = Math.min(REFUSAL_PAUSE_MAX_MS, REFUSAL_PAUSE_MS * 2 ** entry.refusals);
124
+ entry.refusals++;
125
+ entry.epoch++;
126
+ entry.pauseTimer = setTimeout(() => {
127
+ entry.pauseMs = 0;
128
+ entry.pauseTimer = undefined;
129
+ entry.epoch++;
130
+ admitWaiters(entry);
131
+ }, entry.pauseMs);
132
+ }
133
+ function admissible(entry, waiter) {
134
+ // Requests to a worker already in flight count once, even while paused.
135
+ if (entry.inFlight.has(waiter.key))
136
+ return true;
137
+ if (entry.pauseMs > 0)
138
+ return false;
139
+ if (waiter.claim && entry.claims.has(waiter.claim) && waiter.claim.keys.size < waiter.claim.width)
140
+ return true;
141
+ return inUse(entry) < DO_DYNAMIC_WORKER_LIMIT;
142
+ }
143
+ /**
144
+ * Let in every waiter that fits, in the order they asked: each takes its
145
+ * hold here, so a freed slot goes to exactly one waiter and is never left
146
+ * between a wake and a begin. Run after every change that can make room.
147
+ * A waiter let in on a new key lets in the later ones on that key, and the
148
+ * earlier ones too: the scan starts over.
149
+ */
150
+ function admitWaiters(entry) {
151
+ for (let i = 0; i < entry.waiters.length;) {
152
+ const waiter = entry.waiters[i];
153
+ if (!admissible(entry, waiter)) {
154
+ i++;
155
+ continue;
156
+ }
157
+ const joins = entry.inFlight.has(waiter.key);
158
+ entry.waiters.splice(i, 1);
159
+ waiter.admit(hold(entry, waiter.key, waiter.claim));
160
+ if (!joins)
161
+ i = 0;
162
+ }
40
163
  }
41
164
  /**
42
- * Count one call into a dynamic worker as a live Loader fetch; the returned
43
- * function ends it (idempotently), from the caller's own `finally`.
165
+ * Hold the Dynamic Worker `workerKey` in flight on this actor's ledger; the
166
+ * returned function ends the hold (idempotently), from the caller's own
167
+ * `finally`. Holds on one key nest: the worker counts once until the last
168
+ * one ends, as the platform counts it. Under a `claim`, the hold counts
169
+ * inside the claim's width. This never waits: it is for work the actor
170
+ * starts regardless (a resident process); {@link beginLoaderFetchWhenFree}
171
+ * waits for room.
44
172
  *
45
173
  * A begin/end pair rather than a wrapper on purpose, and the shape is
46
174
  * load-bearing: wrapping the stub call in a ledger-owned async frame
@@ -53,43 +181,130 @@ export function recordLoaderId(ctx, id) {
53
181
  * workers: an RPC stub call must stay a direct property call awaited by the
54
182
  * frame that made it, so the ledger only brackets it.
55
183
  */
56
- export function beginLoaderFetch(ctx) {
184
+ export function beginLoaderFetch(ctx, workerKey, claim) {
57
185
  const entry = ledger(ctx);
58
- entry.liveFetches++;
59
- entry.peakLiveFetches = Math.max(entry.peakLiveFetches, entry.liveFetches);
60
- let ended = false;
61
- return () => {
62
- if (ended)
186
+ const end = hold(entry, workerKey, claimOf(ctx, claim));
187
+ admitWaiters(entry);
188
+ return end;
189
+ }
190
+ /**
191
+ * {@link beginLoaderFetch} once the ledger has room: resolves, holding
192
+ * `workerKey`, as soon as that worker is already in flight (holds on it
193
+ * count once) or a distinct worker more fits — within the `claim`'s width,
194
+ * or the actor's headroom. Waits are let in in the order they asked, by
195
+ * whoever's release makes the room: a hold's end, a claim's release, a
196
+ * pause's end. The hold is taken as the wait is let in, so a freed slot
197
+ * wakes one waiter and no other caller can take it first; once resolved, it
198
+ * is the caller's to end.
199
+ *
200
+ * A call refused with "Dynamic worker concurrency limit exceeded" ends its
201
+ * hold with the refusal (`end(error)`) and waits again: the refusal pauses
202
+ * admission (50 ms, doubling to 2 s while refusals continue), because the
203
+ * platform counts a worker for a moment after its call returns and no
204
+ * release can show that.
205
+ *
206
+ * `signal` abandons the wait: it rejects with the signal's reason and holds
207
+ * nothing. A wait outlives nothing on its own: bound it with a signal when
208
+ * room may never come (a resident process holds its worker for as long as
209
+ * it runs).
210
+ *
211
+ * const end = await beginLoaderFetchWhenFree(ctx, key, { signal });
212
+ * try { return await worker.getEntrypoint().run(); }
213
+ * catch (error) { end(error); throw error; }
214
+ * finally { end(); }
215
+ */
216
+ export function beginLoaderFetchWhenFree(ctx, workerKey, options = {}) {
217
+ const { signal } = options;
218
+ return new Promise((resolve, reject) => {
219
+ if (signal?.aborted) {
220
+ reject(signal.reason);
63
221
  return;
64
- ended = true;
65
- entry.liveFetches--;
222
+ }
223
+ const entry = ledger(ctx);
224
+ const abandon = () => {
225
+ const at = entry.waiters.indexOf(waiter);
226
+ if (at < 0)
227
+ return;
228
+ entry.waiters.splice(at, 1);
229
+ reject(signal?.reason);
230
+ };
231
+ const waiter = {
232
+ key: workerKey,
233
+ claim: claimOf(ctx, options.claim),
234
+ admit(end) {
235
+ signal?.removeEventListener('abort', abandon);
236
+ resolve(end);
237
+ },
238
+ };
239
+ entry.waiters.push(waiter);
240
+ signal?.addEventListener('abort', abandon, { once: true });
241
+ admitWaiters(entry);
242
+ });
243
+ }
244
+ /**
245
+ * Distinct Dynamic Workers this actor may still put in flight: the limit
246
+ * less what is held and claimed right now, and none while a limit refusal's
247
+ * pause lasts. Never negative.
248
+ */
249
+ export function dynamicWorkerHeadroom(ctx) {
250
+ return headroom(ledger(ctx));
251
+ }
252
+ /**
253
+ * Claim `width` distinct Dynamic Workers for one fan-out, or null when the
254
+ * headroom cannot hold it. The claim counts until `release` (idempotent), so
255
+ * a second fan-out sizing itself meanwhile sees it; the claimant's own
256
+ * dispatches, held under the claim, count inside it.
257
+ */
258
+ export function claimDynamicWorkers(ctx, width) {
259
+ const entry = ledger(ctx);
260
+ if (width < 1 || width > headroom(entry))
261
+ return null;
262
+ const claim = { width, keys: new Map() };
263
+ entry.claims.add(claim);
264
+ entry.peak = Math.max(entry.peak, inUse(entry));
265
+ const handle = {
266
+ release() {
267
+ if (!entry.claims.delete(claim))
268
+ return;
269
+ admitWaiters(entry);
270
+ },
66
271
  };
272
+ claimEntries.set(handle, { ledger: entry, entry: claim });
273
+ return handle;
67
274
  }
68
275
  /** Snapshot for the diag surface. Pure read; no I/O. */
69
276
  export function loaderLedgerStats(ctx) {
70
277
  const entry = ledger(ctx);
71
278
  return {
72
- idsEverGotten: [...entry.ids],
73
- liveFetches: entry.liveFetches,
74
- peakLiveFetches: entry.peakLiveFetches,
279
+ limit: DO_DYNAMIC_WORKER_LIMIT,
280
+ inFlightWorkers: [...entry.inFlight.keys()],
281
+ claimed: claimedWidth(entry),
282
+ headroom: headroom(entry),
283
+ peak: entry.peak,
284
+ waiting: entry.waiters.length,
285
+ pauseMs: entry.pauseMs,
75
286
  };
76
287
  }
288
+ function claimedWidth(entry) {
289
+ let width = 0;
290
+ for (const claim of entry.claims)
291
+ width += claim.width;
292
+ return width;
293
+ }
77
294
  /**
78
- * Name the per-DO accounting on a "Too many concurrent dynamic workers"
295
+ * Name the per-DO accounting on a "Dynamic worker concurrency limit exceeded"
79
296
  * failure; hand every other error back untouched. The platform's message
80
- * says only that the cap was hit — which ids hold the slots, and that a
81
- * keyed id can never give one back, is what the operator needs to know to
82
- * shrink anything.
297
+ * says only that the limit was hit — which workers were in flight, and what
298
+ * fan-outs had claimed, is what the operator needs to know to shrink anything.
83
299
  */
84
300
  export function withDynamicWorkerCapNamed(ctx, error) {
85
301
  if (classifyError(error) !== 'dynamic_worker_cap')
86
302
  return error;
87
303
  const entry = ledger(ctx);
88
304
  const platform = error instanceof Error ? error.message : String(error);
89
- return new Error(`${platform} — this Durable Object has ${entry.ids.size} loader id(s) permanently `
90
- + `holding dynamic-worker slots (a loader.get id is never released): `
91
- + `${[...entry.ids].join(', ') || '(none recorded)'}; live Loader fetches ${entry.liveFetches}, `
92
- + `peak ${entry.peakLiveFetches}`, { cause: error });
305
+ return new Error(`${platform} — this Durable Object had ${entry.inFlight.size} distinct dynamic worker(s) in flight `
306
+ + `(${[...entry.inFlight.keys()].join(', ') || 'none recorded'}) and ${claimedWidth(entry)} claimed by fan-outs, `
307
+ + `against a limit of ${DO_DYNAMIC_WORKER_LIMIT}; peak ${entry.peak}`, { cause: error });
93
308
  }
94
309
  // ── Dynamic-worker module-map ceiling ───────────────────────────────────────
95
310
  /**
@@ -136,13 +351,18 @@ export function assertModuleMapWithinCodeLimit(modules) {
136
351
  }
137
352
  /**
138
353
  * Bytes one module-map member carries, across the loader's content kinds
139
- * (plain string, `{ js | cjs | py | text }`, `{ wasm | data }`). With an
140
- * encoder, text is measured exactly; without one, by code-unit length.
354
+ * (plain string, `{ js | cjs | py | text }`, `{ wasm | data }`, a bare
355
+ * WebAssembly.Module). With an encoder, text is measured exactly; without
356
+ * one, by code-unit length. A compiled module counts the wire size its host
357
+ * described (host-wasm.ts); one nobody described counts nothing here and is
358
+ * left to the platform's own refusal, as the text undercount is.
141
359
  */
142
360
  function memberBytes(content, encoder) {
143
361
  const textBytes = (text) => encoder ? encoder.encode(text).byteLength : text.length;
144
362
  if (typeof content === 'string')
145
363
  return textBytes(content);
364
+ if (content instanceof WebAssembly.Module)
365
+ return hostWasmIdentity(content)?.bytes ?? 0;
146
366
  if (content !== null && typeof content === 'object') {
147
367
  for (const value of Object.values(content)) {
148
368
  if (typeof value === 'string')
@@ -151,6 +371,8 @@ function memberBytes(content, encoder) {
151
371
  return value.byteLength;
152
372
  if (ArrayBuffer.isView(value))
153
373
  return value.byteLength;
374
+ if (value instanceof WebAssembly.Module)
375
+ return hostWasmIdentity(value)?.bytes ?? 0;
154
376
  }
155
377
  }
156
378
  return 0;
@@ -40,6 +40,7 @@
40
40
  * namespace stubs, where the thunk shape is production-proven in Proteus.
41
41
  */
42
42
  import { type DoCallClass } from '@nimbus-sh/platform/oom-classify.js';
43
+ import { type SpanRecorder } from '@nimbus-sh/platform/tracing.js';
43
44
  export interface DoCallRetryPolicy {
44
45
  maxAttempts?: number;
45
46
  baseDelayMs?: number;
@@ -78,7 +79,26 @@ export interface DoCallRetryPolicy {
78
79
  * its error.
79
80
  */
80
81
  onRetry?(info: DoCallRetryInfo): void;
82
+ /**
83
+ * Where the call's telemetry goes: its span's recorder. Each attempt lost
84
+ * to a transient or overloaded failure is recorded as an exception whose
85
+ * `code` is the failure's class, and once the call has settled it gets
86
+ * `do_call.attempts` (started), `do_call.hedges` (started by a hedge),
87
+ * `do_call.answered_by` (the attempt whose answer the call took, absent
88
+ * when none answered) and `do_call.outcome` ({@link DoCallOutcome}).
89
+ * Nothing recorded can change the call's answer. Records nothing when
90
+ * absent.
91
+ */
92
+ span?: SpanRecorder;
81
93
  }
94
+ /**
95
+ * How an `idempotent` call ended: `answered` (an attempt succeeded),
96
+ * `callee_error` (the callee's own failure, which is an answer),
97
+ * `exhausted` (the last transient failure, no repeat left), `overloaded`
98
+ * (shed, nothing repeated after it), or `caller_error` (the resolver or
99
+ * `onRetry` threw).
100
+ */
101
+ export type DoCallOutcome = 'answered' | 'callee_error' | 'exhausted' | 'overloaded' | 'caller_error';
82
102
  /** What one retry is answering: which call, which platform class, which
83
103
  * attempt just failed out of how many. */
84
104
  export interface DoCallRetryInfo {
@@ -1 +1 @@
1
- {"version":3,"file":"do-calls.d.ts","sourceRoot":"","sources":["../src/do-calls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AAEH,OAAO,EAAqC,KAAK,WAAW,EAAE,MAAM,qCAAqC,CAAC;AAU1G,MAAM,WAAW,iBAAiB;IAChC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;;;;;;;;OAeG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;;;;OAQG;IACH,OAAO,CAAC,CAAC,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;CACvC;AAED;2CAC2C;AAC3C,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,WAAW,CAAC;IAC5B;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,IAAI,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;AAErD;;;;;GAKG;AACH,qBAAa,WAAY,SAAQ,KAAK;IAElC,QAAQ,CAAC,SAAS,EAAE,MAAM;IAC1B,QAAQ,CAAC,IAAI,EAAE,YAAY,GAAG,UAAU;IACxC,QAAQ,CAAC,cAAc,EAAE,WAAW;gBAF3B,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,YAAY,GAAG,UAAU,EAC/B,cAAc,EAAE,WAAW,EACpC,KAAK,EAAE,OAAO;CASjB;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,UAAU,CAAC,CAAC,EAAE,CAAC,EAC7B,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,EACvB,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,EAC7B,MAAM,GAAE,iBAAsB,GAC7B,OAAO,CAAC,CAAC,CAAC,CAmHZ;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,QAAQ,CAAC,CAAC,EAAE,CAAC,EACjC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,EACvB,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,GAC5B,OAAO,CAAC,CAAC,CAAC,CAUZ"}
1
+ {"version":3,"file":"do-calls.d.ts","sourceRoot":"","sources":["../src/do-calls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AAEH,OAAO,EAAqC,KAAK,WAAW,EAAE,MAAM,qCAAqC,CAAC;AAE1G,OAAO,EAAY,KAAK,YAAY,EAAE,MAAM,gCAAgC,CAAC;AAS7E,MAAM,WAAW,iBAAiB;IAChC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;;;;;;;;OAeG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;;;;OAQG;IACH,OAAO,CAAC,CAAC,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;IACtC;;;;;;;;;OASG;IACH,IAAI,CAAC,EAAE,YAAY,CAAC;CACrB;AAED;;;;;;GAMG;AACH,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,cAAc,GAAG,WAAW,GAAG,YAAY,GAAG,cAAc,CAAC;AAEtG;2CAC2C;AAC3C,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,WAAW,CAAC;IAC5B;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,IAAI,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;AAErD;;;;;GAKG;AACH,qBAAa,WAAY,SAAQ,KAAK;IAElC,QAAQ,CAAC,SAAS,EAAE,MAAM;IAC1B,QAAQ,CAAC,IAAI,EAAE,YAAY,GAAG,UAAU;IACxC,QAAQ,CAAC,cAAc,EAAE,WAAW;gBAF3B,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,YAAY,GAAG,UAAU,EAC/B,cAAc,EAAE,WAAW,EACpC,KAAK,EAAE,OAAO;CASjB;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,UAAU,CAAC,CAAC,EAAE,CAAC,EAC7B,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,EACvB,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,EAC7B,MAAM,GAAE,iBAAsB,GAC7B,OAAO,CAAC,CAAC,CAAC,CAgIZ;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,QAAQ,CAAC,CAAC,EAAE,CAAC,EACjC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,EACvB,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,GAC5B,OAAO,CAAC,CAAC,CAAC,CAUZ"}