@nimbus-sh/fabric 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +98 -11
- package/dist/bindings.d.ts +31 -33
- package/dist/bindings.d.ts.map +1 -1
- package/dist/bindings.js +108 -97
- package/dist/budgets.d.ts +102 -29
- package/dist/budgets.d.ts.map +1 -1
- package/dist/budgets.js +266 -44
- package/dist/do-calls.d.ts +20 -0
- package/dist/do-calls.d.ts.map +1 -1
- package/dist/do-calls.js +24 -11
- package/dist/fanout.d.ts +40 -32
- package/dist/fanout.d.ts.map +1 -1
- package/dist/fanout.js +48 -51
- package/dist/fenced-work.d.ts +3 -3
- package/dist/fenced-work.js +3 -3
- package/dist/host-wasm.d.ts +29 -0
- package/dist/host-wasm.d.ts.map +1 -0
- package/dist/host-wasm.js +31 -0
- package/dist/image-store.d.ts +1 -1
- package/dist/image-store.d.ts.map +1 -1
- package/dist/image-store.js +33 -1
- package/dist/inner-do-env.d.ts +83 -0
- package/dist/inner-do-env.d.ts.map +1 -0
- package/dist/inner-do-env.js +181 -0
- package/dist/isolate-pool.d.ts +40 -23
- package/dist/isolate-pool.d.ts.map +1 -1
- package/dist/isolate-pool.js +105 -55
- package/dist/process-fabric.d.ts +26 -11
- package/dist/process-fabric.d.ts.map +1 -1
- package/dist/process-fabric.js +44 -0
- package/dist/timers.d.ts +12 -0
- package/dist/timers.d.ts.map +1 -1
- package/dist/timers.js +44 -9
- package/dist/vendor/types.d.ts +11 -5
- package/dist/vendor/types.d.ts.map +1 -1
- package/dist/workerd-facet-host.d.ts.map +1 -1
- package/dist/workerd-facet-host.js +35 -30
- package/package.json +4 -4
- package/src/bindings.ts +121 -98
- package/src/budgets.ts +311 -53
- package/src/do-calls.ts +45 -11
- package/src/fanout.ts +62 -53
- package/src/fenced-work.ts +3 -3
- package/src/host-wasm.ts +41 -0
- package/src/image-store.ts +29 -2
- package/src/inner-do-env.ts +213 -0
- package/src/isolate-pool.ts +145 -75
- package/src/process-fabric.ts +55 -13
- package/src/timers.ts +45 -9
- package/src/vendor/types.ts +11 -5
- package/src/workerd-facet-host.ts +36 -31
package/dist/budgets.d.ts
CHANGED
|
@@ -1,34 +1,59 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* budgets.ts — per-DO accounting for the platform budgets the fabric spends:
|
|
3
|
-
* the
|
|
4
|
-
* dynamic-worker module-map ceiling.
|
|
3
|
+
* the Durable Object's Dynamic Worker concurrency limit, the facet-ID
|
|
4
|
+
* lifetime budget, and the dynamic-worker module-map ceiling.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
* (`IN_DO_THRESHOLD` = 5 in the fanout pool), which until now meant the slots
|
|
13
|
-
* were counted in prose. This ledger counts them at the fabric's loader call
|
|
14
|
-
* sites instead — the loader pool's slots, a resident process's keyed worker,
|
|
15
|
-
* a one-shot's load — so proximity is measurable and a cap failure can name
|
|
16
|
-
* the ids actually holding slots.
|
|
6
|
+
* The Dynamic Worker model is Cloudflare's documented one
|
|
7
|
+
* ({@link DO_DYNAMIC_WORKER_LIMIT}): a Durable Object may have a fixed number
|
|
8
|
+
* of DISTINCT Dynamic Workers with in-flight requests at once, shared across
|
|
9
|
+
* every concurrent request to that object (one I/O context), and any number
|
|
10
|
+
* of in-flight requests to the same Dynamic Worker count as one. Only
|
|
11
|
+
* in-flight requests count: a loader id with nothing in flight holds nothing.
|
|
17
12
|
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
13
|
+
* The ledger counts, per hosting actor, the distinct workers that are in
|
|
14
|
+
* flight right now, keyed by loader id (a fresh key per unkeyed `load`), plus
|
|
15
|
+
* the width fan-outs have claimed and not yet released. A fan-out spends only
|
|
16
|
+
* the {@link dynamicWorkerHeadroom} that leaves, so work a Durable Object
|
|
17
|
+
* already has in flight — a resident process, the esbuild facet, a git
|
|
18
|
+
* network op, another fan-out — keeps its slots. Work that would rather wait
|
|
19
|
+
* than be refused waits on the ledger ({@link beginLoaderFetchWhenFree}) and
|
|
20
|
+
* is let in, in the order it asked, by whichever release makes room.
|
|
21
21
|
*
|
|
22
22
|
* Keyed weakly off the hosting actor's `ctx`, like the facet slot books: the
|
|
23
|
-
*
|
|
23
|
+
* limit is per Durable Object, and dynamic workers die with the isolate that
|
|
24
24
|
* loaded them, so a ledger that goes away with its host describes nothing
|
|
25
25
|
* that still exists.
|
|
26
26
|
*/
|
|
27
|
-
/** Record a keyed `loader.get(id)` — a permanent slot if the id is new. */
|
|
28
|
-
export declare function recordLoaderId(ctx: object, id: string): void;
|
|
29
27
|
/**
|
|
30
|
-
*
|
|
31
|
-
*
|
|
28
|
+
* Distinct Dynamic Workers one Durable Object may have with in-flight
|
|
29
|
+
* requests at once, shared across all concurrent requests to that object;
|
|
30
|
+
* multiple in-flight requests to one Dynamic Worker count once.
|
|
31
|
+
* https://developers.cloudflare.com/changelog/post/2026-08-28-durable-objects-dynamic-workers-limit/
|
|
32
|
+
*/
|
|
33
|
+
export declare const DO_DYNAMIC_WORKER_LIMIT = 10;
|
|
34
|
+
/**
|
|
35
|
+
* Ends one hold, idempotently. Pass the error the call failed with, if it
|
|
36
|
+
* did: a "Dynamic worker concurrency limit exceeded" refusal pauses the
|
|
37
|
+
* ledger's admissions (see {@link beginLoaderFetchWhenFree}); anything else,
|
|
38
|
+
* or nothing, just ends the hold.
|
|
39
|
+
*/
|
|
40
|
+
export type EndLoaderFetch = (failure?: unknown) => void;
|
|
41
|
+
/**
|
|
42
|
+
* A width one fan-out reserved with {@link claimDynamicWorkers}. Holds taken
|
|
43
|
+
* under it (`beginLoaderFetch(ctx, key, claim)`) count inside that width, not
|
|
44
|
+
* on top of it, until `release` (idempotent).
|
|
45
|
+
*/
|
|
46
|
+
export interface DynamicWorkerClaim {
|
|
47
|
+
release(): void;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Hold the Dynamic Worker `workerKey` in flight on this actor's ledger; the
|
|
51
|
+
* returned function ends the hold (idempotently), from the caller's own
|
|
52
|
+
* `finally`. Holds on one key nest: the worker counts once until the last
|
|
53
|
+
* one ends, as the platform counts it. Under a `claim`, the hold counts
|
|
54
|
+
* inside the claim's width. This never waits: it is for work the actor
|
|
55
|
+
* starts regardless (a resident process); {@link beginLoaderFetchWhenFree}
|
|
56
|
+
* waits for room.
|
|
32
57
|
*
|
|
33
58
|
* A begin/end pair rather than a wrapper on purpose, and the shape is
|
|
34
59
|
* load-bearing: wrapping the stub call in a ledger-owned async frame
|
|
@@ -41,19 +66,67 @@ export declare function recordLoaderId(ctx: object, id: string): void;
|
|
|
41
66
|
* workers: an RPC stub call must stay a direct property call awaited by the
|
|
42
67
|
* frame that made it, so the ledger only brackets it.
|
|
43
68
|
*/
|
|
44
|
-
export declare function beginLoaderFetch(ctx: object
|
|
69
|
+
export declare function beginLoaderFetch(ctx: object, workerKey: string, claim?: DynamicWorkerClaim): EndLoaderFetch;
|
|
70
|
+
/**
|
|
71
|
+
* {@link beginLoaderFetch} once the ledger has room: resolves, holding
|
|
72
|
+
* `workerKey`, as soon as that worker is already in flight (holds on it
|
|
73
|
+
* count once) or a distinct worker more fits — within the `claim`'s width,
|
|
74
|
+
* or the actor's headroom. Waits are let in in the order they asked, by
|
|
75
|
+
* whoever's release makes the room: a hold's end, a claim's release, a
|
|
76
|
+
* pause's end. The hold is taken as the wait is let in, so a freed slot
|
|
77
|
+
* wakes one waiter and no other caller can take it first; once resolved, it
|
|
78
|
+
* is the caller's to end.
|
|
79
|
+
*
|
|
80
|
+
* A call refused with "Dynamic worker concurrency limit exceeded" ends its
|
|
81
|
+
* hold with the refusal (`end(error)`) and waits again: the refusal pauses
|
|
82
|
+
* admission (50 ms, doubling to 2 s while refusals continue), because the
|
|
83
|
+
* platform counts a worker for a moment after its call returns and no
|
|
84
|
+
* release can show that.
|
|
85
|
+
*
|
|
86
|
+
* `signal` abandons the wait: it rejects with the signal's reason and holds
|
|
87
|
+
* nothing. A wait outlives nothing on its own: bound it with a signal when
|
|
88
|
+
* room may never come (a resident process holds its worker for as long as
|
|
89
|
+
* it runs).
|
|
90
|
+
*
|
|
91
|
+
* const end = await beginLoaderFetchWhenFree(ctx, key, { signal });
|
|
92
|
+
* try { return await worker.getEntrypoint().run(); }
|
|
93
|
+
* catch (error) { end(error); throw error; }
|
|
94
|
+
* finally { end(); }
|
|
95
|
+
*/
|
|
96
|
+
export declare function beginLoaderFetchWhenFree(ctx: object, workerKey: string, options?: {
|
|
97
|
+
signal?: AbortSignal;
|
|
98
|
+
claim?: DynamicWorkerClaim;
|
|
99
|
+
}): Promise<EndLoaderFetch>;
|
|
100
|
+
/**
|
|
101
|
+
* Distinct Dynamic Workers this actor may still put in flight: the limit
|
|
102
|
+
* less what is held and claimed right now, and none while a limit refusal's
|
|
103
|
+
* pause lasts. Never negative.
|
|
104
|
+
*/
|
|
105
|
+
export declare function dynamicWorkerHeadroom(ctx: object): number;
|
|
106
|
+
/**
|
|
107
|
+
* Claim `width` distinct Dynamic Workers for one fan-out, or null when the
|
|
108
|
+
* headroom cannot hold it. The claim counts until `release` (idempotent), so
|
|
109
|
+
* a second fan-out sizing itself meanwhile sees it; the claimant's own
|
|
110
|
+
* dispatches, held under the claim, count inside it.
|
|
111
|
+
*/
|
|
112
|
+
export declare function claimDynamicWorkers(ctx: object, width: number): DynamicWorkerClaim | null;
|
|
45
113
|
/** Snapshot for the diag surface. Pure read; no I/O. */
|
|
46
114
|
export declare function loaderLedgerStats(ctx: object): {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
115
|
+
limit: number;
|
|
116
|
+
inFlightWorkers: string[];
|
|
117
|
+
claimed: number;
|
|
118
|
+
headroom: number;
|
|
119
|
+
peak: number;
|
|
120
|
+
/** Waits not yet admitted. */
|
|
121
|
+
waiting: number;
|
|
122
|
+
/** Length of the pause a limit refusal started, while it lasts; 0 when admitting. */
|
|
123
|
+
pauseMs: number;
|
|
50
124
|
};
|
|
51
125
|
/**
|
|
52
|
-
* Name the per-DO accounting on a "
|
|
126
|
+
* Name the per-DO accounting on a "Dynamic worker concurrency limit exceeded"
|
|
53
127
|
* failure; hand every other error back untouched. The platform's message
|
|
54
|
-
* says only that the
|
|
55
|
-
*
|
|
56
|
-
* shrink anything.
|
|
128
|
+
* says only that the limit was hit — which workers were in flight, and what
|
|
129
|
+
* fan-outs had claimed, is what the operator needs to know to shrink anything.
|
|
57
130
|
*/
|
|
58
131
|
export declare function withDynamicWorkerCapNamed<E>(ctx: object, error: E): E | Error;
|
|
59
132
|
/**
|
package/dist/budgets.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"budgets.d.ts","sourceRoot":"","sources":["../src/budgets.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;
|
|
1
|
+
{"version":3,"file":"budgets.d.ts","sourceRoot":"","sources":["../src/budgets.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAKH;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,KAAK,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,IAAI,CAAC;AAEzD;;;;GAIG;AACH,MAAM,WAAW,kBAAkB;IACjC,OAAO,IAAI,IAAI,CAAC;CACjB;AAuJD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,kBAAkB,GAAG,cAAc,CAK3G;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,wBAAwB,CACtC,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,WAAW,CAAC;IAAC,KAAK,CAAC,EAAE,kBAAkB,CAAA;CAAO,GACjE,OAAO,CAAC,cAAc,CAAC,CA0BzB;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,kBAAkB,GAAG,IAAI,CAczF;AAED,wDAAwD;AACxD,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,8BAA8B;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,qFAAqF;IACrF,OAAO,EAAE,MAAM,CAAC;CACjB,CAWA;AAQD;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,GAAG,KAAK,CAU7E;AAID;;;;;;;GAOG;AACH,eAAO,MAAM,+BAA+B,WAAa,CAAC;AAE1D;;;;;;;;;;;;;GAaG;AACH,wBAAgB,8BAA8B,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAoBrF;AA4BD;;;;;GAKG;AACH,eAAO,MAAM,wBAAwB,QAAS,CAAC;AAE/C,sEAAsE;AACtE,eAAO,MAAM,yBAAyB,iCAAiC,CAAC;AAExE,mEAAmE;AACnE,UAAU,sBAAsB;IAC9B,OAAO,EAAE;QACP,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;QAC7C,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;KACjD,CAAC;CACH;AAqCD;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,sBAAsB,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAatF;AAED,4EAA4E;AAC5E,wBAAgB,cAAc,CAAC,GAAG,EAAE,sBAAsB,GAAG,MAAM,CAGlE;AAED,4EAA4E;AAC5E,wBAAsB,qBAAqB,CAAC,GAAG,EAAE,sBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC,CAIxF;AAED;;;;;;;;GAQG;AACH,wBAAsB,aAAa,CACjC,GAAG,EAAE,sBAAsB,GAC1B,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAK/C;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAU9E"}
|
package/dist/budgets.js
CHANGED
|
@@ -1,46 +1,174 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* budgets.ts — per-DO accounting for the platform budgets the fabric spends:
|
|
3
|
-
* the
|
|
4
|
-
* dynamic-worker module-map ceiling.
|
|
3
|
+
* the Durable Object's Dynamic Worker concurrency limit, the facet-ID
|
|
4
|
+
* lifetime budget, and the dynamic-worker module-map ceiling.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
* (`IN_DO_THRESHOLD` = 5 in the fanout pool), which until now meant the slots
|
|
13
|
-
* were counted in prose. This ledger counts them at the fabric's loader call
|
|
14
|
-
* sites instead — the loader pool's slots, a resident process's keyed worker,
|
|
15
|
-
* a one-shot's load — so proximity is measurable and a cap failure can name
|
|
16
|
-
* the ids actually holding slots.
|
|
6
|
+
* The Dynamic Worker model is Cloudflare's documented one
|
|
7
|
+
* ({@link DO_DYNAMIC_WORKER_LIMIT}): a Durable Object may have a fixed number
|
|
8
|
+
* of DISTINCT Dynamic Workers with in-flight requests at once, shared across
|
|
9
|
+
* every concurrent request to that object (one I/O context), and any number
|
|
10
|
+
* of in-flight requests to the same Dynamic Worker count as one. Only
|
|
11
|
+
* in-flight requests count: a loader id with nothing in flight holds nothing.
|
|
17
12
|
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
13
|
+
* The ledger counts, per hosting actor, the distinct workers that are in
|
|
14
|
+
* flight right now, keyed by loader id (a fresh key per unkeyed `load`), plus
|
|
15
|
+
* the width fan-outs have claimed and not yet released. A fan-out spends only
|
|
16
|
+
* the {@link dynamicWorkerHeadroom} that leaves, so work a Durable Object
|
|
17
|
+
* already has in flight — a resident process, the esbuild facet, a git
|
|
18
|
+
* network op, another fan-out — keeps its slots. Work that would rather wait
|
|
19
|
+
* than be refused waits on the ledger ({@link beginLoaderFetchWhenFree}) and
|
|
20
|
+
* is let in, in the order it asked, by whichever release makes room.
|
|
21
21
|
*
|
|
22
22
|
* Keyed weakly off the hosting actor's `ctx`, like the facet slot books: the
|
|
23
|
-
*
|
|
23
|
+
* limit is per Durable Object, and dynamic workers die with the isolate that
|
|
24
24
|
* loaded them, so a ledger that goes away with its host describes nothing
|
|
25
25
|
* that still exists.
|
|
26
26
|
*/
|
|
27
27
|
import { classifyError } from '@nimbus-sh/platform/oom-classify.js';
|
|
28
|
+
import { hostWasmIdentity } from './host-wasm.js';
|
|
29
|
+
/**
|
|
30
|
+
* Distinct Dynamic Workers one Durable Object may have with in-flight
|
|
31
|
+
* requests at once, shared across all concurrent requests to that object;
|
|
32
|
+
* multiple in-flight requests to one Dynamic Worker count once.
|
|
33
|
+
* https://developers.cloudflare.com/changelog/post/2026-08-28-durable-objects-dynamic-workers-limit/
|
|
34
|
+
*/
|
|
35
|
+
export const DO_DYNAMIC_WORKER_LIMIT = 10;
|
|
36
|
+
/**
|
|
37
|
+
* The first pause after a limit refusal, doubling while refusals continue,
|
|
38
|
+
* up to {@link REFUSAL_PAUSE_MAX_MS}. A deployed Durable Object admitted a
|
|
39
|
+
* batch it had refused after a 6 s pause.
|
|
40
|
+
*/
|
|
41
|
+
const REFUSAL_PAUSE_MS = 50;
|
|
42
|
+
const REFUSAL_PAUSE_MAX_MS = 2_000;
|
|
28
43
|
const ledgers = new WeakMap();
|
|
44
|
+
const claimEntries = new WeakMap();
|
|
29
45
|
function ledger(ctx) {
|
|
30
46
|
let entry = ledgers.get(ctx);
|
|
31
47
|
if (!entry) {
|
|
32
|
-
entry = {
|
|
48
|
+
entry = {
|
|
49
|
+
inFlight: new Map(), claims: new Set(), peak: 0,
|
|
50
|
+
waiters: [], pauseMs: 0, pauseTimer: undefined, epoch: 0, refusals: 0,
|
|
51
|
+
};
|
|
33
52
|
ledgers.set(ctx, entry);
|
|
34
53
|
}
|
|
35
54
|
return entry;
|
|
36
55
|
}
|
|
37
|
-
/**
|
|
38
|
-
|
|
39
|
-
|
|
56
|
+
/** Distinct workers counted: each claim's width (or more, if its holds exceed it), plus held keys no claim covers. */
|
|
57
|
+
function inUse(entry) {
|
|
58
|
+
let count = 0;
|
|
59
|
+
const covered = new Set();
|
|
60
|
+
for (const claim of entry.claims) {
|
|
61
|
+
count += Math.max(claim.width, claim.keys.size);
|
|
62
|
+
for (const key of claim.keys.keys())
|
|
63
|
+
covered.add(key);
|
|
64
|
+
}
|
|
65
|
+
for (const key of entry.inFlight.keys())
|
|
66
|
+
if (!covered.has(key))
|
|
67
|
+
count++;
|
|
68
|
+
return count;
|
|
69
|
+
}
|
|
70
|
+
function headroom(entry) {
|
|
71
|
+
return entry.pauseMs > 0 ? 0 : Math.max(0, DO_DYNAMIC_WORKER_LIMIT - inUse(entry));
|
|
72
|
+
}
|
|
73
|
+
function claimOf(ctx, claim) {
|
|
74
|
+
if (claim === undefined)
|
|
75
|
+
return undefined;
|
|
76
|
+
const owned = claimEntries.get(claim);
|
|
77
|
+
if (owned === undefined || owned.ledger !== ledger(ctx)) {
|
|
78
|
+
throw new Error('Nimbus: a Dynamic Worker claim is used only on the ledger of the actor that claimed it');
|
|
79
|
+
}
|
|
80
|
+
return owned.ledger.claims.has(owned.entry) ? owned.entry : undefined;
|
|
81
|
+
}
|
|
82
|
+
function count(map, key, by) {
|
|
83
|
+
const open = (map.get(key) ?? 0) + by;
|
|
84
|
+
if (open > 0)
|
|
85
|
+
map.set(key, open);
|
|
86
|
+
else
|
|
87
|
+
map.delete(key);
|
|
88
|
+
}
|
|
89
|
+
/** Take one hold; the caller admits waiters after. */
|
|
90
|
+
function hold(entry, workerKey, claim) {
|
|
91
|
+
count(entry.inFlight, workerKey, 1);
|
|
92
|
+
if (claim)
|
|
93
|
+
count(claim.keys, workerKey, 1);
|
|
94
|
+
entry.peak = Math.max(entry.peak, inUse(entry));
|
|
95
|
+
const epoch = entry.epoch;
|
|
96
|
+
let ended = false;
|
|
97
|
+
return (failure) => {
|
|
98
|
+
if (ended)
|
|
99
|
+
return;
|
|
100
|
+
ended = true;
|
|
101
|
+
count(entry.inFlight, workerKey, -1);
|
|
102
|
+
if (claim)
|
|
103
|
+
count(claim.keys, workerKey, -1);
|
|
104
|
+
if (classifyError(failure) === 'dynamic_worker_cap')
|
|
105
|
+
refused(entry, epoch);
|
|
106
|
+
else if (epoch === entry.epoch && entry.pauseMs === 0)
|
|
107
|
+
entry.refusals = 0;
|
|
108
|
+
admitWaiters(entry);
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* The platform refused a worker this ledger counted room for: it still
|
|
113
|
+
* counts workers the ledger has released, which no release here can show.
|
|
114
|
+
* So nothing new is admitted until a pause has passed. A refusal of a call
|
|
115
|
+
* that began before the latest pause started or ended is the same lag and
|
|
116
|
+
* changes nothing; one of a call let in after it doubles the next pause.
|
|
117
|
+
*/
|
|
118
|
+
function refused(entry, epoch) {
|
|
119
|
+
if (epoch !== entry.epoch)
|
|
120
|
+
return;
|
|
121
|
+
if (entry.pauseTimer !== undefined)
|
|
122
|
+
clearTimeout(entry.pauseTimer);
|
|
123
|
+
entry.pauseMs = Math.min(REFUSAL_PAUSE_MAX_MS, REFUSAL_PAUSE_MS * 2 ** entry.refusals);
|
|
124
|
+
entry.refusals++;
|
|
125
|
+
entry.epoch++;
|
|
126
|
+
entry.pauseTimer = setTimeout(() => {
|
|
127
|
+
entry.pauseMs = 0;
|
|
128
|
+
entry.pauseTimer = undefined;
|
|
129
|
+
entry.epoch++;
|
|
130
|
+
admitWaiters(entry);
|
|
131
|
+
}, entry.pauseMs);
|
|
132
|
+
}
|
|
133
|
+
function admissible(entry, waiter) {
|
|
134
|
+
// Requests to a worker already in flight count once, even while paused.
|
|
135
|
+
if (entry.inFlight.has(waiter.key))
|
|
136
|
+
return true;
|
|
137
|
+
if (entry.pauseMs > 0)
|
|
138
|
+
return false;
|
|
139
|
+
if (waiter.claim && entry.claims.has(waiter.claim) && waiter.claim.keys.size < waiter.claim.width)
|
|
140
|
+
return true;
|
|
141
|
+
return inUse(entry) < DO_DYNAMIC_WORKER_LIMIT;
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Let in every waiter that fits, in the order they asked: each takes its
|
|
145
|
+
* hold here, so a freed slot goes to exactly one waiter and is never left
|
|
146
|
+
* between a wake and a begin. Run after every change that can make room.
|
|
147
|
+
* A waiter let in on a new key lets in the later ones on that key, and the
|
|
148
|
+
* earlier ones too: the scan starts over.
|
|
149
|
+
*/
|
|
150
|
+
function admitWaiters(entry) {
|
|
151
|
+
for (let i = 0; i < entry.waiters.length;) {
|
|
152
|
+
const waiter = entry.waiters[i];
|
|
153
|
+
if (!admissible(entry, waiter)) {
|
|
154
|
+
i++;
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
const joins = entry.inFlight.has(waiter.key);
|
|
158
|
+
entry.waiters.splice(i, 1);
|
|
159
|
+
waiter.admit(hold(entry, waiter.key, waiter.claim));
|
|
160
|
+
if (!joins)
|
|
161
|
+
i = 0;
|
|
162
|
+
}
|
|
40
163
|
}
|
|
41
164
|
/**
|
|
42
|
-
*
|
|
43
|
-
* function ends
|
|
165
|
+
* Hold the Dynamic Worker `workerKey` in flight on this actor's ledger; the
|
|
166
|
+
* returned function ends the hold (idempotently), from the caller's own
|
|
167
|
+
* `finally`. Holds on one key nest: the worker counts once until the last
|
|
168
|
+
* one ends, as the platform counts it. Under a `claim`, the hold counts
|
|
169
|
+
* inside the claim's width. This never waits: it is for work the actor
|
|
170
|
+
* starts regardless (a resident process); {@link beginLoaderFetchWhenFree}
|
|
171
|
+
* waits for room.
|
|
44
172
|
*
|
|
45
173
|
* A begin/end pair rather than a wrapper on purpose, and the shape is
|
|
46
174
|
* load-bearing: wrapping the stub call in a ledger-owned async frame
|
|
@@ -53,43 +181,130 @@ export function recordLoaderId(ctx, id) {
|
|
|
53
181
|
* workers: an RPC stub call must stay a direct property call awaited by the
|
|
54
182
|
* frame that made it, so the ledger only brackets it.
|
|
55
183
|
*/
|
|
56
|
-
export function beginLoaderFetch(ctx) {
|
|
184
|
+
export function beginLoaderFetch(ctx, workerKey, claim) {
|
|
57
185
|
const entry = ledger(ctx);
|
|
58
|
-
entry
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
186
|
+
const end = hold(entry, workerKey, claimOf(ctx, claim));
|
|
187
|
+
admitWaiters(entry);
|
|
188
|
+
return end;
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* {@link beginLoaderFetch} once the ledger has room: resolves, holding
|
|
192
|
+
* `workerKey`, as soon as that worker is already in flight (holds on it
|
|
193
|
+
* count once) or a distinct worker more fits — within the `claim`'s width,
|
|
194
|
+
* or the actor's headroom. Waits are let in in the order they asked, by
|
|
195
|
+
* whoever's release makes the room: a hold's end, a claim's release, a
|
|
196
|
+
* pause's end. The hold is taken as the wait is let in, so a freed slot
|
|
197
|
+
* wakes one waiter and no other caller can take it first; once resolved, it
|
|
198
|
+
* is the caller's to end.
|
|
199
|
+
*
|
|
200
|
+
* A call refused with "Dynamic worker concurrency limit exceeded" ends its
|
|
201
|
+
* hold with the refusal (`end(error)`) and waits again: the refusal pauses
|
|
202
|
+
* admission (50 ms, doubling to 2 s while refusals continue), because the
|
|
203
|
+
* platform counts a worker for a moment after its call returns and no
|
|
204
|
+
* release can show that.
|
|
205
|
+
*
|
|
206
|
+
* `signal` abandons the wait: it rejects with the signal's reason and holds
|
|
207
|
+
* nothing. A wait outlives nothing on its own: bound it with a signal when
|
|
208
|
+
* room may never come (a resident process holds its worker for as long as
|
|
209
|
+
* it runs).
|
|
210
|
+
*
|
|
211
|
+
* const end = await beginLoaderFetchWhenFree(ctx, key, { signal });
|
|
212
|
+
* try { return await worker.getEntrypoint().run(); }
|
|
213
|
+
* catch (error) { end(error); throw error; }
|
|
214
|
+
* finally { end(); }
|
|
215
|
+
*/
|
|
216
|
+
export function beginLoaderFetchWhenFree(ctx, workerKey, options = {}) {
|
|
217
|
+
const { signal } = options;
|
|
218
|
+
return new Promise((resolve, reject) => {
|
|
219
|
+
if (signal?.aborted) {
|
|
220
|
+
reject(signal.reason);
|
|
63
221
|
return;
|
|
64
|
-
|
|
65
|
-
entry
|
|
222
|
+
}
|
|
223
|
+
const entry = ledger(ctx);
|
|
224
|
+
const abandon = () => {
|
|
225
|
+
const at = entry.waiters.indexOf(waiter);
|
|
226
|
+
if (at < 0)
|
|
227
|
+
return;
|
|
228
|
+
entry.waiters.splice(at, 1);
|
|
229
|
+
reject(signal?.reason);
|
|
230
|
+
};
|
|
231
|
+
const waiter = {
|
|
232
|
+
key: workerKey,
|
|
233
|
+
claim: claimOf(ctx, options.claim),
|
|
234
|
+
admit(end) {
|
|
235
|
+
signal?.removeEventListener('abort', abandon);
|
|
236
|
+
resolve(end);
|
|
237
|
+
},
|
|
238
|
+
};
|
|
239
|
+
entry.waiters.push(waiter);
|
|
240
|
+
signal?.addEventListener('abort', abandon, { once: true });
|
|
241
|
+
admitWaiters(entry);
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* Distinct Dynamic Workers this actor may still put in flight: the limit
|
|
246
|
+
* less what is held and claimed right now, and none while a limit refusal's
|
|
247
|
+
* pause lasts. Never negative.
|
|
248
|
+
*/
|
|
249
|
+
export function dynamicWorkerHeadroom(ctx) {
|
|
250
|
+
return headroom(ledger(ctx));
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Claim `width` distinct Dynamic Workers for one fan-out, or null when the
|
|
254
|
+
* headroom cannot hold it. The claim counts until `release` (idempotent), so
|
|
255
|
+
* a second fan-out sizing itself meanwhile sees it; the claimant's own
|
|
256
|
+
* dispatches, held under the claim, count inside it.
|
|
257
|
+
*/
|
|
258
|
+
export function claimDynamicWorkers(ctx, width) {
|
|
259
|
+
const entry = ledger(ctx);
|
|
260
|
+
if (width < 1 || width > headroom(entry))
|
|
261
|
+
return null;
|
|
262
|
+
const claim = { width, keys: new Map() };
|
|
263
|
+
entry.claims.add(claim);
|
|
264
|
+
entry.peak = Math.max(entry.peak, inUse(entry));
|
|
265
|
+
const handle = {
|
|
266
|
+
release() {
|
|
267
|
+
if (!entry.claims.delete(claim))
|
|
268
|
+
return;
|
|
269
|
+
admitWaiters(entry);
|
|
270
|
+
},
|
|
66
271
|
};
|
|
272
|
+
claimEntries.set(handle, { ledger: entry, entry: claim });
|
|
273
|
+
return handle;
|
|
67
274
|
}
|
|
68
275
|
/** Snapshot for the diag surface. Pure read; no I/O. */
|
|
69
276
|
export function loaderLedgerStats(ctx) {
|
|
70
277
|
const entry = ledger(ctx);
|
|
71
278
|
return {
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
279
|
+
limit: DO_DYNAMIC_WORKER_LIMIT,
|
|
280
|
+
inFlightWorkers: [...entry.inFlight.keys()],
|
|
281
|
+
claimed: claimedWidth(entry),
|
|
282
|
+
headroom: headroom(entry),
|
|
283
|
+
peak: entry.peak,
|
|
284
|
+
waiting: entry.waiters.length,
|
|
285
|
+
pauseMs: entry.pauseMs,
|
|
75
286
|
};
|
|
76
287
|
}
|
|
288
|
+
function claimedWidth(entry) {
|
|
289
|
+
let width = 0;
|
|
290
|
+
for (const claim of entry.claims)
|
|
291
|
+
width += claim.width;
|
|
292
|
+
return width;
|
|
293
|
+
}
|
|
77
294
|
/**
|
|
78
|
-
* Name the per-DO accounting on a "
|
|
295
|
+
* Name the per-DO accounting on a "Dynamic worker concurrency limit exceeded"
|
|
79
296
|
* failure; hand every other error back untouched. The platform's message
|
|
80
|
-
* says only that the
|
|
81
|
-
*
|
|
82
|
-
* shrink anything.
|
|
297
|
+
* says only that the limit was hit — which workers were in flight, and what
|
|
298
|
+
* fan-outs had claimed, is what the operator needs to know to shrink anything.
|
|
83
299
|
*/
|
|
84
300
|
export function withDynamicWorkerCapNamed(ctx, error) {
|
|
85
301
|
if (classifyError(error) !== 'dynamic_worker_cap')
|
|
86
302
|
return error;
|
|
87
303
|
const entry = ledger(ctx);
|
|
88
304
|
const platform = error instanceof Error ? error.message : String(error);
|
|
89
|
-
return new Error(`${platform} — this Durable Object
|
|
90
|
-
+ `
|
|
91
|
-
+
|
|
92
|
-
+ `peak ${entry.peakLiveFetches}`, { cause: error });
|
|
305
|
+
return new Error(`${platform} — this Durable Object had ${entry.inFlight.size} distinct dynamic worker(s) in flight `
|
|
306
|
+
+ `(${[...entry.inFlight.keys()].join(', ') || 'none recorded'}) and ${claimedWidth(entry)} claimed by fan-outs, `
|
|
307
|
+
+ `against a limit of ${DO_DYNAMIC_WORKER_LIMIT}; peak ${entry.peak}`, { cause: error });
|
|
93
308
|
}
|
|
94
309
|
// ── Dynamic-worker module-map ceiling ───────────────────────────────────────
|
|
95
310
|
/**
|
|
@@ -136,13 +351,18 @@ export function assertModuleMapWithinCodeLimit(modules) {
|
|
|
136
351
|
}
|
|
137
352
|
/**
|
|
138
353
|
* Bytes one module-map member carries, across the loader's content kinds
|
|
139
|
-
* (plain string, `{ js | cjs | py | text }`, `{ wasm | data }
|
|
140
|
-
* encoder, text is measured exactly; without
|
|
354
|
+
* (plain string, `{ js | cjs | py | text }`, `{ wasm | data }`, a bare
|
|
355
|
+
* WebAssembly.Module). With an encoder, text is measured exactly; without
|
|
356
|
+
* one, by code-unit length. A compiled module counts the wire size its host
|
|
357
|
+
* described (host-wasm.ts); one nobody described counts nothing here and is
|
|
358
|
+
* left to the platform's own refusal, as the text undercount is.
|
|
141
359
|
*/
|
|
142
360
|
function memberBytes(content, encoder) {
|
|
143
361
|
const textBytes = (text) => encoder ? encoder.encode(text).byteLength : text.length;
|
|
144
362
|
if (typeof content === 'string')
|
|
145
363
|
return textBytes(content);
|
|
364
|
+
if (content instanceof WebAssembly.Module)
|
|
365
|
+
return hostWasmIdentity(content)?.bytes ?? 0;
|
|
146
366
|
if (content !== null && typeof content === 'object') {
|
|
147
367
|
for (const value of Object.values(content)) {
|
|
148
368
|
if (typeof value === 'string')
|
|
@@ -151,6 +371,8 @@ function memberBytes(content, encoder) {
|
|
|
151
371
|
return value.byteLength;
|
|
152
372
|
if (ArrayBuffer.isView(value))
|
|
153
373
|
return value.byteLength;
|
|
374
|
+
if (value instanceof WebAssembly.Module)
|
|
375
|
+
return hostWasmIdentity(value)?.bytes ?? 0;
|
|
154
376
|
}
|
|
155
377
|
}
|
|
156
378
|
return 0;
|
package/dist/do-calls.d.ts
CHANGED
|
@@ -40,6 +40,7 @@
|
|
|
40
40
|
* namespace stubs, where the thunk shape is production-proven in Proteus.
|
|
41
41
|
*/
|
|
42
42
|
import { type DoCallClass } from '@nimbus-sh/platform/oom-classify.js';
|
|
43
|
+
import { type SpanRecorder } from '@nimbus-sh/platform/tracing.js';
|
|
43
44
|
export interface DoCallRetryPolicy {
|
|
44
45
|
maxAttempts?: number;
|
|
45
46
|
baseDelayMs?: number;
|
|
@@ -78,7 +79,26 @@ export interface DoCallRetryPolicy {
|
|
|
78
79
|
* its error.
|
|
79
80
|
*/
|
|
80
81
|
onRetry?(info: DoCallRetryInfo): void;
|
|
82
|
+
/**
|
|
83
|
+
* Where the call's telemetry goes: its span's recorder. Each attempt lost
|
|
84
|
+
* to a transient or overloaded failure is recorded as an exception whose
|
|
85
|
+
* `code` is the failure's class, and once the call has settled it gets
|
|
86
|
+
* `do_call.attempts` (started), `do_call.hedges` (started by a hedge),
|
|
87
|
+
* `do_call.answered_by` (the attempt whose answer the call took, absent
|
|
88
|
+
* when none answered) and `do_call.outcome` ({@link DoCallOutcome}).
|
|
89
|
+
* Nothing recorded can change the call's answer. Records nothing when
|
|
90
|
+
* absent.
|
|
91
|
+
*/
|
|
92
|
+
span?: SpanRecorder;
|
|
81
93
|
}
|
|
94
|
+
/**
|
|
95
|
+
* How an `idempotent` call ended: `answered` (an attempt succeeded),
|
|
96
|
+
* `callee_error` (the callee's own failure, which is an answer),
|
|
97
|
+
* `exhausted` (the last transient failure, no repeat left), `overloaded`
|
|
98
|
+
* (shed, nothing repeated after it), or `caller_error` (the resolver or
|
|
99
|
+
* `onRetry` threw).
|
|
100
|
+
*/
|
|
101
|
+
export type DoCallOutcome = 'answered' | 'callee_error' | 'exhausted' | 'overloaded' | 'caller_error';
|
|
82
102
|
/** What one retry is answering: which call, which platform class, which
|
|
83
103
|
* attempt just failed out of how many. */
|
|
84
104
|
export interface DoCallRetryInfo {
|
package/dist/do-calls.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"do-calls.d.ts","sourceRoot":"","sources":["../src/do-calls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AAEH,OAAO,EAAqC,KAAK,WAAW,EAAE,MAAM,qCAAqC,CAAC;
|
|
1
|
+
{"version":3,"file":"do-calls.d.ts","sourceRoot":"","sources":["../src/do-calls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AAEH,OAAO,EAAqC,KAAK,WAAW,EAAE,MAAM,qCAAqC,CAAC;AAE1G,OAAO,EAAY,KAAK,YAAY,EAAE,MAAM,gCAAgC,CAAC;AAS7E,MAAM,WAAW,iBAAiB;IAChC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;;;;;;;;OAeG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;;;;OAQG;IACH,OAAO,CAAC,CAAC,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;IACtC;;;;;;;;;OASG;IACH,IAAI,CAAC,EAAE,YAAY,CAAC;CACrB;AAED;;;;;;GAMG;AACH,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,cAAc,GAAG,WAAW,GAAG,YAAY,GAAG,cAAc,CAAC;AAEtG;2CAC2C;AAC3C,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,WAAW,CAAC;IAC5B;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,IAAI,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;AAErD;;;;;GAKG;AACH,qBAAa,WAAY,SAAQ,KAAK;IAElC,QAAQ,CAAC,SAAS,EAAE,MAAM;IAC1B,QAAQ,CAAC,IAAI,EAAE,YAAY,GAAG,UAAU;IACxC,QAAQ,CAAC,cAAc,EAAE,WAAW;gBAF3B,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,YAAY,GAAG,UAAU,EAC/B,cAAc,EAAE,WAAW,EACpC,KAAK,EAAE,OAAO;CASjB;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,UAAU,CAAC,CAAC,EAAE,CAAC,EAC7B,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,EACvB,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,EAC7B,MAAM,GAAE,iBAAsB,GAC7B,OAAO,CAAC,CAAC,CAAC,CAgIZ;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,QAAQ,CAAC,CAAC,EAAE,CAAC,EACjC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,EACvB,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,GAC5B,OAAO,CAAC,CAAC,CAAC,CAUZ"}
|