@nimbus-sh/fabric 0.7.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -11
- package/dist/bindings.d.ts +1 -0
- package/dist/bindings.d.ts.map +1 -1
- package/dist/bindings.js +2 -0
- package/dist/budgets.d.ts +50 -29
- package/dist/budgets.d.ts.map +1 -1
- package/dist/budgets.js +88 -40
- package/dist/connections.d.ts +1 -1
- package/dist/connections.js +1 -1
- package/dist/do-calls.d.ts +91 -9
- package/dist/do-calls.d.ts.map +1 -1
- package/dist/do-calls.js +161 -22
- package/dist/facet-pool.d.ts +3 -0
- package/dist/facet-pool.d.ts.map +1 -1
- package/dist/facet-pool.js +3 -0
- package/dist/fanout.d.ts +40 -32
- package/dist/fanout.d.ts.map +1 -1
- package/dist/fanout.js +50 -53
- package/dist/fenced-work.d.ts +3 -3
- package/dist/host-wasm.d.ts +29 -0
- package/dist/host-wasm.d.ts.map +1 -0
- package/dist/host-wasm.js +31 -0
- package/dist/image-store.d.ts +1 -1
- package/dist/image-store.d.ts.map +1 -1
- package/dist/image-store.js +34 -2
- package/dist/inner-do-registry.d.ts +9 -0
- package/dist/inner-do-registry.d.ts.map +1 -1
- package/dist/inner-do-registry.js +35 -0
- package/dist/isolate-pool.d.ts +32 -24
- package/dist/isolate-pool.d.ts.map +1 -1
- package/dist/isolate-pool.js +78 -54
- package/dist/process-fabric.d.ts +42 -23
- package/dist/process-fabric.d.ts.map +1 -1
- package/dist/process-fabric.js +62 -6
- package/dist/process-host.d.ts +3 -1
- package/dist/process-host.d.ts.map +1 -1
- package/dist/process-host.js +13 -14
- package/dist/supervisor-props.d.ts +47 -0
- package/dist/supervisor-props.d.ts.map +1 -0
- package/dist/supervisor-props.js +37 -0
- package/dist/timers.d.ts +12 -0
- package/dist/timers.d.ts.map +1 -1
- package/dist/timers.js +44 -9
- package/dist/vendor/types.d.ts +11 -5
- package/dist/vendor/types.d.ts.map +1 -1
- package/dist/workerd-facet-host.d.ts +4 -17
- package/dist/workerd-facet-host.d.ts.map +1 -1
- package/dist/workerd-facet-host.js +120 -54
- package/package.json +6 -6
- package/src/bindings.ts +2 -0
- package/src/budgets.ts +96 -49
- package/src/connections.ts +1 -1
- package/src/do-calls.ts +216 -25
- package/src/facet-pool.ts +5 -0
- package/src/fanout.ts +64 -56
- package/src/fenced-work.ts +3 -3
- package/src/host-wasm.ts +41 -0
- package/src/image-store.ts +30 -3
- package/src/inner-do-registry.ts +31 -0
- package/src/isolate-pool.ts +108 -74
- package/src/process-fabric.ts +88 -30
- package/src/process-host.ts +16 -14
- package/src/supervisor-props.ts +56 -0
- package/src/timers.ts +45 -9
- package/src/vendor/types.ts +11 -5
- package/src/workerd-facet-host.ts +123 -58
package/src/budgets.ts
CHANGED
|
@@ -1,39 +1,46 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* budgets.ts — per-DO accounting for the platform budgets the fabric spends:
|
|
3
|
-
* the
|
|
4
|
-
* dynamic-worker module-map ceiling.
|
|
3
|
+
* the Durable Object's Dynamic Worker concurrency limit, the facet-ID
|
|
4
|
+
* lifetime budget, and the dynamic-worker module-map ceiling.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
* (`IN_DO_THRESHOLD` = 5 in the fanout pool), which until now meant the slots
|
|
13
|
-
* were counted in prose. This ledger counts them at the fabric's loader call
|
|
14
|
-
* sites instead — the loader pool's slots, a resident process's keyed worker,
|
|
15
|
-
* a one-shot's load — so proximity is measurable and a cap failure can name
|
|
16
|
-
* the ids actually holding slots.
|
|
6
|
+
* The Dynamic Worker model is Cloudflare's documented one
|
|
7
|
+
* ({@link DO_DYNAMIC_WORKER_LIMIT}): a Durable Object may have a fixed number
|
|
8
|
+
* of DISTINCT Dynamic Workers with in-flight requests at once, shared across
|
|
9
|
+
* every concurrent request to that object (one I/O context), and any number
|
|
10
|
+
* of in-flight requests to the same Dynamic Worker count as one. Only
|
|
11
|
+
* in-flight requests count: a loader id with nothing in flight holds nothing.
|
|
17
12
|
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
13
|
+
* The ledger counts, per hosting actor, the distinct workers that are in
|
|
14
|
+
* flight right now, keyed by loader id (a fresh key per unkeyed `load`), plus
|
|
15
|
+
* the width fan-outs have claimed and not yet released. A fan-out spends only
|
|
16
|
+
* the {@link dynamicWorkerHeadroom} that leaves, so work a Durable Object
|
|
17
|
+
* already has in flight — a resident process, the esbuild facet, a git
|
|
18
|
+
* network op, another fan-out — keeps its slots.
|
|
21
19
|
*
|
|
22
20
|
* Keyed weakly off the hosting actor's `ctx`, like the facet slot books: the
|
|
23
|
-
*
|
|
21
|
+
* limit is per Durable Object, and dynamic workers die with the isolate that
|
|
24
22
|
* loaded them, so a ledger that goes away with its host describes nothing
|
|
25
23
|
* that still exists.
|
|
26
24
|
*/
|
|
27
25
|
|
|
28
26
|
import { classifyError } from '@nimbus-sh/platform/oom-classify.js';
|
|
27
|
+
import { hostWasmIdentity } from './host-wasm.js';
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Distinct Dynamic Workers one Durable Object may have with in-flight
|
|
31
|
+
* requests at once, shared across all concurrent requests to that object;
|
|
32
|
+
* multiple in-flight requests to one Dynamic Worker count once.
|
|
33
|
+
* https://developers.cloudflare.com/changelog/post/2026-08-28-durable-objects-dynamic-workers-limit/
|
|
34
|
+
*/
|
|
35
|
+
export const DO_DYNAMIC_WORKER_LIMIT = 10;
|
|
29
36
|
|
|
30
37
|
interface LoaderLedger {
|
|
31
|
-
/**
|
|
32
|
-
|
|
33
|
-
/**
|
|
34
|
-
|
|
35
|
-
/** The most
|
|
36
|
-
|
|
38
|
+
/** Loader id → open holds on it. A key is present only while held. */
|
|
39
|
+
inFlight: Map<string, number>;
|
|
40
|
+
/** Width claimed by fan-outs that have not released it. */
|
|
41
|
+
claimed: number;
|
|
42
|
+
/** The most distinct workers (holds plus claims) ever counted at once. */
|
|
43
|
+
peak: number;
|
|
37
44
|
}
|
|
38
45
|
|
|
39
46
|
const ledgers = new WeakMap<object, LoaderLedger>();
|
|
@@ -41,20 +48,21 @@ const ledgers = new WeakMap<object, LoaderLedger>();
|
|
|
41
48
|
function ledger(ctx: object): LoaderLedger {
|
|
42
49
|
let entry = ledgers.get(ctx);
|
|
43
50
|
if (!entry) {
|
|
44
|
-
entry = {
|
|
51
|
+
entry = { inFlight: new Map(), claimed: 0, peak: 0 };
|
|
45
52
|
ledgers.set(ctx, entry);
|
|
46
53
|
}
|
|
47
54
|
return entry;
|
|
48
55
|
}
|
|
49
56
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
ledger(ctx).ids.add(id);
|
|
57
|
+
function inUse(entry: LoaderLedger): number {
|
|
58
|
+
return entry.inFlight.size + entry.claimed;
|
|
53
59
|
}
|
|
54
60
|
|
|
55
61
|
/**
|
|
56
|
-
*
|
|
57
|
-
* function ends
|
|
62
|
+
* Hold the Dynamic Worker `workerKey` in flight on this actor's ledger; the
|
|
63
|
+
* returned function ends the hold (idempotently), from the caller's own
|
|
64
|
+
* `finally`. Holds on one key nest: the worker counts once until the last
|
|
65
|
+
* one ends, as the platform counts it.
|
|
58
66
|
*
|
|
59
67
|
* A begin/end pair rather than a wrapper on purpose, and the shape is
|
|
60
68
|
* load-bearing: wrapping the stub call in a ledger-owned async frame
|
|
@@ -67,48 +75,82 @@ export function recordLoaderId(ctx: object, id: string): void {
|
|
|
67
75
|
* workers: an RPC stub call must stay a direct property call awaited by the
|
|
68
76
|
* frame that made it, so the ledger only brackets it.
|
|
69
77
|
*/
|
|
70
|
-
export function beginLoaderFetch(ctx: object): () => void {
|
|
78
|
+
export function beginLoaderFetch(ctx: object, workerKey: string): () => void {
|
|
71
79
|
const entry = ledger(ctx);
|
|
72
|
-
entry.
|
|
73
|
-
entry.
|
|
80
|
+
entry.inFlight.set(workerKey, (entry.inFlight.get(workerKey) ?? 0) + 1);
|
|
81
|
+
entry.peak = Math.max(entry.peak, inUse(entry));
|
|
74
82
|
let ended = false;
|
|
75
83
|
return () => {
|
|
76
84
|
if (ended) return;
|
|
77
85
|
ended = true;
|
|
78
|
-
entry.
|
|
86
|
+
const open = (entry.inFlight.get(workerKey) ?? 1) - 1;
|
|
87
|
+
if (open > 0) entry.inFlight.set(workerKey, open);
|
|
88
|
+
else entry.inFlight.delete(workerKey);
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Distinct Dynamic Workers this actor may still put in flight: the limit
|
|
94
|
+
* less what is held and claimed right now. Never negative.
|
|
95
|
+
*/
|
|
96
|
+
export function dynamicWorkerHeadroom(ctx: object): number {
|
|
97
|
+
return Math.max(0, DO_DYNAMIC_WORKER_LIMIT - inUse(ledger(ctx)));
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Claim `width` distinct Dynamic Workers for one fan-out, or null when the
|
|
102
|
+
* headroom cannot hold it. The claim counts until `release` (idempotent), so
|
|
103
|
+
* a second fan-out sizing itself meanwhile sees it; the claimant's own
|
|
104
|
+
* dispatches are held as well while they run, which only ever over-counts
|
|
105
|
+
* toward sending that second fan-out elsewhere.
|
|
106
|
+
*/
|
|
107
|
+
export function claimDynamicWorkers(ctx: object, width: number): { release(): void } | null {
|
|
108
|
+
const entry = ledger(ctx);
|
|
109
|
+
if (width < 1 || width > DO_DYNAMIC_WORKER_LIMIT - inUse(entry)) return null;
|
|
110
|
+
entry.claimed += width;
|
|
111
|
+
entry.peak = Math.max(entry.peak, inUse(entry));
|
|
112
|
+
let released = false;
|
|
113
|
+
return {
|
|
114
|
+
release() {
|
|
115
|
+
if (released) return;
|
|
116
|
+
released = true;
|
|
117
|
+
entry.claimed -= width;
|
|
118
|
+
},
|
|
79
119
|
};
|
|
80
120
|
}
|
|
81
121
|
|
|
82
122
|
/** Snapshot for the diag surface. Pure read; no I/O. */
|
|
83
123
|
export function loaderLedgerStats(ctx: object): {
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
124
|
+
limit: number;
|
|
125
|
+
inFlightWorkers: string[];
|
|
126
|
+
claimed: number;
|
|
127
|
+
headroom: number;
|
|
128
|
+
peak: number;
|
|
87
129
|
} {
|
|
88
130
|
const entry = ledger(ctx);
|
|
89
131
|
return {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
132
|
+
limit: DO_DYNAMIC_WORKER_LIMIT,
|
|
133
|
+
inFlightWorkers: [...entry.inFlight.keys()],
|
|
134
|
+
claimed: entry.claimed,
|
|
135
|
+
headroom: dynamicWorkerHeadroom(ctx),
|
|
136
|
+
peak: entry.peak,
|
|
93
137
|
};
|
|
94
138
|
}
|
|
95
139
|
|
|
96
140
|
/**
|
|
97
|
-
* Name the per-DO accounting on a "
|
|
141
|
+
* Name the per-DO accounting on a "Dynamic worker concurrency limit exceeded"
|
|
98
142
|
* failure; hand every other error back untouched. The platform's message
|
|
99
|
-
* says only that the
|
|
100
|
-
*
|
|
101
|
-
* shrink anything.
|
|
143
|
+
* says only that the limit was hit — which workers were in flight, and what
|
|
144
|
+
* fan-outs had claimed, is what the operator needs to know to shrink anything.
|
|
102
145
|
*/
|
|
103
146
|
export function withDynamicWorkerCapNamed<E>(ctx: object, error: E): E | Error {
|
|
104
147
|
if (classifyError(error) !== 'dynamic_worker_cap') return error;
|
|
105
148
|
const entry = ledger(ctx);
|
|
106
149
|
const platform = error instanceof Error ? error.message : String(error);
|
|
107
150
|
return new Error(
|
|
108
|
-
`${platform} — this Durable Object
|
|
109
|
-
+ `
|
|
110
|
-
+
|
|
111
|
-
+ `peak ${entry.peakLiveFetches}`,
|
|
151
|
+
`${platform} — this Durable Object had ${entry.inFlight.size} distinct dynamic worker(s) in flight `
|
|
152
|
+
+ `(${[...entry.inFlight.keys()].join(', ') || 'none recorded'}) and ${entry.claimed} claimed by fan-outs, `
|
|
153
|
+
+ `against a limit of ${DO_DYNAMIC_WORKER_LIMIT}; peak ${entry.peak}`,
|
|
112
154
|
{ cause: error },
|
|
113
155
|
);
|
|
114
156
|
}
|
|
@@ -163,18 +205,23 @@ export function assertModuleMapWithinCodeLimit(modules: Record<string, unknown>)
|
|
|
163
205
|
|
|
164
206
|
/**
|
|
165
207
|
* Bytes one module-map member carries, across the loader's content kinds
|
|
166
|
-
* (plain string, `{ js | cjs | py | text }`, `{ wasm | data }
|
|
167
|
-
* encoder, text is measured exactly; without
|
|
208
|
+
* (plain string, `{ js | cjs | py | text }`, `{ wasm | data }`, a bare
|
|
209
|
+
* WebAssembly.Module). With an encoder, text is measured exactly; without
|
|
210
|
+
* one, by code-unit length. A compiled module counts the wire size its host
|
|
211
|
+
* described (host-wasm.ts); one nobody described counts nothing here and is
|
|
212
|
+
* left to the platform's own refusal, as the text undercount is.
|
|
168
213
|
*/
|
|
169
214
|
function memberBytes(content: unknown, encoder: TextEncoder | null): number {
|
|
170
215
|
const textBytes = (text: string): number =>
|
|
171
216
|
encoder ? encoder.encode(text).byteLength : text.length;
|
|
172
217
|
if (typeof content === 'string') return textBytes(content);
|
|
218
|
+
if (content instanceof WebAssembly.Module) return hostWasmIdentity(content)?.bytes ?? 0;
|
|
173
219
|
if (content !== null && typeof content === 'object') {
|
|
174
220
|
for (const value of Object.values(content)) {
|
|
175
221
|
if (typeof value === 'string') return textBytes(value);
|
|
176
222
|
if (value instanceof ArrayBuffer) return value.byteLength;
|
|
177
223
|
if (ArrayBuffer.isView(value)) return value.byteLength;
|
|
224
|
+
if (value instanceof WebAssembly.Module) return hostWasmIdentity(value)?.bytes ?? 0;
|
|
178
225
|
}
|
|
179
226
|
}
|
|
180
227
|
return 0;
|
package/src/connections.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* state over the WebSocket attachment.
|
|
4
4
|
*
|
|
5
5
|
* Specified from Proteus's DeviceSocketHub (`cf-backend/src/user/device-hub.ts`)
|
|
6
|
-
* and CLI rpc gate (`cf-backend/src/cli/rpc-gate.ts`), which split the
|
|
6
|
+
* and its original CLI rpc gate (`cf-backend/src/cli/rpc-gate.ts`), which split the
|
|
7
7
|
* pattern into its two halves:
|
|
8
8
|
* - a TAG is the immutable-at-accept lookup key and authorization — it
|
|
9
9
|
* rides the hibernation state, which is why the rpc gate persists auth
|
package/src/do-calls.ts
CHANGED
|
@@ -3,12 +3,14 @@
|
|
|
3
3
|
* the one property that decides whether a retry is safe.
|
|
4
4
|
*
|
|
5
5
|
* Both consumers asked for this. Proteus hand-wrote the retry
|
|
6
|
-
* (`cf-backend/src/lib/do-rpc.ts`) with the rule its header states:
|
|
6
|
+
* (originally `cf-backend/src/lib/do-rpc.ts`) with the rule its header states:
|
|
7
7
|
* "An operation that appends, sends, charges or mints is never wrapped: a
|
|
8
8
|
* dropped call there may already have run, so a retry is a correctness bug
|
|
9
9
|
* wearing resilience as a costume." agent-core has no retry machinery at all
|
|
10
10
|
* and its backlog calls the gap "the most production-proven gap in the
|
|
11
11
|
* corpus". Here the rule is a type: `idempotent` retries, `mutating` cannot.
|
|
12
|
+
* A mutation earns a retry only by carrying an identity its callee applies
|
|
13
|
+
* at most once; it is then `idempotent` by construction (see `mutating`).
|
|
12
14
|
*
|
|
13
15
|
* What the platform contract requires, and this keeps:
|
|
14
16
|
* - a FRESH stub per attempt. Cloudflare documents that many exceptions
|
|
@@ -20,6 +22,11 @@
|
|
|
20
22
|
* object is what overloaded it.
|
|
21
23
|
* - attempts and backoff are the consumer-proven bounds: 3 attempts total,
|
|
22
24
|
* full-jitter delays in [0, 2**attempt * 60ms).
|
|
25
|
+
* - an `idempotent` call may also be HEDGED (`hedgeAfterMs`): an attempt
|
|
26
|
+
* that has not answered by then is joined by the same call on a fresh
|
|
27
|
+
* stub, both left running, the first success taken. Hedges count
|
|
28
|
+
* against the attempts, and a callee that joins a repeat to the call it
|
|
29
|
+
* is already serving makes one that did arrive cost nothing.
|
|
23
30
|
*
|
|
24
31
|
* The resolver MINTS a stub per call and the verb disposes each one it
|
|
25
32
|
* minted — that ownership is what makes the fresh-stub retry real.
|
|
@@ -35,6 +42,7 @@
|
|
|
35
42
|
|
|
36
43
|
import { classifyDoCall, isRetryableDoCall, type DoCallClass } from '@nimbus-sh/platform/oom-classify.js';
|
|
37
44
|
import { disposeRpcResource } from '@nimbus-sh/platform/rpc-dispose.js';
|
|
45
|
+
import { untraced, type SpanRecorder } from '@nimbus-sh/platform/tracing.js';
|
|
38
46
|
|
|
39
47
|
/** Total attempts. Two retries is what a dropped connection or a deploy
|
|
40
48
|
* bounce needs; beyond that the object is not coming back inside this
|
|
@@ -47,20 +55,71 @@ export interface DoCallRetryPolicy {
|
|
|
47
55
|
maxAttempts?: number;
|
|
48
56
|
baseDelayMs?: number;
|
|
49
57
|
/**
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
58
|
+
* No repeat — retry or hedge — starts once this long has passed since the
|
|
59
|
+
* first attempt did; the failure in hand surfaces instead. A mutation made
|
|
60
|
+
* repeatable by an identity its callee dedupes needs it: the callee keeps
|
|
61
|
+
* what answers a repeat for a bounded time, so the caller's repeats must
|
|
62
|
+
* stop well inside it. Unbounded when absent.
|
|
63
|
+
*/
|
|
64
|
+
retryWindowMs?: number;
|
|
65
|
+
/**
|
|
66
|
+
* Hedge an attempt that has not answered after this long: send the same
|
|
67
|
+
* call again on a fresh stub while the first stays in flight. The caller
|
|
68
|
+
* gets the first success; an answer after it is disposed and dropped. That
|
|
69
|
+
* is only harmless when a second delivery of the call changes nothing — a
|
|
70
|
+
* read — and cheap only when the callee joins a repeat to the call it is
|
|
71
|
+
* already serving. A hedge is an attempt: it counts against `maxAttempts`,
|
|
72
|
+
* and has its own deadline. Never hedged when absent.
|
|
73
|
+
*
|
|
74
|
+
* With attempts overlapping, a failure decides less. A retryable one is
|
|
75
|
+
* retried after its backoff while attempts remain, and `overloaded` stops
|
|
76
|
+
* every further repeat; either way the call keeps waiting on the attempts
|
|
77
|
+
* still in flight, and fails with the last failure only once none is. Any
|
|
78
|
+
* other failure is the callee's answer — the call ran — and is the call's
|
|
79
|
+
* at once.
|
|
80
|
+
*/
|
|
81
|
+
hedgeAfterMs?: number;
|
|
82
|
+
/**
|
|
83
|
+
* Called once per retry, as it starts — after its backoff delay — with the
|
|
84
|
+
* failure the retry is answering. A retry a hedge made unnecessary, or
|
|
85
|
+
* that no attempt is left for, is never announced. The consumer's logging
|
|
86
|
+
* seam: Proteus's hand-rolled predecessor logged every retry so a flaky
|
|
87
|
+
* object is visible in Workers Logs rather than silently absorbed, and
|
|
88
|
+
* `operation` names it there. A callback that throws fails the call with
|
|
89
|
+
* its error.
|
|
54
90
|
*/
|
|
55
91
|
onRetry?(info: DoCallRetryInfo): void;
|
|
92
|
+
/**
|
|
93
|
+
* Where the call's telemetry goes: its span's recorder. Each attempt lost
|
|
94
|
+
* to a transient or overloaded failure is recorded as an exception whose
|
|
95
|
+
* `code` is the failure's class, and once the call has settled it gets
|
|
96
|
+
* `do_call.attempts` (started), `do_call.hedges` (started by a hedge),
|
|
97
|
+
* `do_call.answered_by` (the attempt whose answer the call took, absent
|
|
98
|
+
* when none answered) and `do_call.outcome` ({@link DoCallOutcome}).
|
|
99
|
+
* Nothing recorded can change the call's answer. Records nothing when
|
|
100
|
+
* absent.
|
|
101
|
+
*/
|
|
102
|
+
span?: SpanRecorder;
|
|
56
103
|
}
|
|
57
104
|
|
|
105
|
+
/**
|
|
106
|
+
* How an `idempotent` call ended: `answered` (an attempt succeeded),
|
|
107
|
+
* `callee_error` (the callee's own failure, which is an answer),
|
|
108
|
+
* `exhausted` (the last transient failure, no repeat left), `overloaded`
|
|
109
|
+
* (shed, nothing repeated after it), or `caller_error` (the resolver or
|
|
110
|
+
* `onRetry` threw).
|
|
111
|
+
*/
|
|
112
|
+
export type DoCallOutcome = 'answered' | 'callee_error' | 'exhausted' | 'overloaded' | 'caller_error';
|
|
113
|
+
|
|
58
114
|
/** What one retry is answering: which call, which platform class, which
|
|
59
115
|
* attempt just failed out of how many. */
|
|
60
116
|
export interface DoCallRetryInfo {
|
|
61
117
|
operation: string;
|
|
62
118
|
classification: DoCallClass;
|
|
63
|
-
/**
|
|
119
|
+
/**
|
|
120
|
+
* The 1-based number of the attempt that failed. Without hedging the retry
|
|
121
|
+
* is attempt+1; with it, other attempts may have started in between.
|
|
122
|
+
*/
|
|
64
123
|
attempt: number;
|
|
65
124
|
maxAttempts: number;
|
|
66
125
|
error: unknown;
|
|
@@ -102,11 +161,20 @@ export class DoCallError extends Error {
|
|
|
102
161
|
|
|
103
162
|
/**
|
|
104
163
|
* Call another Durable Object with an operation that is safe to repeat: a
|
|
105
|
-
* read,
|
|
106
|
-
*
|
|
107
|
-
*
|
|
164
|
+
* read, a converge-to-a-value write, or a mutation carrying an identity its
|
|
165
|
+
* callee applies at most once (see {@link mutating}). Transient failures
|
|
166
|
+
* retry on a fresh stub with full-jitter backoff; overloaded and permanent
|
|
167
|
+
* failures surface unchanged, as does the last error once no attempt may be
|
|
168
|
+
* repeated — attempts spent, or the policy's retry window closed. With
|
|
169
|
+
* `hedgeAfterMs`, an attempt still unanswered by then is joined by another
|
|
170
|
+
* on a fresh stub, and the first answer is taken: a success, or the
|
|
171
|
+
* callee's own error. A transient or overloaded failure then ends the call
|
|
172
|
+
* only once no attempt is left in flight.
|
|
173
|
+
*
|
|
174
|
+
* A failure of the resolver or of `onRetry` is the caller's own, and fails
|
|
175
|
+
* the call with it at once.
|
|
108
176
|
*/
|
|
109
|
-
export
|
|
177
|
+
export function idempotent<S, T>(
|
|
110
178
|
operation: string,
|
|
111
179
|
stub: DoStubResolver<S>,
|
|
112
180
|
call: (stub: S) => Promise<T>,
|
|
@@ -114,23 +182,131 @@ export async function idempotent<S, T>(
|
|
|
114
182
|
): Promise<T> {
|
|
115
183
|
const maxAttempts = policy.maxAttempts ?? MAX_ATTEMPTS;
|
|
116
184
|
const baseDelayMs = policy.baseDelayMs ?? BASE_DELAY_MS;
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
185
|
+
const { hedgeAfterMs, retryWindowMs, span = untraced } = policy;
|
|
186
|
+
const startedAt = Date.now();
|
|
187
|
+
// The executor form: fabric's library target predates Promise.withResolvers.
|
|
188
|
+
return new Promise<T>((resolve, reject) => {
|
|
189
|
+
const hedges = new Set<ReturnType<typeof setTimeout>>();
|
|
190
|
+
let started = 0;
|
|
191
|
+
// Attempts in flight, or backing off before their retry: while one is,
|
|
192
|
+
// a failure is not the call's answer.
|
|
193
|
+
let live = 0;
|
|
194
|
+
// An attempt was shed as overloaded: nothing is repeated after it.
|
|
195
|
+
let refused = false;
|
|
196
|
+
let settled = false;
|
|
197
|
+
// Attempts a hedge started rather than a retry or the first send.
|
|
198
|
+
let hedged = 0;
|
|
199
|
+
|
|
200
|
+
const settle = (outcome: DoCallOutcome, answeredBy: number | undefined, answer: () => void): void => {
|
|
201
|
+
if (settled) return;
|
|
202
|
+
settled = true;
|
|
203
|
+
for (const timer of hedges) clearTimeout(timer);
|
|
204
|
+
hedges.clear();
|
|
205
|
+
answer();
|
|
206
|
+
span.set({
|
|
207
|
+
'do_call.attempts': started,
|
|
208
|
+
'do_call.hedges': hedged,
|
|
209
|
+
'do_call.answered_by': answeredBy,
|
|
210
|
+
'do_call.outcome': outcome,
|
|
211
|
+
});
|
|
212
|
+
};
|
|
213
|
+
/** May another attempt start at `at`? */
|
|
214
|
+
const canRepeat = (at: number): boolean =>
|
|
215
|
+
!settled && !refused && started < maxAttempts
|
|
216
|
+
&& (retryWindowMs === undefined || at - startedAt <= retryWindowMs);
|
|
217
|
+
/** `error` ended an attempt that will not be repeated: the call's answer, once nothing else is live. */
|
|
218
|
+
const exhausted = <E>(error: E): void => {
|
|
219
|
+
if (live === 0) settle(refused ? 'overloaded' : 'exhausted', undefined, () => reject(error));
|
|
220
|
+
};
|
|
221
|
+
|
|
222
|
+
/** A failed attempt, numbered: repeat it after its backoff, or let it stand. */
|
|
223
|
+
const failed = async <E>(number: number, error: E): Promise<void> => {
|
|
224
|
+
if (settled) return;
|
|
126
225
|
const classification = classifyDoCall(error);
|
|
127
|
-
if (!isRetryableDoCall(classification)
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
226
|
+
if (!isRetryableDoCall(classification) && classification !== 'overloaded') {
|
|
227
|
+
// The call ran and its answer is this error — ENOENT is a read's answer as much as bytes are.
|
|
228
|
+
settle('callee_error', number, () => reject(error));
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
// A lost attempt: the call's span says which, and why.
|
|
232
|
+
span.exception(error, classification, `attempt ${number} of ${maxAttempts}: `);
|
|
233
|
+
if (classification === 'overloaded') {
|
|
234
|
+
// A shed call is no answer: nothing more is sent, and an attempt
|
|
235
|
+
// still in flight may yet answer.
|
|
236
|
+
refused = true;
|
|
237
|
+
exhausted(error);
|
|
238
|
+
return;
|
|
239
|
+
}
|
|
240
|
+
const delayMs = Math.floor(Math.random() * 2 ** number * baseDelayMs);
|
|
241
|
+
if (!canRepeat(Date.now() + delayMs)) {
|
|
242
|
+
exhausted(error);
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
live++;
|
|
246
|
+
await new Promise<void>((wake) => {
|
|
247
|
+
setTimeout(wake, delayMs);
|
|
131
248
|
});
|
|
132
|
-
|
|
133
|
-
|
|
249
|
+
live--;
|
|
250
|
+
// A hedge may have taken the last attempt, or answered, meanwhile.
|
|
251
|
+
if (!canRepeat(Date.now())) {
|
|
252
|
+
exhausted(error);
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
policy.onRetry?.({ operation, classification, attempt: number, maxAttempts, error });
|
|
256
|
+
attempt();
|
|
257
|
+
};
|
|
258
|
+
|
|
259
|
+
const run = async (): Promise<void> => {
|
|
260
|
+
const number = ++started;
|
|
261
|
+
live++;
|
|
262
|
+
const minted = await stub();
|
|
263
|
+
if (settled) {
|
|
264
|
+
live--;
|
|
265
|
+
disposeRpcResource(minted);
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
268
|
+
const hedge = hedgeAfterMs === undefined ? undefined : setTimeout(() => {
|
|
269
|
+
if (hedge !== undefined) hedges.delete(hedge);
|
|
270
|
+
if (canRepeat(Date.now())) {
|
|
271
|
+
hedged++;
|
|
272
|
+
attempt();
|
|
273
|
+
}
|
|
274
|
+
}, hedgeAfterMs);
|
|
275
|
+
if (hedge !== undefined) hedges.add(hedge);
|
|
276
|
+
/** This attempt has its answer: it hedges no more, and its stub goes. */
|
|
277
|
+
const answered = (): void => {
|
|
278
|
+
if (hedge !== undefined) {
|
|
279
|
+
clearTimeout(hedge);
|
|
280
|
+
hedges.delete(hedge);
|
|
281
|
+
}
|
|
282
|
+
live--;
|
|
283
|
+
// A stub that threw may be permanently broken; none is ever reused.
|
|
284
|
+
disposeRpcResource(minted);
|
|
285
|
+
};
|
|
286
|
+
let result: T;
|
|
287
|
+
try {
|
|
288
|
+
result = await call(minted);
|
|
289
|
+
} catch (error) {
|
|
290
|
+
answered();
|
|
291
|
+
await failed(number, error);
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
294
|
+
answered();
|
|
295
|
+
if (settled) {
|
|
296
|
+
// Another attempt answered first: this answer is dropped, so nothing of it is kept.
|
|
297
|
+
disposeRpcResource(result);
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
settle('answered', number, () => resolve(result));
|
|
301
|
+
};
|
|
302
|
+
|
|
303
|
+
/** Start an attempt. Whatever it throws outside the call itself fails the call. */
|
|
304
|
+
const attempt = (): void => {
|
|
305
|
+
run().catch((error) => settle('caller_error', undefined, () => reject(error)));
|
|
306
|
+
};
|
|
307
|
+
|
|
308
|
+
attempt();
|
|
309
|
+
});
|
|
134
310
|
}
|
|
135
311
|
|
|
136
312
|
/**
|
|
@@ -138,6 +314,21 @@ export async function idempotent<S, T>(
|
|
|
138
314
|
* or mints. NEVER retried — a dropped call may already have run. Failure
|
|
139
315
|
* surfaces as a {@link DoCallError} carrying the classification, so the
|
|
140
316
|
* caller can tell a refusal from an indeterminate drop.
|
|
317
|
+
*
|
|
318
|
+
* The rule is about the call as sent, not the operation's kind. A mutation
|
|
319
|
+
* the callee applies at most once per identity the call carries is
|
|
320
|
+
* repeatable by construction: a repeat of one that already ran is answered
|
|
321
|
+
* from the callee's record and applies nothing. Nimbus has two:
|
|
322
|
+
* - delivered filesystem mutations (@nimbus-sh/core supervisor-delivery):
|
|
323
|
+
* a delivery id plus the callee INSTANCE's incarnation. The record lives
|
|
324
|
+
* in that instance's memory, and any other instance — or a callee that
|
|
325
|
+
* predates delivery — refuses the call permanently rather than apply it
|
|
326
|
+
* without one;
|
|
327
|
+
* - appends: writer, module incarnation and operation sequence, recorded
|
|
328
|
+
* durably until acknowledged.
|
|
329
|
+
* Such a call goes through {@link idempotent}, re-sending the same identity
|
|
330
|
+
* on every attempt, with a `retryWindowMs` inside the callee's retention of
|
|
331
|
+
* that record. Without such an identity, a mutation stays here.
|
|
141
332
|
*/
|
|
142
333
|
export async function mutating<S, T>(
|
|
143
334
|
operation: string,
|
package/src/facet-pool.ts
CHANGED
|
@@ -42,6 +42,8 @@
|
|
|
42
42
|
* and the facet-id ledger here counts only the names this pool minted.
|
|
43
43
|
*/
|
|
44
44
|
|
|
45
|
+
import { forgetFacetStorage } from '@nimbus-sh/core/runtime/storage-ledger.js';
|
|
46
|
+
import type { SqlDatabase } from '@nimbus-sh/core/runtime/os-contracts.js';
|
|
45
47
|
import {
|
|
46
48
|
FACET_ID_LIFETIME_BUDGET,
|
|
47
49
|
facetNameCount,
|
|
@@ -63,6 +65,8 @@ export interface FacetPoolContext {
|
|
|
63
65
|
storage: {
|
|
64
66
|
get(key: string): Promise<unknown> | unknown;
|
|
65
67
|
put(key: string, value: unknown): Promise<void>;
|
|
68
|
+
/** The session's SQL, where the storage ledger (N18) records facet databases. */
|
|
69
|
+
sql?: SqlDatabase;
|
|
66
70
|
};
|
|
67
71
|
}
|
|
68
72
|
|
|
@@ -134,6 +138,7 @@ export class FacetPool {
|
|
|
134
138
|
if (keepStorage) return;
|
|
135
139
|
try {
|
|
136
140
|
facets.delete(name);
|
|
141
|
+
if (this.ctx.storage.sql) forgetFacetStorage(this.ctx.storage.sql, name);
|
|
137
142
|
} catch (e) {
|
|
138
143
|
throw new Error(
|
|
139
144
|
`fabric: facet '${name}' was evicted but its storage was not reclaimed — `
|