@nimbus-sh/fabric 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -0
- package/dist/bindings.d.ts +31 -33
- package/dist/bindings.d.ts.map +1 -1
- package/dist/bindings.js +108 -97
- package/dist/budgets.d.ts +61 -9
- package/dist/budgets.d.ts.map +1 -1
- package/dist/budgets.js +205 -31
- package/dist/fanout.d.ts.map +1 -1
- package/dist/fanout.js +4 -3
- package/dist/fenced-work.d.ts +3 -3
- package/dist/fenced-work.js +3 -3
- package/dist/inner-do-env.d.ts +83 -0
- package/dist/inner-do-env.d.ts.map +1 -0
- package/dist/inner-do-env.js +181 -0
- package/dist/isolate-pool.d.ts +9 -0
- package/dist/isolate-pool.d.ts.map +1 -1
- package/dist/isolate-pool.js +55 -27
- package/dist/workerd-facet-host.d.ts.map +1 -1
- package/dist/workerd-facet-host.js +5 -0
- package/package.json +2 -2
- package/src/bindings.ts +121 -98
- package/src/budgets.ts +244 -33
- package/src/fanout.ts +5 -3
- package/src/fenced-work.ts +3 -3
- package/src/inner-do-env.ts +213 -0
- package/src/isolate-pool.ts +65 -27
- package/src/workerd-facet-host.ts +4 -0
package/src/budgets.ts
CHANGED
|
@@ -15,7 +15,9 @@
|
|
|
15
15
|
* the width fan-outs have claimed and not yet released. A fan-out spends only
|
|
16
16
|
* the {@link dynamicWorkerHeadroom} that leaves, so work a Durable Object
|
|
17
17
|
* already has in flight — a resident process, the esbuild facet, a git
|
|
18
|
-
* network op, another fan-out — keeps its slots.
|
|
18
|
+
* network op, another fan-out — keeps its slots. Work that would rather wait
|
|
19
|
+
* than be refused waits on the ledger ({@link beginLoaderFetchWhenFree}) and
|
|
20
|
+
* is let in, in the order it asked, by whichever release makes room.
|
|
19
21
|
*
|
|
20
22
|
* Keyed weakly off the hosting actor's `ctx`, like the facet slot books: the
|
|
21
23
|
* limit is per Durable Object, and dynamic workers die with the isolate that
|
|
@@ -34,35 +36,180 @@ import { hostWasmIdentity } from './host-wasm.js';
|
|
|
34
36
|
*/
|
|
35
37
|
export const DO_DYNAMIC_WORKER_LIMIT = 10;
|
|
36
38
|
|
|
39
|
+
/**
|
|
40
|
+
* Ends one hold, idempotently. Pass the error the call failed with, if it
|
|
41
|
+
* did: a "Dynamic worker concurrency limit exceeded" refusal pauses the
|
|
42
|
+
* ledger's admissions (see {@link beginLoaderFetchWhenFree}); anything else,
|
|
43
|
+
* or nothing, just ends the hold.
|
|
44
|
+
*/
|
|
45
|
+
export type EndLoaderFetch = (failure?: unknown) => void;
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* A width one fan-out reserved with {@link claimDynamicWorkers}. Holds taken
|
|
49
|
+
* under it (`beginLoaderFetch(ctx, key, claim)`) count inside that width, not
|
|
50
|
+
* on top of it, until `release` (idempotent).
|
|
51
|
+
*/
|
|
52
|
+
export interface DynamicWorkerClaim {
|
|
53
|
+
release(): void;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
interface ClaimEntry {
|
|
57
|
+
width: number;
|
|
58
|
+
/** Loader id → open holds taken under this claim. */
|
|
59
|
+
keys: Map<string, number>;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
interface Waiter {
|
|
63
|
+
key: string;
|
|
64
|
+
claim: ClaimEntry | undefined;
|
|
65
|
+
admit(end: EndLoaderFetch): void;
|
|
66
|
+
}
|
|
67
|
+
|
|
37
68
|
interface LoaderLedger {
|
|
38
|
-
/** Loader id → open holds on it. A key is present only while held. */
|
|
69
|
+
/** Loader id → open holds on it, under a claim or not. A key is present only while held. */
|
|
39
70
|
inFlight: Map<string, number>;
|
|
40
|
-
/**
|
|
41
|
-
|
|
71
|
+
/** Claims not yet released. */
|
|
72
|
+
claims: Set<ClaimEntry>;
|
|
42
73
|
/** The most distinct workers (holds plus claims) ever counted at once. */
|
|
43
74
|
peak: number;
|
|
75
|
+
/** Waits not yet admitted, in the order they asked. */
|
|
76
|
+
waiters: Waiter[];
|
|
77
|
+
/** Length of the pause a refusal started, while it lasts; 0 when admitting. */
|
|
78
|
+
pauseMs: number;
|
|
79
|
+
pauseTimer: ReturnType<typeof setTimeout> | undefined;
|
|
80
|
+
/** Advanced when a pause starts and when it ends; a hold keeps the one it began in. */
|
|
81
|
+
epoch: number;
|
|
82
|
+
/** Pauses in a row with no worker admitted between them. */
|
|
83
|
+
refusals: number;
|
|
44
84
|
}
|
|
45
85
|
|
|
86
|
+
/**
|
|
87
|
+
* The first pause after a limit refusal, doubling while refusals continue,
|
|
88
|
+
* up to {@link REFUSAL_PAUSE_MAX_MS}. A deployed Durable Object admitted a
|
|
89
|
+
* batch it had refused after a 6 s pause.
|
|
90
|
+
*/
|
|
91
|
+
const REFUSAL_PAUSE_MS = 50;
|
|
92
|
+
const REFUSAL_PAUSE_MAX_MS = 2_000;
|
|
93
|
+
|
|
46
94
|
const ledgers = new WeakMap<object, LoaderLedger>();
|
|
95
|
+
const claimEntries = new WeakMap<DynamicWorkerClaim, { ledger: LoaderLedger; entry: ClaimEntry }>();
|
|
47
96
|
|
|
48
97
|
function ledger(ctx: object): LoaderLedger {
|
|
49
98
|
let entry = ledgers.get(ctx);
|
|
50
99
|
if (!entry) {
|
|
51
|
-
entry = {
|
|
100
|
+
entry = {
|
|
101
|
+
inFlight: new Map(), claims: new Set(), peak: 0,
|
|
102
|
+
waiters: [], pauseMs: 0, pauseTimer: undefined, epoch: 0, refusals: 0,
|
|
103
|
+
};
|
|
52
104
|
ledgers.set(ctx, entry);
|
|
53
105
|
}
|
|
54
106
|
return entry;
|
|
55
107
|
}
|
|
56
108
|
|
|
109
|
+
/** Distinct workers counted: each claim's width (or more, if its holds exceed it), plus held keys no claim covers. */
|
|
57
110
|
function inUse(entry: LoaderLedger): number {
|
|
58
|
-
|
|
111
|
+
let count = 0;
|
|
112
|
+
const covered = new Set<string>();
|
|
113
|
+
for (const claim of entry.claims) {
|
|
114
|
+
count += Math.max(claim.width, claim.keys.size);
|
|
115
|
+
for (const key of claim.keys.keys()) covered.add(key);
|
|
116
|
+
}
|
|
117
|
+
for (const key of entry.inFlight.keys()) if (!covered.has(key)) count++;
|
|
118
|
+
return count;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function headroom(entry: LoaderLedger): number {
|
|
122
|
+
return entry.pauseMs > 0 ? 0 : Math.max(0, DO_DYNAMIC_WORKER_LIMIT - inUse(entry));
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function claimOf(ctx: object, claim: DynamicWorkerClaim | undefined): ClaimEntry | undefined {
|
|
126
|
+
if (claim === undefined) return undefined;
|
|
127
|
+
const owned = claimEntries.get(claim);
|
|
128
|
+
if (owned === undefined || owned.ledger !== ledger(ctx)) {
|
|
129
|
+
throw new Error('Nimbus: a Dynamic Worker claim is used only on the ledger of the actor that claimed it');
|
|
130
|
+
}
|
|
131
|
+
return owned.ledger.claims.has(owned.entry) ? owned.entry : undefined;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function count(map: Map<string, number>, key: string, by: 1 | -1): void {
|
|
135
|
+
const open = (map.get(key) ?? 0) + by;
|
|
136
|
+
if (open > 0) map.set(key, open);
|
|
137
|
+
else map.delete(key);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** Take one hold; the caller admits waiters after. */
|
|
141
|
+
function hold(entry: LoaderLedger, workerKey: string, claim: ClaimEntry | undefined): EndLoaderFetch {
|
|
142
|
+
count(entry.inFlight, workerKey, 1);
|
|
143
|
+
if (claim) count(claim.keys, workerKey, 1);
|
|
144
|
+
entry.peak = Math.max(entry.peak, inUse(entry));
|
|
145
|
+
const epoch = entry.epoch;
|
|
146
|
+
let ended = false;
|
|
147
|
+
return (failure) => {
|
|
148
|
+
if (ended) return;
|
|
149
|
+
ended = true;
|
|
150
|
+
count(entry.inFlight, workerKey, -1);
|
|
151
|
+
if (claim) count(claim.keys, workerKey, -1);
|
|
152
|
+
if (classifyError(failure) === 'dynamic_worker_cap') refused(entry, epoch);
|
|
153
|
+
else if (epoch === entry.epoch && entry.pauseMs === 0) entry.refusals = 0;
|
|
154
|
+
admitWaiters(entry);
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* The platform refused a worker this ledger counted room for: it still
|
|
160
|
+
* counts workers the ledger has released, which no release here can show.
|
|
161
|
+
* So nothing new is admitted until a pause has passed. A refusal of a call
|
|
162
|
+
* that began before the latest pause started or ended is the same lag and
|
|
163
|
+
* changes nothing; one of a call let in after it doubles the next pause.
|
|
164
|
+
*/
|
|
165
|
+
function refused(entry: LoaderLedger, epoch: number): void {
|
|
166
|
+
if (epoch !== entry.epoch) return;
|
|
167
|
+
if (entry.pauseTimer !== undefined) clearTimeout(entry.pauseTimer);
|
|
168
|
+
entry.pauseMs = Math.min(REFUSAL_PAUSE_MAX_MS, REFUSAL_PAUSE_MS * 2 ** entry.refusals);
|
|
169
|
+
entry.refusals++;
|
|
170
|
+
entry.epoch++;
|
|
171
|
+
entry.pauseTimer = setTimeout(() => {
|
|
172
|
+
entry.pauseMs = 0;
|
|
173
|
+
entry.pauseTimer = undefined;
|
|
174
|
+
entry.epoch++;
|
|
175
|
+
admitWaiters(entry);
|
|
176
|
+
}, entry.pauseMs);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
function admissible(entry: LoaderLedger, waiter: Waiter): boolean {
|
|
180
|
+
// Requests to a worker already in flight count once, even while paused.
|
|
181
|
+
if (entry.inFlight.has(waiter.key)) return true;
|
|
182
|
+
if (entry.pauseMs > 0) return false;
|
|
183
|
+
if (waiter.claim && entry.claims.has(waiter.claim) && waiter.claim.keys.size < waiter.claim.width) return true;
|
|
184
|
+
return inUse(entry) < DO_DYNAMIC_WORKER_LIMIT;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* Let in every waiter that fits, in the order they asked: each takes its
|
|
189
|
+
* hold here, so a freed slot goes to exactly one waiter and is never left
|
|
190
|
+
* between a wake and a begin. Run after every change that can make room.
|
|
191
|
+
* A waiter let in on a new key lets in the later ones on that key, and the
|
|
192
|
+
* earlier ones too: the scan starts over.
|
|
193
|
+
*/
|
|
194
|
+
function admitWaiters(entry: LoaderLedger): void {
|
|
195
|
+
for (let i = 0; i < entry.waiters.length;) {
|
|
196
|
+
const waiter = entry.waiters[i];
|
|
197
|
+
if (!admissible(entry, waiter)) { i++; continue; }
|
|
198
|
+
const joins = entry.inFlight.has(waiter.key);
|
|
199
|
+
entry.waiters.splice(i, 1);
|
|
200
|
+
waiter.admit(hold(entry, waiter.key, waiter.claim));
|
|
201
|
+
if (!joins) i = 0;
|
|
202
|
+
}
|
|
59
203
|
}
|
|
60
204
|
|
|
61
205
|
/**
|
|
62
206
|
* Hold the Dynamic Worker `workerKey` in flight on this actor's ledger; the
|
|
63
207
|
* returned function ends the hold (idempotently), from the caller's own
|
|
64
208
|
* `finally`. Holds on one key nest: the worker counts once until the last
|
|
65
|
-
* one ends, as the platform counts it.
|
|
209
|
+
* one ends, as the platform counts it. Under a `claim`, the hold counts
|
|
210
|
+
* inside the claim's width. This never waits: it is for work the actor
|
|
211
|
+
* starts regardless (a resident process); {@link beginLoaderFetchWhenFree}
|
|
212
|
+
* waits for room.
|
|
66
213
|
*
|
|
67
214
|
* A begin/end pair rather than a wrapper on purpose, and the shape is
|
|
68
215
|
* load-bearing: wrapping the stub call in a ledger-owned async frame
|
|
@@ -75,48 +222,100 @@ function inUse(entry: LoaderLedger): number {
|
|
|
75
222
|
* workers: an RPC stub call must stay a direct property call awaited by the
|
|
76
223
|
* frame that made it, so the ledger only brackets it.
|
|
77
224
|
*/
|
|
78
|
-
export function beginLoaderFetch(ctx: object, workerKey: string):
|
|
225
|
+
export function beginLoaderFetch(ctx: object, workerKey: string, claim?: DynamicWorkerClaim): EndLoaderFetch {
|
|
79
226
|
const entry = ledger(ctx);
|
|
80
|
-
entry
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
227
|
+
const end = hold(entry, workerKey, claimOf(ctx, claim));
|
|
228
|
+
admitWaiters(entry);
|
|
229
|
+
return end;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* {@link beginLoaderFetch} once the ledger has room: resolves, holding
|
|
234
|
+
* `workerKey`, as soon as that worker is already in flight (holds on it
|
|
235
|
+
* count once) or a distinct worker more fits — within the `claim`'s width,
|
|
236
|
+
* or the actor's headroom. Waits are let in in the order they asked, by
|
|
237
|
+
* whoever's release makes the room: a hold's end, a claim's release, a
|
|
238
|
+
* pause's end. The hold is taken as the wait is let in, so a freed slot
|
|
239
|
+
* wakes one waiter and no other caller can take it first; once resolved, it
|
|
240
|
+
* is the caller's to end.
|
|
241
|
+
*
|
|
242
|
+
* A call refused with "Dynamic worker concurrency limit exceeded" ends its
|
|
243
|
+
* hold with the refusal (`end(error)`) and waits again: the refusal pauses
|
|
244
|
+
* admission (50 ms, doubling to 2 s while refusals continue), because the
|
|
245
|
+
* platform counts a worker for a moment after its call returns and no
|
|
246
|
+
* release can show that.
|
|
247
|
+
*
|
|
248
|
+
* `signal` abandons the wait: it rejects with the signal's reason and holds
|
|
249
|
+
* nothing. A wait outlives nothing on its own: bound it with a signal when
|
|
250
|
+
* room may never come (a resident process holds its worker for as long as
|
|
251
|
+
* it runs).
|
|
252
|
+
*
|
|
253
|
+
* const end = await beginLoaderFetchWhenFree(ctx, key, { signal });
|
|
254
|
+
* try { return await worker.getEntrypoint().run(); }
|
|
255
|
+
* catch (error) { end(error); throw error; }
|
|
256
|
+
* finally { end(); }
|
|
257
|
+
*/
|
|
258
|
+
export function beginLoaderFetchWhenFree(
|
|
259
|
+
ctx: object,
|
|
260
|
+
workerKey: string,
|
|
261
|
+
options: { signal?: AbortSignal; claim?: DynamicWorkerClaim } = {},
|
|
262
|
+
): Promise<EndLoaderFetch> {
|
|
263
|
+
const { signal } = options;
|
|
264
|
+
return new Promise<EndLoaderFetch>((resolve, reject) => {
|
|
265
|
+
if (signal?.aborted) {
|
|
266
|
+
reject(signal.reason);
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
const entry = ledger(ctx);
|
|
270
|
+
const abandon = () => {
|
|
271
|
+
const at = entry.waiters.indexOf(waiter);
|
|
272
|
+
if (at < 0) return;
|
|
273
|
+
entry.waiters.splice(at, 1);
|
|
274
|
+
reject(signal?.reason);
|
|
275
|
+
};
|
|
276
|
+
const waiter: Waiter = {
|
|
277
|
+
key: workerKey,
|
|
278
|
+
claim: claimOf(ctx, options.claim),
|
|
279
|
+
admit(end) {
|
|
280
|
+
signal?.removeEventListener('abort', abandon);
|
|
281
|
+
resolve(end);
|
|
282
|
+
},
|
|
283
|
+
};
|
|
284
|
+
entry.waiters.push(waiter);
|
|
285
|
+
signal?.addEventListener('abort', abandon, { once: true });
|
|
286
|
+
admitWaiters(entry);
|
|
287
|
+
});
|
|
90
288
|
}
|
|
91
289
|
|
|
92
290
|
/**
|
|
93
291
|
* Distinct Dynamic Workers this actor may still put in flight: the limit
|
|
94
|
-
* less what is held and claimed right now
|
|
292
|
+
* less what is held and claimed right now, and none while a limit refusal's
|
|
293
|
+
* pause lasts. Never negative.
|
|
95
294
|
*/
|
|
96
295
|
export function dynamicWorkerHeadroom(ctx: object): number {
|
|
97
|
-
return
|
|
296
|
+
return headroom(ledger(ctx));
|
|
98
297
|
}
|
|
99
298
|
|
|
100
299
|
/**
|
|
101
300
|
* Claim `width` distinct Dynamic Workers for one fan-out, or null when the
|
|
102
301
|
* headroom cannot hold it. The claim counts until `release` (idempotent), so
|
|
103
302
|
* a second fan-out sizing itself meanwhile sees it; the claimant's own
|
|
104
|
-
* dispatches
|
|
105
|
-
* toward sending that second fan-out elsewhere.
|
|
303
|
+
* dispatches, held under the claim, count inside it.
|
|
106
304
|
*/
|
|
107
|
-
export function claimDynamicWorkers(ctx: object, width: number):
|
|
305
|
+
export function claimDynamicWorkers(ctx: object, width: number): DynamicWorkerClaim | null {
|
|
108
306
|
const entry = ledger(ctx);
|
|
109
|
-
if (width < 1 || width >
|
|
110
|
-
|
|
307
|
+
if (width < 1 || width > headroom(entry)) return null;
|
|
308
|
+
const claim: ClaimEntry = { width, keys: new Map() };
|
|
309
|
+
entry.claims.add(claim);
|
|
111
310
|
entry.peak = Math.max(entry.peak, inUse(entry));
|
|
112
|
-
|
|
113
|
-
return {
|
|
311
|
+
const handle: DynamicWorkerClaim = {
|
|
114
312
|
release() {
|
|
115
|
-
if (
|
|
116
|
-
|
|
117
|
-
entry.claimed -= width;
|
|
313
|
+
if (!entry.claims.delete(claim)) return;
|
|
314
|
+
admitWaiters(entry);
|
|
118
315
|
},
|
|
119
316
|
};
|
|
317
|
+
claimEntries.set(handle, { ledger: entry, entry: claim });
|
|
318
|
+
return handle;
|
|
120
319
|
}
|
|
121
320
|
|
|
122
321
|
/** Snapshot for the diag surface. Pure read; no I/O. */
|
|
@@ -126,17 +325,29 @@ export function loaderLedgerStats(ctx: object): {
|
|
|
126
325
|
claimed: number;
|
|
127
326
|
headroom: number;
|
|
128
327
|
peak: number;
|
|
328
|
+
/** Waits not yet admitted. */
|
|
329
|
+
waiting: number;
|
|
330
|
+
/** Length of the pause a limit refusal started, while it lasts; 0 when admitting. */
|
|
331
|
+
pauseMs: number;
|
|
129
332
|
} {
|
|
130
333
|
const entry = ledger(ctx);
|
|
131
334
|
return {
|
|
132
335
|
limit: DO_DYNAMIC_WORKER_LIMIT,
|
|
133
336
|
inFlightWorkers: [...entry.inFlight.keys()],
|
|
134
|
-
claimed: entry
|
|
135
|
-
headroom:
|
|
337
|
+
claimed: claimedWidth(entry),
|
|
338
|
+
headroom: headroom(entry),
|
|
136
339
|
peak: entry.peak,
|
|
340
|
+
waiting: entry.waiters.length,
|
|
341
|
+
pauseMs: entry.pauseMs,
|
|
137
342
|
};
|
|
138
343
|
}
|
|
139
344
|
|
|
345
|
+
function claimedWidth(entry: LoaderLedger): number {
|
|
346
|
+
let width = 0;
|
|
347
|
+
for (const claim of entry.claims) width += claim.width;
|
|
348
|
+
return width;
|
|
349
|
+
}
|
|
350
|
+
|
|
140
351
|
/**
|
|
141
352
|
* Name the per-DO accounting on a "Dynamic worker concurrency limit exceeded"
|
|
142
353
|
* failure; hand every other error back untouched. The platform's message
|
|
@@ -149,7 +360,7 @@ export function withDynamicWorkerCapNamed<E>(ctx: object, error: E): E | Error {
|
|
|
149
360
|
const platform = error instanceof Error ? error.message : String(error);
|
|
150
361
|
return new Error(
|
|
151
362
|
`${platform} — this Durable Object had ${entry.inFlight.size} distinct dynamic worker(s) in flight `
|
|
152
|
-
+ `(${[...entry.inFlight.keys()].join(', ') || 'none recorded'}) and ${entry
|
|
363
|
+
+ `(${[...entry.inFlight.keys()].join(', ') || 'none recorded'}) and ${claimedWidth(entry)} claimed by fan-outs, `
|
|
153
364
|
+ `against a limit of ${DO_DYNAMIC_WORKER_LIMIT}; peak ${entry.peak}`,
|
|
154
365
|
{ cause: error },
|
|
155
366
|
);
|
package/src/fanout.ts
CHANGED
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
import { serializeFunction } from './vendor/serialize.js';
|
|
22
22
|
import { BindingError } from './vendor/errors.js';
|
|
23
23
|
import { IsolatePool, type FacetTaskFn } from './isolate-pool.js';
|
|
24
|
-
import { claimDynamicWorkers, dynamicWorkerHeadroom } from './budgets.js';
|
|
24
|
+
import { claimDynamicWorkers, dynamicWorkerHeadroom, type DynamicWorkerClaim } from './budgets.js';
|
|
25
25
|
import { hostRoute } from './composition.js';
|
|
26
26
|
import { disposeRpcResource } from '@nimbus-sh/platform/rpc-dispose.js';
|
|
27
27
|
import { classifyDoCall, describeError, isRetryableDoCall } from '@nimbus-sh/platform/oom-classify.js';
|
|
@@ -254,7 +254,7 @@ export class Fanout {
|
|
|
254
254
|
this.opts.onRoute?.({ topology: claim ? 'in-do' : 'peer-do', tasks: tasks.length, headroom });
|
|
255
255
|
if (!claim) return this._dispatchPeerDo<A, R>(tasks, fn);
|
|
256
256
|
try {
|
|
257
|
-
return await this._dispatchInDo<A, R>(tasks, fn);
|
|
257
|
+
return await this._dispatchInDo<A, R>(tasks, fn, claim);
|
|
258
258
|
} finally {
|
|
259
259
|
claim.release();
|
|
260
260
|
}
|
|
@@ -283,11 +283,13 @@ export class Fanout {
|
|
|
283
283
|
private async _dispatchInDo<A, R>(
|
|
284
284
|
tasks: FanoutTask<A>[],
|
|
285
285
|
fn: FacetTaskFn<A, R>,
|
|
286
|
+
claim: DynamicWorkerClaim,
|
|
286
287
|
): Promise<R[]> {
|
|
287
288
|
// One slot — one Dynamic Worker — per task; submitMany has claimed
|
|
288
|
-
// that width on the ledger.
|
|
289
|
+
// that width on the ledger, and the pool's dispatches are held inside it.
|
|
289
290
|
const pool = new IsolatePool(this.env, this.ctx, {
|
|
290
291
|
concurrency: tasks.length,
|
|
292
|
+
claim,
|
|
291
293
|
timeoutMs: this.opts.timeoutMs,
|
|
292
294
|
tag: this.opts.tag,
|
|
293
295
|
preamble: this.opts.preamble,
|
package/src/fenced-work.ts
CHANGED
|
@@ -262,9 +262,9 @@ export class FencedWork<R extends FencedWorkRecord> {
|
|
|
262
262
|
* Runs once per instance — re-calls in the same instance are no-ops — and
|
|
263
263
|
* re-drives every row whose pid is `> 0` and at or below `generationBase()`
|
|
264
264
|
* with `attempt < FENCED_WORK_MAX_ATTEMPT`; the rest are abandoned. What the
|
|
265
|
-
* re-drive resolver receives is the journalled
|
|
266
|
-
* env and
|
|
267
|
-
*
|
|
265
|
+
* re-drive resolver receives is the journalled record and nothing else: a
|
|
266
|
+
* host keeps env and secrets out of it, so the resolver's embedder
|
|
267
|
+
* re-resolves them rather than reading them back.
|
|
268
268
|
*/
|
|
269
269
|
async recoverInterrupted(): Promise<void> {
|
|
270
270
|
if (this.recovered) return;
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* inner-do-env.ts — a classic Durable Object binding inside an inner Worker.
|
|
3
|
+
*
|
|
4
|
+
* `nimbus wrangler dev` loads the user's Worker as a dynamic worker, and its
|
|
5
|
+
* Durable Object bindings run as facets of the session Durable Object. The
|
|
6
|
+
* binding the loader passes (`NimbusDurableObjectNamespace`, an entrypoint of
|
|
7
|
+
* the session's isolate) answers RPC, so it cannot be the inner Worker's
|
|
8
|
+
* namespace: a DurableObjectNamespace's API is synchronous (`env.P.get(
|
|
9
|
+
* env.P.idFromName('x'))` takes no await), and an RpcPromise cannot travel as
|
|
10
|
+
* an argument ("Could not serialize object of type RpcPromise").
|
|
11
|
+
*
|
|
12
|
+
* So a module the inner Worker runs before any of its own code (its main
|
|
13
|
+
* module's first import, `innerWorkerModules`) replaces each such binding in
|
|
14
|
+
* the isolate's env, which every handler, entrypoint and Durable Object of the
|
|
15
|
+
* isolate sees, with a local namespace. Ids are made locally, and `get`
|
|
16
|
+
* answers at once an RPC stub (`new RpcStub(target)`) of a local target that
|
|
17
|
+
* relays each member the stub's caller reaches (a call, a read, or a path
|
|
18
|
+
* through members, fetch included) to the binding's `callOn` or `getOn`,
|
|
19
|
+
* which the session runs on the object's facet. An RPC stub is what a
|
|
20
|
+
* Durable Object stub is to the runtime: callable by any method name, read by
|
|
21
|
+
* any property name, pipelined, bound to its request, and transferable, as an
|
|
22
|
+
* argument or an answer, where an entrypoint of a dynamically-loaded Worker
|
|
23
|
+
* is not. Arguments and answers cross natively, stubs, functions and streams
|
|
24
|
+
* included.
|
|
25
|
+
*
|
|
26
|
+
* Its prototype is not RpcStub's but one shaped as a Durable Object stub's:
|
|
27
|
+
* its constructor is a class `DurableObject` that cannot be constructed, its
|
|
28
|
+
* tag is 'DurableObject', and it has no `dup` or Symbol.dispose of its own,
|
|
29
|
+
* so `dup` is a member, which the runtime refuses as on Cloudflare. It
|
|
30
|
+
* differs from a Durable Object stub in two ways the runtime fixes. `typeof`
|
|
31
|
+
* is 'function'. And it is not persistent, so a Worker Loader env cannot carry
|
|
32
|
+
* it ("RpcStub cannot be serialized in this context because it is not a
|
|
33
|
+
* persistent stub"): the loader shim (NimbusLoaderRPC) keeps a child's code
|
|
34
|
+
* and loads it again in each later request, so the child's env can carry
|
|
35
|
+
* nothing made in one request, the session's own stubs included.
|
|
36
|
+
*/
|
|
37
|
+
|
|
38
|
+
import type { RpcStub, WorkerEntrypoint } from 'cloudflare:workers';
|
|
39
|
+
import { ESBUILD_NAME_MODULE_SHIM } from '@nimbus-sh/core/_shared/esbuild-facet-shim.js';
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* The id string a name gives: deterministic (FNV-style, 64-bit hex), with the
|
|
43
|
+
* prefix `name:` so it never collides with a `uniq:` id. Self-contained: its
|
|
44
|
+
* source also runs in the inner isolate (innerDoAdapter).
|
|
45
|
+
*/
|
|
46
|
+
export function innerDoIdFromName(name: string): string {
|
|
47
|
+
let h1 = 0xdeadbeef ^ name.length;
|
|
48
|
+
let h2 = 0x41c6ce57 ^ name.length;
|
|
49
|
+
for (let i = 0; i < name.length; i++) {
|
|
50
|
+
const ch = name.charCodeAt(i);
|
|
51
|
+
h1 = Math.imul(h1 ^ ch, 2654435761);
|
|
52
|
+
h2 = Math.imul(h2 ^ ch, 1597334677);
|
|
53
|
+
}
|
|
54
|
+
h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507) ^ Math.imul(h2 ^ (h2 >>> 13), 3266489909);
|
|
55
|
+
h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) ^ Math.imul(h1 ^ (h1 >>> 13), 3266489909);
|
|
56
|
+
return 'name:' + (h1 >>> 0).toString(16).padStart(8, '0') + (h2 >>> 0).toString(16).padStart(8, '0');
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** What the binding the loader passes answers: one access to one object. */
|
|
60
|
+
export interface InnerDoRemote {
|
|
61
|
+
/** The member of object `id` at `path` (names from the object down), called with `args`. */
|
|
62
|
+
callOn(id: string, path: string[], args: unknown[]): Promise<unknown>;
|
|
63
|
+
/** The member of object `id` at `path`, read. */
|
|
64
|
+
getOn(id: string, path: string[]): Promise<unknown>;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** What the adapter runs over: the inner isolate's `cloudflare:workers`. */
|
|
68
|
+
export interface InnerDoRuntime {
|
|
69
|
+
env: object;
|
|
70
|
+
WorkerEntrypoint: typeof WorkerEntrypoint;
|
|
71
|
+
RpcStub: typeof RpcStub;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** The entrypoint a build asks which Durable Object classes are missing, unless the bundle spells it. */
|
|
75
|
+
const CLASSES_ENTRYPOINT = 'NimbusDurableObjectClasses';
|
|
76
|
+
/**
|
|
77
|
+
* The adapter, as it runs in the inner isolate: it replaces each of `names`
|
|
78
|
+
* in `runtime.env` that holds the binding with a local DurableObjectNamespace,
|
|
79
|
+
* and answers the class check the main module exports. `main` is the main
|
|
80
|
+
* module's namespace. Self-contained (serialized with toString): it reaches
|
|
81
|
+
* nothing outside itself but its arguments.
|
|
82
|
+
*/
|
|
83
|
+
export function innerDoAdapter(
|
|
84
|
+
idFromName: (name: string) => string,
|
|
85
|
+
names: readonly string[],
|
|
86
|
+
main: object,
|
|
87
|
+
runtime: InnerDoRuntime,
|
|
88
|
+
): { NimbusDurableObjectClasses: unknown } {
|
|
89
|
+
/** A Durable Object id: its string, and the name it was made from. */
|
|
90
|
+
class DurableObjectId {
|
|
91
|
+
readonly name?: string;
|
|
92
|
+
readonly #id: string;
|
|
93
|
+
constructor(id: string, name?: string) {
|
|
94
|
+
this.#id = id;
|
|
95
|
+
if (name !== undefined) this.name = name;
|
|
96
|
+
}
|
|
97
|
+
toString(): string { return this.#id; }
|
|
98
|
+
equals(other: unknown): boolean { return other instanceof DurableObjectId && String(other) === this.#id; }
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Whether `value` is the binding the loader passes (an RPC stub's methods are its properties). */
|
|
102
|
+
function isRemote(value: unknown): value is InnerDoRemote {
|
|
103
|
+
return value !== null && (typeof value === 'object' || typeof value === 'function')
|
|
104
|
+
&& typeof Reflect.get(value, 'callOn') === 'function' && typeof Reflect.get(value, 'getOn') === 'function';
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* The member of object `id` at `path`, as the runtime reaches it in an RPC
|
|
109
|
+
* to the stub: called, read (it is thenable, and the runtime awaits what a
|
|
110
|
+
* read answers), or walked through to one of its own members (the runtime
|
|
111
|
+
* walks a path through own properties only, so every name is reported as
|
|
112
|
+
* one). At the empty path it is the stub's target.
|
|
113
|
+
*/
|
|
114
|
+
function member(remote: InnerDoRemote, id: string, path: readonly string[]): (...args: unknown[]) => unknown {
|
|
115
|
+
const next = (name: string) => member(remote, id, [...path, name]);
|
|
116
|
+
return new Proxy((..._args: unknown[]): unknown => undefined, {
|
|
117
|
+
apply: (_target, _self, args: unknown[]) => remote.callOn(id, [...path], args),
|
|
118
|
+
get: (target, name) => {
|
|
119
|
+
if (name === 'then') {
|
|
120
|
+
return (resolve: (value: unknown) => unknown, reject: (reason: unknown) => unknown) => remote.getOn(id, [...path]).then(resolve, reject);
|
|
121
|
+
}
|
|
122
|
+
return typeof name === 'string' ? next(name) : Reflect.get(target, name);
|
|
123
|
+
},
|
|
124
|
+
getOwnPropertyDescriptor: (target, name) => (typeof name === 'string' && name !== 'then'
|
|
125
|
+
? { value: next(name), writable: true, enumerable: true, configurable: true }
|
|
126
|
+
: Reflect.getOwnPropertyDescriptor(target, name)),
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* A stub's prototype, as a Durable Object stub's shows: its constructor
|
|
132
|
+
* cannot be called or constructed, and it is tagged 'DurableObject'.
|
|
133
|
+
*/
|
|
134
|
+
const stubPrototype: object = Object.create(Object.prototype, {
|
|
135
|
+
constructor: {
|
|
136
|
+
value: function DurableObject(): never { throw new TypeError('Illegal constructor'); },
|
|
137
|
+
writable: true,
|
|
138
|
+
configurable: true,
|
|
139
|
+
},
|
|
140
|
+
[Symbol.toStringTag]: { value: 'DurableObject', configurable: true },
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
/** Asked once by a build: which classes the main module does not export. */
|
|
144
|
+
class NimbusDurableObjectClasses extends runtime.WorkerEntrypoint {
|
|
145
|
+
missing(classNames: string[]): string[] {
|
|
146
|
+
return classNames.filter((name) => typeof Reflect.get(main, name) !== 'function');
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** env.MY_DO: the namespace, made locally; its stubs relay to `remote`. */
|
|
151
|
+
class DurableObjectNamespace {
|
|
152
|
+
readonly #remote: InnerDoRemote;
|
|
153
|
+
constructor(remote: InnerDoRemote) { this.#remote = remote; }
|
|
154
|
+
idFromName(name: string): DurableObjectId { return new DurableObjectId(idFromName(String(name)), String(name)); }
|
|
155
|
+
newUniqueId(): DurableObjectId { return new DurableObjectId('uniq:' + crypto.randomUUID().replaceAll('-', '')); }
|
|
156
|
+
idFromString(id: string): DurableObjectId { return new DurableObjectId(String(id)); }
|
|
157
|
+
get(id: DurableObjectId | string): object {
|
|
158
|
+
const at = id instanceof DurableObjectId ? id : new DurableObjectId(String(id));
|
|
159
|
+
const stub = Object.setPrototypeOf(new runtime.RpcStub(member(this.#remote, String(at), [])), stubPrototype);
|
|
160
|
+
// As a Durable Object stub has them: its own, enumerable, in this order.
|
|
161
|
+
return Object.defineProperties(stub, {
|
|
162
|
+
name: { value: at.name, enumerable: true },
|
|
163
|
+
id: { value: at, enumerable: true },
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
getByName(name: string): object { return this.get(this.idFromName(name)); }
|
|
167
|
+
jurisdiction(): DurableObjectNamespace { return this; }
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
for (const name of names) {
|
|
171
|
+
const remote: unknown = Reflect.get(runtime.env, name);
|
|
172
|
+
if (isRemote(remote)) Reflect.set(runtime.env, name, new DurableObjectNamespace(remote));
|
|
173
|
+
}
|
|
174
|
+
return { NimbusDurableObjectClasses };
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** The inner Worker's own module, as bundled, and the adapter's. */
|
|
178
|
+
const MAIN_MODULE = 'worker.js';
|
|
179
|
+
const ADAPTER_MODULE = 'nimbus-do-env.js';
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* The modules an inner Worker runs with Durable Object bindings `names`: its
|
|
183
|
+
* bundle as the main module, whose first import is the adapter (so the
|
|
184
|
+
* adapter has run before any of the Worker's code) and which exports the
|
|
185
|
+
* class check as `classesEntrypoint`, and the adapter. The import shares the
|
|
186
|
+
* bundle's first line, so line numbers stay the bundle's. A Worker with no
|
|
187
|
+
* such binding runs its bundle as it is, and has no class check (null).
|
|
188
|
+
*
|
|
189
|
+
* `classesEntrypoint` is a name the bundle never spells, so it exports no
|
|
190
|
+
* such name itself (a bundler prints an ASCII name as it is).
|
|
191
|
+
*/
|
|
192
|
+
export function innerWorkerModules(bundle: string, names: readonly string[]): {
|
|
193
|
+
mainModule: string;
|
|
194
|
+
modules: Record<string, string>;
|
|
195
|
+
classesEntrypoint: string | null;
|
|
196
|
+
} {
|
|
197
|
+
if (names.length === 0) return { mainModule: MAIN_MODULE, modules: { [MAIN_MODULE]: bundle }, classesEntrypoint: null };
|
|
198
|
+
let classesEntrypoint = CLASSES_ENTRYPOINT;
|
|
199
|
+
for (let n = 2; bundle.includes(classesEntrypoint); n++) classesEntrypoint = `${CLASSES_ENTRYPOINT}_${n}`;
|
|
200
|
+
const head = `export { ${CLASSES_ENTRYPOINT} as ${classesEntrypoint} } from './${ADAPTER_MODULE}';`;
|
|
201
|
+
// A hashbang must stay first.
|
|
202
|
+
const at = bundle.startsWith('#!') ? bundle.indexOf('\n') + 1 : 0;
|
|
203
|
+
const main = bundle.slice(0, at) + head + bundle.slice(at);
|
|
204
|
+
const adapter = [
|
|
205
|
+
"import { env, RpcStub, WorkerEntrypoint } from 'cloudflare:workers';",
|
|
206
|
+
`import * as main from './${MAIN_MODULE}';`,
|
|
207
|
+
// The functions below are serialized from the bundled worker, which wraps them in __name.
|
|
208
|
+
ESBUILD_NAME_MODULE_SHIM,
|
|
209
|
+
`const { ${CLASSES_ENTRYPOINT} } = (${innerDoAdapter.toString()})(${innerDoIdFromName.toString()}, ${JSON.stringify(names)}, main, { env, RpcStub, WorkerEntrypoint });`,
|
|
210
|
+
`export { ${CLASSES_ENTRYPOINT} };`,
|
|
211
|
+
].join('\n');
|
|
212
|
+
return { mainModule: MAIN_MODULE, modules: { [MAIN_MODULE]: main, [ADAPTER_MODULE]: adapter }, classesEntrypoint };
|
|
213
|
+
}
|