@nimbus-sh/fabric 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +487 -0
- package/dist/alarms.d.ts +134 -0
- package/dist/alarms.d.ts.map +1 -0
- package/dist/alarms.js +214 -0
- package/dist/bindings.d.ts +316 -0
- package/dist/bindings.d.ts.map +1 -0
- package/dist/bindings.js +678 -0
- package/dist/ctx-exports.d.ts +47 -0
- package/dist/ctx-exports.d.ts.map +1 -0
- package/dist/ctx-exports.js +54 -0
- package/dist/facet-image-store.d.ts +112 -0
- package/dist/facet-image-store.d.ts.map +1 -0
- package/dist/facet-image-store.js +181 -0
- package/dist/fanout-pool.d.ts +223 -0
- package/dist/fanout-pool.d.ts.map +1 -0
- package/dist/fanout-pool.js +368 -0
- package/dist/index.d.ts +26 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +25 -0
- package/dist/inner-do-registry.d.ts +41 -0
- package/dist/inner-do-registry.d.ts.map +1 -0
- package/dist/inner-do-registry.js +51 -0
- package/dist/launch-journal.d.ts +170 -0
- package/dist/launch-journal.d.ts.map +1 -0
- package/dist/launch-journal.js +154 -0
- package/dist/launch-pacer.d.ts +173 -0
- package/dist/launch-pacer.d.ts.map +1 -0
- package/dist/launch-pacer.js +193 -0
- package/dist/loader-ledger.d.ts +57 -0
- package/dist/loader-ledger.d.ts.map +1 -0
- package/dist/loader-ledger.js +91 -0
- package/dist/loader-pool.d.ts +315 -0
- package/dist/loader-pool.d.ts.map +1 -0
- package/dist/loader-pool.js +666 -0
- package/dist/process-fabric.d.ts +524 -0
- package/dist/process-fabric.d.ts.map +1 -0
- package/dist/process-fabric.js +388 -0
- package/dist/process-host.d.ts +132 -0
- package/dist/process-host.d.ts.map +1 -0
- package/dist/process-host.js +444 -0
- package/dist/vendor/errors.d.ts +24 -0
- package/dist/vendor/errors.d.ts.map +1 -0
- package/dist/vendor/errors.js +46 -0
- package/dist/vendor/serialize.d.ts +3 -0
- package/dist/vendor/serialize.d.ts.map +1 -0
- package/dist/vendor/serialize.js +25 -0
- package/dist/vendor/types.d.ts +69 -0
- package/dist/vendor/types.d.ts.map +1 -0
- package/dist/vendor/types.js +4 -0
- package/dist/workerd-facet-host.d.ts +207 -0
- package/dist/workerd-facet-host.d.ts.map +1 -0
- package/dist/workerd-facet-host.js +508 -0
- package/dist/ws-hibernation-config.d.ts +73 -0
- package/dist/ws-hibernation-config.d.ts.map +1 -0
- package/dist/ws-hibernation-config.js +93 -0
- package/package.json +62 -0
- package/src/alarms.ts +275 -0
- package/src/bindings.ts +871 -0
- package/src/ctx-exports.ts +77 -0
- package/src/facet-image-store.ts +196 -0
- package/src/fanout-pool.ts +503 -0
- package/src/index.ts +26 -0
- package/src/inner-do-registry.ts +58 -0
- package/src/launch-journal.ts +229 -0
- package/src/launch-pacer.ts +231 -0
- package/src/loader-ledger.ts +112 -0
- package/src/loader-pool.ts +984 -0
- package/src/process-fabric.ts +729 -0
- package/src/process-host.ts +566 -0
- package/src/vendor/errors.ts +56 -0
- package/src/vendor/serialize.ts +37 -0
- package/src/vendor/types.ts +75 -0
- package/src/workerd-facet-host.ts +694 -0
- package/src/ws-hibernation-config.ts +123 -0
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Two-tier fan-out primitive for work that must execute in Worker Loader
|
|
3
|
+
* facets without tripping workerd's per-DO dynamic-worker ceiling.
|
|
4
|
+
*
|
|
5
|
+
* A single Durable Object method can drive at most four concurrent
|
|
6
|
+
* Worker Loader fetches before extra dispatches serialize or fail. Small
|
|
7
|
+
* batches therefore run in the coordinator DO through LoaderPool.
|
|
8
|
+
* Wider batches are sharded across sibling NimbusSession DOs, each of
|
|
9
|
+
* which owns its own four-loader budget.
|
|
10
|
+
*
|
|
11
|
+
* Routing is deterministic: each task has a stable key, and the key maps
|
|
12
|
+
* to a sibling DO shard. There is no silent fallback to width-1 execution;
|
|
13
|
+
* missing LOADER or NIMBUS_SESSION bindings fail loudly so install and
|
|
14
|
+
* runtime operations do not appear successful after partial dispatch.
|
|
15
|
+
*/
|
|
16
|
+
import { serializeFunction } from './vendor/serialize.js';
|
|
17
|
+
import { BindingError } from './vendor/errors.js';
|
|
18
|
+
import { LoaderPool } from './loader-pool.js';
|
|
19
|
+
import { disposeRpcResource } from '@nimbus-sh/core/_shared/rpc-dispose.js';
|
|
20
|
+
import { describeError, isDoOverloaded, isTransientDoReset } from '@nimbus-sh/core/observability/oom-classify.js';
|
|
21
|
+
/**
|
|
22
|
+
* Threshold at which routing switches from coordinator-local loaders to
|
|
23
|
+
* sibling Durable Objects.
|
|
24
|
+
*
|
|
25
|
+
* Set to **5** so the in-DO path stays below the V8 4-loaders-per-method
|
|
26
|
+
* cap by construction. width < 5 stays local; width >= 5 uses sibling DOs.
|
|
27
|
+
*/
|
|
28
|
+
export const IN_DO_THRESHOLD = 5;
|
|
29
|
+
/**
|
|
30
|
+
* Hard cap on concurrent peer DOs per single submitMany call. Throughput stays
|
|
31
|
+
* flat through this width while keeping per-request scheduler pressure bounded.
|
|
32
|
+
*/
|
|
33
|
+
export const MAX_PEER_FANOUT = 32;
|
|
34
|
+
/**
|
|
35
|
+
* Bounded retries for a peer-DO shard dispatch that rejects with a
|
|
36
|
+
* transient platform reset (code roll-over, storage cold-start hiccup).
|
|
37
|
+
* Sibling DOs are addressed by stable name, so the retry re-dispatches
|
|
38
|
+
* the SAME shard to the re-provisioning object; the fanned-out work
|
|
39
|
+
* (packument resolution, tarball materialisation) is idempotent, so
|
|
40
|
+
* re-running a shard is safe. Budget mirrors the resolve-facet's own
|
|
41
|
+
* per-fetch retry policy so a single flaky cold start no longer fails a
|
|
42
|
+
* whole install. The same budget covers an overloaded peer, on the longer
|
|
43
|
+
* schedule below. Non-transient rejections (OOM, count mismatch, genuine
|
|
44
|
+
* task throw) are NOT retried — they propagate on the first hit.
|
|
45
|
+
*/
|
|
46
|
+
export const PEER_TRANSIENT_RESET_RETRIES = 3;
|
|
47
|
+
export const PEER_RETRY_BACKOFF_MS = [250, 750, 1500];
|
|
48
|
+
/**
|
|
49
|
+
* Backoff for a shard whose peer DO was shed as overloaded. The object is
|
|
50
|
+
* alive and the shard never ran; what it needs is time for the input-gate
|
|
51
|
+
* queue to drain, so the schedule is an order of magnitude longer than the
|
|
52
|
+
* reset schedule. A whole-batch abort here used to fail an entire install.
|
|
53
|
+
*/
|
|
54
|
+
export const PEER_OVERLOAD_BACKOFF_MS = [1000, 3000, 6000];
|
|
55
|
+
/**
|
|
56
|
+
* Peer shards dispatched per phase. Each phase is a barrier that costs its
|
|
57
|
+
* slowest member, so a wide fan-out pays ⌈shards / FANOUT_PHASE_SIZE⌉ serial
|
|
58
|
+
* round-trips; the size trades that serialization against simultaneous cold
|
|
59
|
+
* sibling DO starts.
|
|
60
|
+
*
|
|
61
|
+
* The six-barrier profile once measured on a 123-package install (21 shards of
|
|
62
|
+
* ~6 packages, 10.6/6.8/21.8/7.9/34.6/4.8 s) came from the shard count, not
|
|
63
|
+
* from this width. Capping install shards at INSTALL_PEER_CAP fixed it at the
|
|
64
|
+
* source and that install now clears in two phases. Widening to 8 on top of
|
|
65
|
+
* that bought one further barrier and doubled the simultaneous cold sibling-DO
|
|
66
|
+
* starts, which is the account-level pressure the phasing exists for: twelve
|
|
67
|
+
* concurrent Markflow installs went from 48 simultaneous peer starts to 96 and
|
|
68
|
+
* began timing out. Phasing does not change how many peers start, only how
|
|
69
|
+
* many start at once, so this width is set by the burst the scheduler
|
|
70
|
+
* tolerates rather than by the barrier count.
|
|
71
|
+
*/
|
|
72
|
+
export const FANOUT_PHASE_SIZE = 4;
|
|
73
|
+
function isFanoutPeerStub(value) {
|
|
74
|
+
if ((typeof value !== 'object' && typeof value !== 'function') || value === null) {
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
const execute = Reflect.get(value, '_rpcFanoutExecute');
|
|
78
|
+
return typeof execute === 'function';
|
|
79
|
+
}
|
|
80
|
+
function fanoutPeerStub(value) {
|
|
81
|
+
if ((typeof value !== 'object' && typeof value !== 'function') || value === null) {
|
|
82
|
+
throw new BindingError('FanoutPool: NIMBUS_SESSION.get() did not return a peer stub.');
|
|
83
|
+
}
|
|
84
|
+
if (!isFanoutPeerStub(value)) {
|
|
85
|
+
throw new BindingError('FanoutPool: peer stub does not expose _rpcFanoutExecute().');
|
|
86
|
+
}
|
|
87
|
+
return value;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Two-tier fan-out pool. Constructed by the supervisor DO; routes
|
|
91
|
+
* each `submitMany` call automatically based on width.
|
|
92
|
+
*
|
|
93
|
+
* Lifetime: cheap to construct (no async init). Multiple submitMany
|
|
94
|
+
* calls share NO state — each is dispatched fresh. The class
|
|
95
|
+
* exists primarily as a clean API surface; per-call dispatch state
|
|
96
|
+
* lives only inside submitMany's promise.
|
|
97
|
+
*/
|
|
98
|
+
export class FanoutPool {
|
|
99
|
+
env;
|
|
100
|
+
ctx;
|
|
101
|
+
opts;
|
|
102
|
+
coordDoId;
|
|
103
|
+
coordDoIdShort;
|
|
104
|
+
constructor(rawEnv, ctx, opts) {
|
|
105
|
+
// A host hands its whole env over; the bindings are claimed here and the
|
|
106
|
+
// LOADER claim is checked immediately. Hard-fail on a missing LOADER —
|
|
107
|
+
// LoaderPool also enforces this, but checking up front points the
|
|
108
|
+
// diagnostic at the fanout-pool construction site rather than the
|
|
109
|
+
// deferred loader-pool one.
|
|
110
|
+
const env = rawEnv ?? {};
|
|
111
|
+
if (!env.LOADER || typeof env.LOADER.get !== 'function') {
|
|
112
|
+
throw new BindingError('FanoutPool: env.LOADER binding missing or invalid. ' +
|
|
113
|
+
'Add a [[worker_loaders]] entry to wrangler.jsonc.');
|
|
114
|
+
}
|
|
115
|
+
this.env = env;
|
|
116
|
+
this.ctx = ctx;
|
|
117
|
+
this.opts = opts;
|
|
118
|
+
this.coordDoId = ctx.id.toString();
|
|
119
|
+
this.coordDoIdShort = this.coordDoId.slice(0, 12);
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Dispatch `tasks` across the appropriate topology and return
|
|
123
|
+
* results in input order.
|
|
124
|
+
*
|
|
125
|
+
* Routing:
|
|
126
|
+
* tasks.length < 5 -> coordinator-local LoaderPool
|
|
127
|
+
* tasks.length >= 5 -> sibling NimbusSession DOs
|
|
128
|
+
*
|
|
129
|
+
* Backpressure: if `tasks.length > MAX_PEER_FANOUT (32)`, tasks
|
|
130
|
+
* are sharded modulo `MAX_PEER_FANOUT` and each shard's bucket
|
|
131
|
+
* runs serially inside its assigned peer DO via the in-peer
|
|
132
|
+
* LoaderPool's concurrency (capped at 4 there too). A
|
|
133
|
+
* single submitMany call returns when ALL tasks complete (or any
|
|
134
|
+
* throws).
|
|
135
|
+
*
|
|
136
|
+
* `fn` is the user function executed per task. It runs INSIDE a
|
|
137
|
+
* Worker Loader isolate (in the in-DO path) or inside a peer DO's
|
|
138
|
+
* Worker Loader isolate (in the peer-DO path); same trust posture
|
|
139
|
+
* as LoaderPool.submit. The function is serialized via
|
|
140
|
+
* the vendored serializeFunction (same as LoaderPool#prepare).
|
|
141
|
+
*/
|
|
142
|
+
async submitMany(tasks, fn) {
|
|
143
|
+
if (tasks.length === 0)
|
|
144
|
+
return [];
|
|
145
|
+
if (tasks.length < IN_DO_THRESHOLD) {
|
|
146
|
+
return this._dispatchInDo(tasks, fn);
|
|
147
|
+
}
|
|
148
|
+
return this._dispatchPeerDo(tasks, fn);
|
|
149
|
+
}
|
|
150
|
+
/** Report which topology a task count uses without dispatching. */
|
|
151
|
+
topologyFor(taskCount) {
|
|
152
|
+
if (taskCount === 0)
|
|
153
|
+
return 'empty';
|
|
154
|
+
return taskCount < IN_DO_THRESHOLD ? 'in-do' : 'peer-do';
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Compute the deterministic peer-DO id for a task key and peer count.
|
|
158
|
+
*
|
|
159
|
+
* Shape: `nbf:${tag}:${coordDoIdShort}:${shard}` where
|
|
160
|
+
* `shard = hash(key) mod peerCount`. Peer count is
|
|
161
|
+
* `min(tasks.length, MAX_PEER_FANOUT)`.
|
|
162
|
+
*/
|
|
163
|
+
peerSiblingId(key, peerCount) {
|
|
164
|
+
const shard = hashKeyToShard(key, peerCount);
|
|
165
|
+
return `nbf:${this.opts.tag}:${this.coordDoIdShort}:${shard}`;
|
|
166
|
+
}
|
|
167
|
+
// ── Private: in-DO dispatch (in-DO fanout) ──────────────────────────────
|
|
168
|
+
async _dispatchInDo(tasks, fn) {
|
|
169
|
+
// Use the existing LoaderPool. Concurrency = task count
|
|
170
|
+
// (capped at 4 by constructor — tasks.length is already < 5
|
|
171
|
+
// here, so the cap won't bite). Each task = one pool.submit;
|
|
172
|
+
// pool.map runs them with stable-slot reuse.
|
|
173
|
+
const concurrency = Math.min(tasks.length, IN_DO_THRESHOLD - 1);
|
|
174
|
+
const pool = new LoaderPool(this.env, this.ctx, {
|
|
175
|
+
concurrency,
|
|
176
|
+
timeoutMs: this.opts.timeoutMs,
|
|
177
|
+
tag: this.opts.tag,
|
|
178
|
+
preamble: this.opts.preamble,
|
|
179
|
+
wasmModules: this.opts.wasmModules,
|
|
180
|
+
extraBindings: this.opts.extraBindings,
|
|
181
|
+
omitSupervisor: this.opts.omitSupervisor,
|
|
182
|
+
supervisorPid: this.opts.supervisorPid,
|
|
183
|
+
});
|
|
184
|
+
try {
|
|
185
|
+
// pool.map runs the function over `items` with concurrency-bounded
|
|
186
|
+
// slot reuse. Each slot is one warm loader isolate; we get exactly
|
|
187
|
+
// `concurrency` loader isolates total — well under the 4-cap.
|
|
188
|
+
const items = tasks.map((t) => t.args);
|
|
189
|
+
const results = await pool.map(fn, items);
|
|
190
|
+
// pool.map returns Array<R | null> (null on per-item failure with
|
|
191
|
+
// onError='null'/'skip'). Default onError='throw' rejects on
|
|
192
|
+
// first failure, so successful settle here implies all R values.
|
|
193
|
+
return results;
|
|
194
|
+
}
|
|
195
|
+
finally {
|
|
196
|
+
try {
|
|
197
|
+
pool.dispose();
|
|
198
|
+
}
|
|
199
|
+
catch { /* best-effort */ }
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
// ── Private: peer-DO dispatch (peer-DO fanout) ────────────────────────────
|
|
203
|
+
async _dispatchPeerDo(tasks, fn) {
|
|
204
|
+
const ns = this.env?.NIMBUS_SESSION;
|
|
205
|
+
if (!ns || typeof ns.idFromName !== 'function' || typeof ns.get !== 'function') {
|
|
206
|
+
throw new BindingError('FanoutPool: env.NIMBUS_SESSION binding missing or invalid. ' +
|
|
207
|
+
'The peer-DO topology requires it. ' +
|
|
208
|
+
'Add the binding via durable_objects.bindings in wrangler.jsonc.');
|
|
209
|
+
}
|
|
210
|
+
// Serialize the user function ONCE here on the supervisor side.
|
|
211
|
+
// Each peer DO receives the same fnSource string; warm peer
|
|
212
|
+
// loader isolates (keyed on fnHash) reuse across calls with
|
|
213
|
+
// identical fns.
|
|
214
|
+
const fnSource = serializeFunction(fn);
|
|
215
|
+
// Cap peer count at MAX_PEER_FANOUT. Tasks beyond N=32 are
|
|
216
|
+
// bucketed into existing shards — each shard's peer DO then
|
|
217
|
+
// runs its bucket through its in-DO LoaderPool.map
|
|
218
|
+
// (concurrency capped at 4 there).
|
|
219
|
+
const peerCount = Math.min(tasks.length, this.opts.maxPeers ?? MAX_PEER_FANOUT);
|
|
220
|
+
// Group tasks by deterministic shard. Same key → same shard, so
|
|
221
|
+
// tests can predict which peer handles which task.
|
|
222
|
+
const shards = new Map();
|
|
223
|
+
for (const t of tasks) {
|
|
224
|
+
const shard = hashKeyToShard(t.key, peerCount);
|
|
225
|
+
let bucket = shards.get(shard);
|
|
226
|
+
if (!bucket) {
|
|
227
|
+
bucket = [];
|
|
228
|
+
shards.set(shard, bucket);
|
|
229
|
+
}
|
|
230
|
+
bucket.push(t);
|
|
231
|
+
}
|
|
232
|
+
// Dispatch each shard to its peer DO. Build a map from
|
|
233
|
+
// task → its place in the original tasks array so we can
|
|
234
|
+
// reassemble results in input order.
|
|
235
|
+
const taskIndex = new Map();
|
|
236
|
+
tasks.forEach((t, i) => {
|
|
237
|
+
taskIndex.set(t, i);
|
|
238
|
+
});
|
|
239
|
+
const results = new Array(tasks.length);
|
|
240
|
+
// Build one async dispatcher per shard (closure capturing siblingName,
|
|
241
|
+
// bucket). NOT eagerly-started — wrapped in a thunk so we can stagger
|
|
242
|
+
// dispatch via Promise chains without forcing all shards to start
|
|
243
|
+
// simultaneously.
|
|
244
|
+
const dispatchers = [];
|
|
245
|
+
for (const [shard, bucket] of shards) {
|
|
246
|
+
const siblingName = `nbf:${this.opts.tag}:${this.coordDoIdShort}:${shard}`;
|
|
247
|
+
const id = ns.idFromName(siblingName);
|
|
248
|
+
const peerArgs = bucket.map((t) => t.args);
|
|
249
|
+
dispatchers.push(async () => {
|
|
250
|
+
for (let attempt = 0;; attempt++) {
|
|
251
|
+
// Fresh stub per attempt: after a transient reset the previous
|
|
252
|
+
// stub points at a torn-down object, so a retry re-resolves the
|
|
253
|
+
// sibling by its stable id.
|
|
254
|
+
const peerStub = ns.get(id);
|
|
255
|
+
const stub = fanoutPeerStub(peerStub);
|
|
256
|
+
try {
|
|
257
|
+
// Each peer DO RPC call uses ONE LOADER worker on its side.
|
|
258
|
+
// Supervisor → peer DO is a stub.fetch / RPC method call,
|
|
259
|
+
// NOT an env.LOADER.get(); that's the cap-sidestep that
|
|
260
|
+
// makes peer-DO fanout work.
|
|
261
|
+
const rpcResp = await stub._rpcFanoutExecute(fnSource, peerArgs, {
|
|
262
|
+
tag: this.opts.tag,
|
|
263
|
+
timeoutMs: this.opts.timeoutMs,
|
|
264
|
+
preamble: this.opts.preamble,
|
|
265
|
+
wasmModules: this.opts.wasmModules,
|
|
266
|
+
extraBindings: this.opts.extraBindings,
|
|
267
|
+
omitSupervisor: this.opts.omitSupervisor,
|
|
268
|
+
// INSTALL-HONESTY: forward the COORDINATOR's full doId so
|
|
269
|
+
// the peer's LoaderPool can mint a SUPERVISOR
|
|
270
|
+
// binding that routes back HERE (the user's session DO),
|
|
271
|
+
// not to the peer DO itself. Without this, peer DOs'
|
|
272
|
+
// env.SUPERVISOR.writeBatch / writeBatchStream / stdout /
|
|
273
|
+
// ... write into the peer's own VFS — invisible to the
|
|
274
|
+
// user. See INSTALL-HONESTY-retro.md.
|
|
275
|
+
coordinatorDoId: this.coordDoId,
|
|
276
|
+
// Credential source for peer-side writeBatchStream — the
|
|
277
|
+
// invoking process pid, so package writes are authorized
|
|
278
|
+
// as the user (not rejected as pid:0).
|
|
279
|
+
supervisorPid: this.opts.supervisorPid,
|
|
280
|
+
});
|
|
281
|
+
try {
|
|
282
|
+
const peerResults = rpcResp.results ?? [];
|
|
283
|
+
if (peerResults.length !== bucket.length) {
|
|
284
|
+
throw new Error(`peer DO returned ${peerResults.length} results for ${bucket.length} tasks ` +
|
|
285
|
+
`(siblingName=${siblingName})`);
|
|
286
|
+
}
|
|
287
|
+
// Place each result back into its original input slot.
|
|
288
|
+
for (let i = 0; i < bucket.length; i++) {
|
|
289
|
+
const origIdx = taskIndex.get(bucket[i]);
|
|
290
|
+
if (origIdx === undefined) {
|
|
291
|
+
throw new Error(`peer DO result had no original task index (siblingName=${siblingName})`);
|
|
292
|
+
}
|
|
293
|
+
results[origIdx] = peerResults[i];
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
finally {
|
|
297
|
+
disposeRpcResource(rpcResp);
|
|
298
|
+
}
|
|
299
|
+
return;
|
|
300
|
+
}
|
|
301
|
+
catch (err) {
|
|
302
|
+
const schedule = isTransientDoReset(err) ? PEER_RETRY_BACKOFF_MS
|
|
303
|
+
: isDoOverloaded(err) ? PEER_OVERLOAD_BACKOFF_MS
|
|
304
|
+
: null;
|
|
305
|
+
if (schedule && attempt < PEER_TRANSIENT_RESET_RETRIES) {
|
|
306
|
+
const backoff = schedule[Math.min(attempt, schedule.length - 1)];
|
|
307
|
+
await new Promise((r) => setTimeout(r, backoff));
|
|
308
|
+
continue;
|
|
309
|
+
}
|
|
310
|
+
// Re-throwing bare loses everything only this frame knows: which
|
|
311
|
+
// sibling ran the shard, how wide it was, and how many attempts
|
|
312
|
+
// it already cost. Callers report the message, so a rejection the
|
|
313
|
+
// platform words as `internal error` arrived at the user with no
|
|
314
|
+
// way to tell a one-off peer from a shard that had exhausted its
|
|
315
|
+
// retries. `cause` keeps the original for anything that inspects
|
|
316
|
+
// errors rather than reads them.
|
|
317
|
+
throw new Error(`peer shard ${siblingName} (${bucket.length} task${bucket.length === 1 ? '' : 's'}) `
|
|
318
|
+
+ `failed after ${attempt + 1} attempt${attempt === 0 ? '' : 's'}: ${describeError(err)}`, { cause: err });
|
|
319
|
+
}
|
|
320
|
+
finally {
|
|
321
|
+
disposeRpcResource(peerStub);
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
});
|
|
325
|
+
}
|
|
326
|
+
// Dispatch peer shards in bounded phases. A single Promise.all across all
|
|
327
|
+
// shards can create too many simultaneous cold sibling DO starts under
|
|
328
|
+
// concurrent installs. Promise-chain phasing limits scheduler pressure
|
|
329
|
+
// without sleeps, timers, or idle gaps between phases.
|
|
330
|
+
//
|
|
331
|
+
// A phase is a hard barrier: no shard in phase N+1 starts until every
|
|
332
|
+
// shard in phase N returns, so `width@ms` per phase is what separates
|
|
333
|
+
// "the shards are slow" from "there are too many barriers".
|
|
334
|
+
for (let i = 0; i < dispatchers.length; i += FANOUT_PHASE_SIZE) {
|
|
335
|
+
const phase = dispatchers.slice(i, i + FANOUT_PHASE_SIZE);
|
|
336
|
+
const phaseStartedAt = Date.now();
|
|
337
|
+
await Promise.all(phase.map((d) => d()));
|
|
338
|
+
this.opts.onDispatchPhase?.(phase.length, Date.now() - phaseStartedAt);
|
|
339
|
+
}
|
|
340
|
+
return results;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
/**
|
|
344
|
+
* Stable hash → shard. Uses a fresh djb2 over the key (NOT
|
|
345
|
+
* hashSource) and modulos by peerCount.
|
|
346
|
+
*
|
|
347
|
+
* Why not reuse hashSource: hashSource returns a base-36 string,
|
|
348
|
+
* NOT hex — its alphabet is `[0-9a-z]`. parseInt(str, 16) on a
|
|
349
|
+
* base-36 string aborts at the first non-hex char (any of g-z),
|
|
350
|
+
* which produces extremely poor distribution: keys with the same
|
|
351
|
+
* leading-hex-prefix collide regardless of their suffix. (Seen in
|
|
352
|
+
* the wild: `task-0 .. task-7` all collided onto shard 4.)
|
|
353
|
+
*
|
|
354
|
+
* Deterministic: same key + same peerCount → same shard, every run.
|
|
355
|
+
* Tests use this to predict placement.
|
|
356
|
+
*/
|
|
357
|
+
export function hashKeyToShard(key, peerCount) {
|
|
358
|
+
if (peerCount <= 1)
|
|
359
|
+
return 0;
|
|
360
|
+
// djb2, returning an unsigned 32-bit integer — full 2^32 range,
|
|
361
|
+
// no string-format conversion gotchas. peerCount <= MAX_PEER_FANOUT
|
|
362
|
+
// (32) << 2^32, so the modulo distributes uniformly for any input.
|
|
363
|
+
let h = 5381;
|
|
364
|
+
for (let i = 0; i < key.length; i++) {
|
|
365
|
+
h = ((h << 5) + h + key.charCodeAt(i)) | 0;
|
|
366
|
+
}
|
|
367
|
+
return (h >>> 0) % peerCount;
|
|
368
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @nimbus-sh/fabric — the Cloudflare-specific DO/facet machinery of Nimbus.
|
|
3
|
+
*
|
|
4
|
+
* Root export for workerd contexts. `bindings.ts` imports
|
|
5
|
+
* `cloudflare:workers`, so importing this root module outside workerd fails
|
|
6
|
+
* at resolution; non-workerd consumers (tests, tooling) import the subpath
|
|
7
|
+
* modules they need instead.
|
|
8
|
+
*/
|
|
9
|
+
export * from './alarms.js';
|
|
10
|
+
export * from './bindings.js';
|
|
11
|
+
export * from './ctx-exports.js';
|
|
12
|
+
export * from './facet-image-store.js';
|
|
13
|
+
export * from './fanout-pool.js';
|
|
14
|
+
export * from './inner-do-registry.js';
|
|
15
|
+
export * from './launch-journal.js';
|
|
16
|
+
export * from './launch-pacer.js';
|
|
17
|
+
export * from './loader-ledger.js';
|
|
18
|
+
export * from './loader-pool.js';
|
|
19
|
+
export * from './process-fabric.js';
|
|
20
|
+
export * from './process-host.js';
|
|
21
|
+
export * from './workerd-facet-host.js';
|
|
22
|
+
export * from './ws-hibernation-config.js';
|
|
23
|
+
export * from './vendor/errors.js';
|
|
24
|
+
export * from './vendor/serialize.js';
|
|
25
|
+
export * from './vendor/types.js';
|
|
26
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,cAAc,aAAa,CAAC;AAC5B,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,wBAAwB,CAAC;AACvC,cAAc,kBAAkB,CAAC;AACjC,cAAc,wBAAwB,CAAC;AACvC,cAAc,qBAAqB,CAAC;AACpC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,qBAAqB,CAAC;AACpC,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,oBAAoB,CAAC;AACnC,cAAc,uBAAuB,CAAC;AACtC,cAAc,mBAAmB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @nimbus-sh/fabric — the Cloudflare-specific DO/facet machinery of Nimbus.
|
|
3
|
+
*
|
|
4
|
+
* Root export for workerd contexts. `bindings.ts` imports
|
|
5
|
+
* `cloudflare:workers`, so importing this root module outside workerd fails
|
|
6
|
+
* at resolution; non-workerd consumers (tests, tooling) import the subpath
|
|
7
|
+
* modules they need instead.
|
|
8
|
+
*/
|
|
9
|
+
export * from './alarms.js';
|
|
10
|
+
export * from './bindings.js';
|
|
11
|
+
export * from './ctx-exports.js';
|
|
12
|
+
export * from './facet-image-store.js';
|
|
13
|
+
export * from './fanout-pool.js';
|
|
14
|
+
export * from './inner-do-registry.js';
|
|
15
|
+
export * from './launch-journal.js';
|
|
16
|
+
export * from './launch-pacer.js';
|
|
17
|
+
export * from './loader-ledger.js';
|
|
18
|
+
export * from './loader-pool.js';
|
|
19
|
+
export * from './process-fabric.js';
|
|
20
|
+
export * from './process-host.js';
|
|
21
|
+
export * from './workerd-facet-host.js';
|
|
22
|
+
export * from './ws-hibernation-config.js';
|
|
23
|
+
export * from './vendor/errors.js';
|
|
24
|
+
export * from './vendor/serialize.js';
|
|
25
|
+
export * from './vendor/types.js';
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* inner-do-registry.ts — Module-level registry of inner DO classes.
|
|
3
|
+
*
|
|
4
|
+
* `nimbus-wrangler` populates this on each successful buildAndLoad()
|
|
5
|
+
* with the DurableObject classes extracted from the freshly-loaded
|
|
6
|
+
* inner Worker via `worker.getDurableObjectClass(name)`. The supervisor
|
|
7
|
+
* DO (`NimbusSession`) reads it on each inner-DO fetch to synthesize a
|
|
8
|
+
* NimbusDurableObjectNamespace stub for `env.MY_DO`.
|
|
9
|
+
*
|
|
10
|
+
* Why a separate leaf module:
|
|
11
|
+
* Before this extraction, the registry lived inside nimbus-session.ts,
|
|
12
|
+
* which forced nimbus-wrangler.ts to import from nimbus-session.ts —
|
|
13
|
+
* producing the cycle
|
|
14
|
+
* index.ts -> nimbus-session.ts -> nimbus-wrangler.ts -> nimbus-session.ts
|
|
15
|
+
* Promoting the registry to its own leaf breaks that cycle without
|
|
16
|
+
* changing any semantics. The Map identity is preserved across the
|
|
17
|
+
* isolate (it's still process-scoped — module-level state — survives
|
|
18
|
+
* across DO instances in the same workerd process).
|
|
19
|
+
*
|
|
20
|
+
* Key shape:
|
|
21
|
+
* `<supervisor-DO-id>:<binding-name>` — both halves are required to
|
|
22
|
+
* prevent multiple supervisor DOs in the same isolate from clobbering
|
|
23
|
+
* each other's registrations.
|
|
24
|
+
*
|
|
25
|
+
* The values are the inner worker's own Durable Object classes, as
|
|
26
|
+
* `worker.getDurableObjectClass(name)` hands them over: opaque tokens whose
|
|
27
|
+
* only use is `ctx.facets.get(name, { class: cls, id })`, which is exactly what
|
|
28
|
+
* workerd's `DurableObjectClass` models.
|
|
29
|
+
*/
|
|
30
|
+
/** Look up a registered inner-DO class. Returns undefined if not found. */
|
|
31
|
+
export declare function getInnerDoClass(supervisorDoId: string, bindingName: string): DurableObjectClass | undefined;
|
|
32
|
+
/**
|
|
33
|
+
* Register an inner DO class for synthesis. Called by nimbus-wrangler.ts
|
|
34
|
+
* after each successful buildAndLoad() with the class extracted from
|
|
35
|
+
* the fresh worker stub. Keys are `<doId>:<bindingName>` so multiple
|
|
36
|
+
* supervisor DOs don't collide.
|
|
37
|
+
*/
|
|
38
|
+
export declare function registerInnerDoClass(supervisorDoId: string, bindingName: string, cls: DurableObjectClass): void;
|
|
39
|
+
/** Clear all registrations belonging to a supervisor DO (called on rebuild). */
|
|
40
|
+
export declare function clearInnerDoClasses(supervisorDoId: string): void;
|
|
41
|
+
//# sourceMappingURL=inner-do-registry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inner-do-registry.d.ts","sourceRoot":"","sources":["../src/inner-do-registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAIH,2EAA2E;AAC3E,wBAAgB,eAAe,CAAC,cAAc,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,kBAAkB,GAAG,SAAS,CAE3G;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAClC,cAAc,EAAE,MAAM,EACtB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,kBAAkB,GACtB,IAAI,CAEN;AAED,gFAAgF;AAChF,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAKhE"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* inner-do-registry.ts — Module-level registry of inner DO classes.
|
|
3
|
+
*
|
|
4
|
+
* `nimbus-wrangler` populates this on each successful buildAndLoad()
|
|
5
|
+
* with the DurableObject classes extracted from the freshly-loaded
|
|
6
|
+
* inner Worker via `worker.getDurableObjectClass(name)`. The supervisor
|
|
7
|
+
* DO (`NimbusSession`) reads it on each inner-DO fetch to synthesize a
|
|
8
|
+
* NimbusDurableObjectNamespace stub for `env.MY_DO`.
|
|
9
|
+
*
|
|
10
|
+
* Why a separate leaf module:
|
|
11
|
+
* Before this extraction, the registry lived inside nimbus-session.ts,
|
|
12
|
+
* which forced nimbus-wrangler.ts to import from nimbus-session.ts —
|
|
13
|
+
* producing the cycle
|
|
14
|
+
* index.ts -> nimbus-session.ts -> nimbus-wrangler.ts -> nimbus-session.ts
|
|
15
|
+
* Promoting the registry to its own leaf breaks that cycle without
|
|
16
|
+
* changing any semantics. The Map identity is preserved across the
|
|
17
|
+
* isolate (it's still process-scoped — module-level state — survives
|
|
18
|
+
* across DO instances in the same workerd process).
|
|
19
|
+
*
|
|
20
|
+
* Key shape:
|
|
21
|
+
* `<supervisor-DO-id>:<binding-name>` — both halves are required to
|
|
22
|
+
* prevent multiple supervisor DOs in the same isolate from clobbering
|
|
23
|
+
* each other's registrations.
|
|
24
|
+
*
|
|
25
|
+
* The values are the inner worker's own Durable Object classes, as
|
|
26
|
+
* `worker.getDurableObjectClass(name)` hands them over: opaque tokens whose
|
|
27
|
+
* only use is `ctx.facets.get(name, { class: cls, id })`, which is exactly what
|
|
28
|
+
* workerd's `DurableObjectClass` models.
|
|
29
|
+
*/
|
|
30
|
+
const _NIMBUS_INNER_DO_CLASSES = new Map();
|
|
31
|
+
/** Look up a registered inner-DO class. Returns undefined if not found. */
|
|
32
|
+
export function getInnerDoClass(supervisorDoId, bindingName) {
|
|
33
|
+
return _NIMBUS_INNER_DO_CLASSES.get(supervisorDoId + ':' + bindingName);
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Register an inner DO class for synthesis. Called by nimbus-wrangler.ts
|
|
37
|
+
* after each successful buildAndLoad() with the class extracted from
|
|
38
|
+
* the fresh worker stub. Keys are `<doId>:<bindingName>` so multiple
|
|
39
|
+
* supervisor DOs don't collide.
|
|
40
|
+
*/
|
|
41
|
+
export function registerInnerDoClass(supervisorDoId, bindingName, cls) {
|
|
42
|
+
_NIMBUS_INNER_DO_CLASSES.set(supervisorDoId + ':' + bindingName, cls);
|
|
43
|
+
}
|
|
44
|
+
/** Clear all registrations belonging to a supervisor DO (called on rebuild). */
|
|
45
|
+
export function clearInnerDoClasses(supervisorDoId) {
|
|
46
|
+
const prefix = supervisorDoId + ':';
|
|
47
|
+
for (const k of _NIMBUS_INNER_DO_CLASSES.keys()) {
|
|
48
|
+
if (k.startsWith(prefix))
|
|
49
|
+
_NIMBUS_INNER_DO_CLASSES.delete(k);
|
|
50
|
+
}
|
|
51
|
+
}
|