@nimbus-sh/fabric 0.7.1 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/README.md +26 -11
  2. package/dist/bindings.d.ts +1 -0
  3. package/dist/bindings.d.ts.map +1 -1
  4. package/dist/bindings.js +2 -0
  5. package/dist/budgets.d.ts +50 -29
  6. package/dist/budgets.d.ts.map +1 -1
  7. package/dist/budgets.js +88 -40
  8. package/dist/connections.d.ts +1 -1
  9. package/dist/connections.js +1 -1
  10. package/dist/do-calls.d.ts +91 -9
  11. package/dist/do-calls.d.ts.map +1 -1
  12. package/dist/do-calls.js +161 -22
  13. package/dist/facet-pool.d.ts +3 -0
  14. package/dist/facet-pool.d.ts.map +1 -1
  15. package/dist/facet-pool.js +3 -0
  16. package/dist/fanout.d.ts +40 -32
  17. package/dist/fanout.d.ts.map +1 -1
  18. package/dist/fanout.js +50 -53
  19. package/dist/fenced-work.d.ts +3 -3
  20. package/dist/host-wasm.d.ts +29 -0
  21. package/dist/host-wasm.d.ts.map +1 -0
  22. package/dist/host-wasm.js +31 -0
  23. package/dist/image-store.d.ts +1 -1
  24. package/dist/image-store.d.ts.map +1 -1
  25. package/dist/image-store.js +34 -2
  26. package/dist/inner-do-registry.d.ts +9 -0
  27. package/dist/inner-do-registry.d.ts.map +1 -1
  28. package/dist/inner-do-registry.js +35 -0
  29. package/dist/isolate-pool.d.ts +32 -24
  30. package/dist/isolate-pool.d.ts.map +1 -1
  31. package/dist/isolate-pool.js +78 -54
  32. package/dist/process-fabric.d.ts +42 -23
  33. package/dist/process-fabric.d.ts.map +1 -1
  34. package/dist/process-fabric.js +62 -6
  35. package/dist/process-host.d.ts +3 -1
  36. package/dist/process-host.d.ts.map +1 -1
  37. package/dist/process-host.js +13 -14
  38. package/dist/supervisor-props.d.ts +47 -0
  39. package/dist/supervisor-props.d.ts.map +1 -0
  40. package/dist/supervisor-props.js +37 -0
  41. package/dist/timers.d.ts +12 -0
  42. package/dist/timers.d.ts.map +1 -1
  43. package/dist/timers.js +44 -9
  44. package/dist/vendor/types.d.ts +11 -5
  45. package/dist/vendor/types.d.ts.map +1 -1
  46. package/dist/workerd-facet-host.d.ts +4 -17
  47. package/dist/workerd-facet-host.d.ts.map +1 -1
  48. package/dist/workerd-facet-host.js +120 -54
  49. package/package.json +6 -6
  50. package/src/bindings.ts +2 -0
  51. package/src/budgets.ts +96 -49
  52. package/src/connections.ts +1 -1
  53. package/src/do-calls.ts +216 -25
  54. package/src/facet-pool.ts +5 -0
  55. package/src/fanout.ts +64 -56
  56. package/src/fenced-work.ts +3 -3
  57. package/src/host-wasm.ts +41 -0
  58. package/src/image-store.ts +30 -3
  59. package/src/inner-do-registry.ts +31 -0
  60. package/src/isolate-pool.ts +108 -74
  61. package/src/process-fabric.ts +88 -30
  62. package/src/process-host.ts +16 -14
  63. package/src/supervisor-props.ts +56 -0
  64. package/src/timers.ts +45 -9
  65. package/src/vendor/types.ts +11 -5
  66. package/src/workerd-facet-host.ts +123 -58
package/src/budgets.ts CHANGED
@@ -1,39 +1,46 @@
1
1
  /**
2
2
  * budgets.ts — per-DO accounting for the platform budgets the fabric spends:
3
- * the Worker Loader's two caps, the facet-ID lifetime budget, and the
4
- * dynamic-worker module-map ceiling.
3
+ * the Durable Object's Dynamic Worker concurrency limit, the facet-ID
4
+ * lifetime budget, and the dynamic-worker module-map ceiling.
5
5
  *
6
- * Measured on production workerd: a Durable Object admits ~5–6 concurrent
7
- * dynamic workers before the platform refuses with "Too many concurrent
8
- * dynamic workers", one DO method can drive at most 4 concurrent Loader
9
- * fetches, and loader-cache entries are never released — every DISTINCT
10
- * `loader.get(id)` permanently consumes one of the dynamic-worker slots for
11
- * the object's lifetime. Nimbus stays under the caps by construction
12
- * (`IN_DO_THRESHOLD` = 5 in the fanout pool), which until now meant the slots
13
- * were counted in prose. This ledger counts them at the fabric's loader call
14
- * sites instead — the loader pool's slots, a resident process's keyed worker,
15
- * a one-shot's load — so proximity is measurable and a cap failure can name
16
- * the ids actually holding slots.
6
+ * The Dynamic Worker model is Cloudflare's documented one
7
+ * ({@link DO_DYNAMIC_WORKER_LIMIT}): a Durable Object may have a fixed number
8
+ * of DISTINCT Dynamic Workers with in-flight requests at once, shared across
9
+ * every concurrent request to that object (one I/O context), and any number
10
+ * of in-flight requests to the same Dynamic Worker count as one. Only
11
+ * in-flight requests count: a loader id with nothing in flight holds nothing.
17
12
  *
18
- * Measurement only: no admission control. The caps are the platform's, they
19
- * are approximate ("~5–6"), and a gate on an approximate number would refuse
20
- * work the platform would have run.
13
+ * The ledger counts, per hosting actor, the distinct workers that are in
14
+ * flight right now, keyed by loader id (a fresh key per unkeyed `load`), plus
15
+ * the width fan-outs have claimed and not yet released. A fan-out spends only
16
+ * the {@link dynamicWorkerHeadroom} that leaves, so work a Durable Object
17
+ * already has in flight — a resident process, the esbuild facet, a git
18
+ * network op, another fan-out — keeps its slots.
21
19
  *
22
20
  * Keyed weakly off the hosting actor's `ctx`, like the facet slot books: the
23
- * caps are per Durable Object, and dynamic workers die with the isolate that
21
+ * limit is per Durable Object, and dynamic workers die with the isolate that
24
22
  * loaded them, so a ledger that goes away with its host describes nothing
25
23
  * that still exists.
26
24
  */
27
25
 
28
26
  import { classifyError } from '@nimbus-sh/platform/oom-classify.js';
27
+ import { hostWasmIdentity } from './host-wasm.js';
28
+
29
+ /**
30
+ * Distinct Dynamic Workers one Durable Object may have with in-flight
31
+ * requests at once, shared across all concurrent requests to that object;
32
+ * multiple in-flight requests to one Dynamic Worker count once.
33
+ * https://developers.cloudflare.com/changelog/post/2026-08-28-durable-objects-dynamic-workers-limit/
34
+ */
35
+ export const DO_DYNAMIC_WORKER_LIMIT = 10;
29
36
 
30
37
  interface LoaderLedger {
31
- /** Distinct loader ids ever gotten — each one a permanently consumed slot. */
32
- ids: Set<string>;
33
- /** In-flight calls into dynamic workers, right now. */
34
- liveFetches: number;
35
- /** The most that were ever in flight at once. */
36
- peakLiveFetches: number;
38
+ /** Loader id → open holds on it. A key is present only while held. */
39
+ inFlight: Map<string, number>;
40
+ /** Width claimed by fan-outs that have not released it. */
41
+ claimed: number;
42
+ /** The most distinct workers (holds plus claims) ever counted at once. */
43
+ peak: number;
37
44
  }
38
45
 
39
46
  const ledgers = new WeakMap<object, LoaderLedger>();
@@ -41,20 +48,21 @@ const ledgers = new WeakMap<object, LoaderLedger>();
41
48
  function ledger(ctx: object): LoaderLedger {
42
49
  let entry = ledgers.get(ctx);
43
50
  if (!entry) {
44
- entry = { ids: new Set(), liveFetches: 0, peakLiveFetches: 0 };
51
+ entry = { inFlight: new Map(), claimed: 0, peak: 0 };
45
52
  ledgers.set(ctx, entry);
46
53
  }
47
54
  return entry;
48
55
  }
49
56
 
50
- /** Record a keyed `loader.get(id)` — a permanent slot if the id is new. */
51
- export function recordLoaderId(ctx: object, id: string): void {
52
- ledger(ctx).ids.add(id);
57
+ function inUse(entry: LoaderLedger): number {
58
+ return entry.inFlight.size + entry.claimed;
53
59
  }
54
60
 
55
61
  /**
56
- * Count one call into a dynamic worker as a live Loader fetch; the returned
57
- * function ends it (idempotently), from the caller's own `finally`.
62
+ * Hold the Dynamic Worker `workerKey` in flight on this actor's ledger; the
63
+ * returned function ends the hold (idempotently), from the caller's own
64
+ * `finally`. Holds on one key nest: the worker counts once until the last
65
+ * one ends, as the platform counts it.
58
66
  *
59
67
  * A begin/end pair rather than a wrapper on purpose, and the shape is
60
68
  * load-bearing: wrapping the stub call in a ledger-owned async frame
@@ -67,48 +75,82 @@ export function recordLoaderId(ctx: object, id: string): void {
67
75
  * workers: an RPC stub call must stay a direct property call awaited by the
68
76
  * frame that made it, so the ledger only brackets it.
69
77
  */
70
- export function beginLoaderFetch(ctx: object): () => void {
78
+ export function beginLoaderFetch(ctx: object, workerKey: string): () => void {
71
79
  const entry = ledger(ctx);
72
- entry.liveFetches++;
73
- entry.peakLiveFetches = Math.max(entry.peakLiveFetches, entry.liveFetches);
80
+ entry.inFlight.set(workerKey, (entry.inFlight.get(workerKey) ?? 0) + 1);
81
+ entry.peak = Math.max(entry.peak, inUse(entry));
74
82
  let ended = false;
75
83
  return () => {
76
84
  if (ended) return;
77
85
  ended = true;
78
- entry.liveFetches--;
86
+ const open = (entry.inFlight.get(workerKey) ?? 1) - 1;
87
+ if (open > 0) entry.inFlight.set(workerKey, open);
88
+ else entry.inFlight.delete(workerKey);
89
+ };
90
+ }
91
+
92
+ /**
93
+ * Distinct Dynamic Workers this actor may still put in flight: the limit
94
+ * less what is held and claimed right now. Never negative.
95
+ */
96
+ export function dynamicWorkerHeadroom(ctx: object): number {
97
+ return Math.max(0, DO_DYNAMIC_WORKER_LIMIT - inUse(ledger(ctx)));
98
+ }
99
+
100
+ /**
101
+ * Claim `width` distinct Dynamic Workers for one fan-out, or null when the
102
+ * headroom cannot hold it. The claim counts until `release` (idempotent), so
103
+ * a second fan-out sizing itself meanwhile sees it; the claimant's own
104
+ * dispatches are held as well while they run, which only ever over-counts
105
+ * toward sending that second fan-out elsewhere.
106
+ */
107
+ export function claimDynamicWorkers(ctx: object, width: number): { release(): void } | null {
108
+ const entry = ledger(ctx);
109
+ if (width < 1 || width > DO_DYNAMIC_WORKER_LIMIT - inUse(entry)) return null;
110
+ entry.claimed += width;
111
+ entry.peak = Math.max(entry.peak, inUse(entry));
112
+ let released = false;
113
+ return {
114
+ release() {
115
+ if (released) return;
116
+ released = true;
117
+ entry.claimed -= width;
118
+ },
79
119
  };
80
120
  }
81
121
 
82
122
  /** Snapshot for the diag surface. Pure read; no I/O. */
83
123
  export function loaderLedgerStats(ctx: object): {
84
- idsEverGotten: string[];
85
- liveFetches: number;
86
- peakLiveFetches: number;
124
+ limit: number;
125
+ inFlightWorkers: string[];
126
+ claimed: number;
127
+ headroom: number;
128
+ peak: number;
87
129
  } {
88
130
  const entry = ledger(ctx);
89
131
  return {
90
- idsEverGotten: [...entry.ids],
91
- liveFetches: entry.liveFetches,
92
- peakLiveFetches: entry.peakLiveFetches,
132
+ limit: DO_DYNAMIC_WORKER_LIMIT,
133
+ inFlightWorkers: [...entry.inFlight.keys()],
134
+ claimed: entry.claimed,
135
+ headroom: dynamicWorkerHeadroom(ctx),
136
+ peak: entry.peak,
93
137
  };
94
138
  }
95
139
 
96
140
  /**
97
- * Name the per-DO accounting on a "Too many concurrent dynamic workers"
141
+ * Name the per-DO accounting on a "Dynamic worker concurrency limit exceeded"
98
142
  * failure; hand every other error back untouched. The platform's message
99
- * says only that the cap was hit — which ids hold the slots, and that a
100
- * keyed id can never give one back, is what the operator needs to know to
101
- * shrink anything.
143
+ * says only that the limit was hit — which workers were in flight, and what
144
+ * fan-outs had claimed, is what the operator needs to know to shrink anything.
102
145
  */
103
146
  export function withDynamicWorkerCapNamed<E>(ctx: object, error: E): E | Error {
104
147
  if (classifyError(error) !== 'dynamic_worker_cap') return error;
105
148
  const entry = ledger(ctx);
106
149
  const platform = error instanceof Error ? error.message : String(error);
107
150
  return new Error(
108
- `${platform} — this Durable Object has ${entry.ids.size} loader id(s) permanently `
109
- + `holding dynamic-worker slots (a loader.get id is never released): `
110
- + `${[...entry.ids].join(', ') || '(none recorded)'}; live Loader fetches ${entry.liveFetches}, `
111
- + `peak ${entry.peakLiveFetches}`,
151
+ `${platform} — this Durable Object had ${entry.inFlight.size} distinct dynamic worker(s) in flight `
152
+ + `(${[...entry.inFlight.keys()].join(', ') || 'none recorded'}) and ${entry.claimed} claimed by fan-outs, `
153
+ + `against a limit of ${DO_DYNAMIC_WORKER_LIMIT}; peak ${entry.peak}`,
112
154
  { cause: error },
113
155
  );
114
156
  }
@@ -163,18 +205,23 @@ export function assertModuleMapWithinCodeLimit(modules: Record<string, unknown>)
163
205
 
164
206
  /**
165
207
  * Bytes one module-map member carries, across the loader's content kinds
166
- * (plain string, `{ js | cjs | py | text }`, `{ wasm | data }`). With an
167
- * encoder, text is measured exactly; without one, by code-unit length.
208
+ * (plain string, `{ js | cjs | py | text }`, `{ wasm | data }`, a bare
209
+ * WebAssembly.Module). With an encoder, text is measured exactly; without
210
+ * one, by code-unit length. A compiled module counts the wire size its host
211
+ * described (host-wasm.ts); one nobody described counts nothing here and is
212
+ * left to the platform's own refusal, as the text undercount is.
168
213
  */
169
214
  function memberBytes(content: unknown, encoder: TextEncoder | null): number {
170
215
  const textBytes = (text: string): number =>
171
216
  encoder ? encoder.encode(text).byteLength : text.length;
172
217
  if (typeof content === 'string') return textBytes(content);
218
+ if (content instanceof WebAssembly.Module) return hostWasmIdentity(content)?.bytes ?? 0;
173
219
  if (content !== null && typeof content === 'object') {
174
220
  for (const value of Object.values(content)) {
175
221
  if (typeof value === 'string') return textBytes(value);
176
222
  if (value instanceof ArrayBuffer) return value.byteLength;
177
223
  if (ArrayBuffer.isView(value)) return value.byteLength;
224
+ if (value instanceof WebAssembly.Module) return hostWasmIdentity(value)?.bytes ?? 0;
178
225
  }
179
226
  }
180
227
  return 0;
@@ -3,7 +3,7 @@
3
3
  * state over the WebSocket attachment.
4
4
  *
5
5
  * Specified from Proteus's DeviceSocketHub (`cf-backend/src/user/device-hub.ts`)
6
- * and CLI rpc gate (`cf-backend/src/cli/rpc-gate.ts`), which split the
6
+ * and its original CLI rpc gate (`cf-backend/src/cli/rpc-gate.ts`), which split the
7
7
  * pattern into its two halves:
8
8
  * - a TAG is the immutable-at-accept lookup key and authorization — it
9
9
  * rides the hibernation state, which is why the rpc gate persists auth
package/src/do-calls.ts CHANGED
@@ -3,12 +3,14 @@
3
3
  * the one property that decides whether a retry is safe.
4
4
  *
5
5
  * Both consumers asked for this. Proteus hand-wrote the retry
6
- * (`cf-backend/src/lib/do-rpc.ts`) with the rule its header states:
6
+ * (originally `cf-backend/src/lib/do-rpc.ts`) with the rule its header states:
7
7
  * "An operation that appends, sends, charges or mints is never wrapped: a
8
8
  * dropped call there may already have run, so a retry is a correctness bug
9
9
  * wearing resilience as a costume." agent-core has no retry machinery at all
10
10
  * and its backlog calls the gap "the most production-proven gap in the
11
11
  * corpus". Here the rule is a type: `idempotent` retries, `mutating` cannot.
12
+ * A mutation earns a retry only by carrying an identity its callee applies
13
+ * at most once; it is then `idempotent` by construction (see `mutating`).
12
14
  *
13
15
  * What the platform contract requires, and this keeps:
14
16
  * - a FRESH stub per attempt. Cloudflare documents that many exceptions
@@ -20,6 +22,11 @@
20
22
  * object is what overloaded it.
21
23
  * - attempts and backoff are the consumer-proven bounds: 3 attempts total,
22
24
  * full-jitter delays in [0, 2**attempt * 60ms).
25
+ * - an `idempotent` call may also be HEDGED (`hedgeAfterMs`): an attempt
26
+ * that has not answered by then is joined by the same call on a fresh
27
+ * stub, both left running, the first success taken. Hedges count
28
+ * against the attempts, and a callee that joins a repeat to the call it
29
+ * is already serving makes one that did arrive cost nothing.
23
30
  *
24
31
  * The resolver MINTS a stub per call and the verb disposes each one it
25
32
  * minted — that ownership is what makes the fresh-stub retry real.
@@ -35,6 +42,7 @@
35
42
 
36
43
  import { classifyDoCall, isRetryableDoCall, type DoCallClass } from '@nimbus-sh/platform/oom-classify.js';
37
44
  import { disposeRpcResource } from '@nimbus-sh/platform/rpc-dispose.js';
45
+ import { untraced, type SpanRecorder } from '@nimbus-sh/platform/tracing.js';
38
46
 
39
47
  /** Total attempts. Two retries is what a dropped connection or a deploy
40
48
  * bounce needs; beyond that the object is not coming back inside this
@@ -47,20 +55,71 @@ export interface DoCallRetryPolicy {
47
55
  maxAttempts?: number;
48
56
  baseDelayMs?: number;
49
57
  /**
50
- * Called once per retry, before its backoff delay, with the failure the
51
- * retry is answering. The consumer's logging seam: Proteus's hand-rolled
52
- * predecessor logged every retry so a flaky object is visible in Workers
53
- * Logs rather than silently absorbed, and `operation` names it there.
58
+ * No repeat — retry or hedge — starts once this long has passed since the
59
+ * first attempt did; the failure in hand surfaces instead. A mutation made
60
+ * repeatable by an identity its callee dedupes needs it: the callee keeps
61
+ * what answers a repeat for a bounded time, so the caller's repeats must
62
+ * stop well inside it. Unbounded when absent.
63
+ */
64
+ retryWindowMs?: number;
65
+ /**
66
+ * Hedge an attempt that has not answered after this long: send the same
67
+ * call again on a fresh stub while the first stays in flight. The caller
68
+ * gets the first success; an answer after it is disposed and dropped. That
69
+ * is only harmless when a second delivery of the call changes nothing — a
70
+ * read — and cheap only when the callee joins a repeat to the call it is
71
+ * already serving. A hedge is an attempt: it counts against `maxAttempts`,
72
+ * and has its own deadline. Never hedged when absent.
73
+ *
74
+ * With attempts overlapping, a failure decides less. A retryable one is
75
+ * retried after its backoff while attempts remain, and `overloaded` stops
76
+ * every further repeat; either way the call keeps waiting on the attempts
77
+ * still in flight, and fails with the last failure only once none is. Any
78
+ * other failure is the callee's answer — the call ran — and is the call's
79
+ * at once.
80
+ */
81
+ hedgeAfterMs?: number;
82
+ /**
83
+ * Called once per retry, as it starts — after its backoff delay — with the
84
+ * failure the retry is answering. A retry a hedge made unnecessary, or
85
+ * that no attempt is left for, is never announced. The consumer's logging
86
+ * seam: Proteus's hand-rolled predecessor logged every retry so a flaky
87
+ * object is visible in Workers Logs rather than silently absorbed, and
88
+ * `operation` names it there. A callback that throws fails the call with
89
+ * its error.
54
90
  */
55
91
  onRetry?(info: DoCallRetryInfo): void;
92
+ /**
93
+ * Where the call's telemetry goes: its span's recorder. Each attempt lost
94
+ * to a transient or overloaded failure is recorded as an exception whose
95
+ * `code` is the failure's class, and once the call has settled it gets
96
+ * `do_call.attempts` (started), `do_call.hedges` (started by a hedge),
97
+ * `do_call.answered_by` (the attempt whose answer the call took, absent
98
+ * when none answered) and `do_call.outcome` ({@link DoCallOutcome}).
99
+ * Nothing recorded can change the call's answer. Records nothing when
100
+ * absent.
101
+ */
102
+ span?: SpanRecorder;
56
103
  }
57
104
 
105
+ /**
106
+ * How an `idempotent` call ended: `answered` (an attempt succeeded),
107
+ * `callee_error` (the callee's own failure, which is an answer),
108
+ * `exhausted` (the last transient failure, no repeat left), `overloaded`
109
+ * (shed, nothing repeated after it), or `caller_error` (the resolver or
110
+ * `onRetry` threw).
111
+ */
112
+ export type DoCallOutcome = 'answered' | 'callee_error' | 'exhausted' | 'overloaded' | 'caller_error';
113
+
58
114
  /** What one retry is answering: which call, which platform class, which
59
115
  * attempt just failed out of how many. */
60
116
  export interface DoCallRetryInfo {
61
117
  operation: string;
62
118
  classification: DoCallClass;
63
- /** The 1-based attempt that failed; the retry about to run is attempt+1. */
119
+ /**
120
+ * The 1-based number of the attempt that failed. Without hedging the retry
121
+ * is attempt+1; with it, other attempts may have started in between.
122
+ */
64
123
  attempt: number;
65
124
  maxAttempts: number;
66
125
  error: unknown;
@@ -102,11 +161,20 @@ export class DoCallError extends Error {
102
161
 
103
162
  /**
104
163
  * Call another Durable Object with an operation that is safe to repeat: a
105
- * read, or a converge-to-a-value write. Transient failures retry on a fresh
106
- * stub with full-jitter backoff; overloaded and permanent failures surface
107
- * unchanged, as does the last error at exhaustion.
164
+ * read, a converge-to-a-value write, or a mutation carrying an identity its
165
+ * callee applies at most once (see {@link mutating}). Transient failures
166
+ * retry on a fresh stub with full-jitter backoff; overloaded and permanent
167
+ * failures surface unchanged, as does the last error once no attempt may be
168
+ * repeated — attempts spent, or the policy's retry window closed. With
169
+ * `hedgeAfterMs`, an attempt still unanswered by then is joined by another
170
+ * on a fresh stub, and the first answer is taken: a success, or the
171
+ * callee's own error. A transient or overloaded failure then ends the call
172
+ * only once no attempt is left in flight.
173
+ *
174
+ * A failure of the resolver or of `onRetry` is the caller's own, and fails
175
+ * the call with it at once.
108
176
  */
109
- export async function idempotent<S, T>(
177
+ export function idempotent<S, T>(
110
178
  operation: string,
111
179
  stub: DoStubResolver<S>,
112
180
  call: (stub: S) => Promise<T>,
@@ -114,23 +182,131 @@ export async function idempotent<S, T>(
114
182
  ): Promise<T> {
115
183
  const maxAttempts = policy.maxAttempts ?? MAX_ATTEMPTS;
116
184
  const baseDelayMs = policy.baseDelayMs ?? BASE_DELAY_MS;
117
- for (let attempt = 1; ; attempt++) {
118
- const minted = await stub();
119
- try {
120
- const result = await call(minted);
121
- disposeRpcResource(minted);
122
- return result;
123
- } catch (error) {
124
- // A stub that threw may be permanently broken; it is never reused.
125
- disposeRpcResource(minted);
185
+ const { hedgeAfterMs, retryWindowMs, span = untraced } = policy;
186
+ const startedAt = Date.now();
187
+ // The executor form: fabric's library target predates Promise.withResolvers.
188
+ return new Promise<T>((resolve, reject) => {
189
+ const hedges = new Set<ReturnType<typeof setTimeout>>();
190
+ let started = 0;
191
+ // Attempts in flight, or backing off before their retry: while one is,
192
+ // a failure is not the call's answer.
193
+ let live = 0;
194
+ // An attempt was shed as overloaded: nothing is repeated after it.
195
+ let refused = false;
196
+ let settled = false;
197
+ // Attempts a hedge started rather than a retry or the first send.
198
+ let hedged = 0;
199
+
200
+ const settle = (outcome: DoCallOutcome, answeredBy: number | undefined, answer: () => void): void => {
201
+ if (settled) return;
202
+ settled = true;
203
+ for (const timer of hedges) clearTimeout(timer);
204
+ hedges.clear();
205
+ answer();
206
+ span.set({
207
+ 'do_call.attempts': started,
208
+ 'do_call.hedges': hedged,
209
+ 'do_call.answered_by': answeredBy,
210
+ 'do_call.outcome': outcome,
211
+ });
212
+ };
213
+ /** May another attempt start at `at`? */
214
+ const canRepeat = (at: number): boolean =>
215
+ !settled && !refused && started < maxAttempts
216
+ && (retryWindowMs === undefined || at - startedAt <= retryWindowMs);
217
+ /** `error` ended an attempt that will not be repeated: the call's answer, once nothing else is live. */
218
+ const exhausted = <E>(error: E): void => {
219
+ if (live === 0) settle(refused ? 'overloaded' : 'exhausted', undefined, () => reject(error));
220
+ };
221
+
222
+ /** A failed attempt, numbered: repeat it after its backoff, or let it stand. */
223
+ const failed = async <E>(number: number, error: E): Promise<void> => {
224
+ if (settled) return;
126
225
  const classification = classifyDoCall(error);
127
- if (!isRetryableDoCall(classification) || attempt >= maxAttempts) throw error;
128
- policy.onRetry?.({ operation, classification, attempt, maxAttempts, error });
129
- await new Promise<void>((resolve) => {
130
- setTimeout(resolve, Math.floor(Math.random() * 2 ** attempt * baseDelayMs));
226
+ if (!isRetryableDoCall(classification) && classification !== 'overloaded') {
227
+ // The call ran and its answer is this error — ENOENT is a read's answer as much as bytes are.
228
+ settle('callee_error', number, () => reject(error));
229
+ return;
230
+ }
231
+ // A lost attempt: the call's span says which, and why.
232
+ span.exception(error, classification, `attempt ${number} of ${maxAttempts}: `);
233
+ if (classification === 'overloaded') {
234
+ // A shed call is no answer: nothing more is sent, and an attempt
235
+ // still in flight may yet answer.
236
+ refused = true;
237
+ exhausted(error);
238
+ return;
239
+ }
240
+ const delayMs = Math.floor(Math.random() * 2 ** number * baseDelayMs);
241
+ if (!canRepeat(Date.now() + delayMs)) {
242
+ exhausted(error);
243
+ return;
244
+ }
245
+ live++;
246
+ await new Promise<void>((wake) => {
247
+ setTimeout(wake, delayMs);
131
248
  });
132
- }
133
- }
249
+ live--;
250
+ // A hedge may have taken the last attempt, or answered, meanwhile.
251
+ if (!canRepeat(Date.now())) {
252
+ exhausted(error);
253
+ return;
254
+ }
255
+ policy.onRetry?.({ operation, classification, attempt: number, maxAttempts, error });
256
+ attempt();
257
+ };
258
+
259
+ const run = async (): Promise<void> => {
260
+ const number = ++started;
261
+ live++;
262
+ const minted = await stub();
263
+ if (settled) {
264
+ live--;
265
+ disposeRpcResource(minted);
266
+ return;
267
+ }
268
+ const hedge = hedgeAfterMs === undefined ? undefined : setTimeout(() => {
269
+ if (hedge !== undefined) hedges.delete(hedge);
270
+ if (canRepeat(Date.now())) {
271
+ hedged++;
272
+ attempt();
273
+ }
274
+ }, hedgeAfterMs);
275
+ if (hedge !== undefined) hedges.add(hedge);
276
+ /** This attempt has its answer: it hedges no more, and its stub goes. */
277
+ const answered = (): void => {
278
+ if (hedge !== undefined) {
279
+ clearTimeout(hedge);
280
+ hedges.delete(hedge);
281
+ }
282
+ live--;
283
+ // A stub that threw may be permanently broken; none is ever reused.
284
+ disposeRpcResource(minted);
285
+ };
286
+ let result: T;
287
+ try {
288
+ result = await call(minted);
289
+ } catch (error) {
290
+ answered();
291
+ await failed(number, error);
292
+ return;
293
+ }
294
+ answered();
295
+ if (settled) {
296
+ // Another attempt answered first: this answer is dropped, so nothing of it is kept.
297
+ disposeRpcResource(result);
298
+ return;
299
+ }
300
+ settle('answered', number, () => resolve(result));
301
+ };
302
+
303
+ /** Start an attempt. Whatever it throws outside the call itself fails the call. */
304
+ const attempt = (): void => {
305
+ run().catch((error) => settle('caller_error', undefined, () => reject(error)));
306
+ };
307
+
308
+ attempt();
309
+ });
134
310
  }
135
311
 
136
312
  /**
@@ -138,6 +314,21 @@ export async function idempotent<S, T>(
138
314
  * or mints. NEVER retried — a dropped call may already have run. Failure
139
315
  * surfaces as a {@link DoCallError} carrying the classification, so the
140
316
  * caller can tell a refusal from an indeterminate drop.
317
+ *
318
+ * The rule is about the call as sent, not the operation's kind. A mutation
319
+ * the callee applies at most once per identity the call carries is
320
+ * repeatable by construction: a repeat of one that already ran is answered
321
+ * from the callee's record and applies nothing. Nimbus has two:
322
+ * - delivered filesystem mutations (@nimbus-sh/core supervisor-delivery):
323
+ * a delivery id plus the callee INSTANCE's incarnation. The record lives
324
+ * in that instance's memory, and any other instance — or a callee that
325
+ * predates delivery — refuses the call permanently rather than apply it
326
+ * without one;
327
+ * - appends: writer, module incarnation and operation sequence, recorded
328
+ * durably until acknowledged.
329
+ * Such a call goes through {@link idempotent}, re-sending the same identity
330
+ * on every attempt, with a `retryWindowMs` inside the callee's retention of
331
+ * that record. Without such an identity, a mutation stays here.
141
332
  */
142
333
  export async function mutating<S, T>(
143
334
  operation: string,
package/src/facet-pool.ts CHANGED
@@ -42,6 +42,8 @@
42
42
  * and the facet-id ledger here counts only the names this pool minted.
43
43
  */
44
44
 
45
+ import { forgetFacetStorage } from '@nimbus-sh/core/runtime/storage-ledger.js';
46
+ import type { SqlDatabase } from '@nimbus-sh/core/runtime/os-contracts.js';
45
47
  import {
46
48
  FACET_ID_LIFETIME_BUDGET,
47
49
  facetNameCount,
@@ -63,6 +65,8 @@ export interface FacetPoolContext {
63
65
  storage: {
64
66
  get(key: string): Promise<unknown> | unknown;
65
67
  put(key: string, value: unknown): Promise<void>;
68
+ /** The session's SQL, where the storage ledger (N18) records facet databases. */
69
+ sql?: SqlDatabase;
66
70
  };
67
71
  }
68
72
 
@@ -134,6 +138,7 @@ export class FacetPool {
134
138
  if (keepStorage) return;
135
139
  try {
136
140
  facets.delete(name);
141
+ if (this.ctx.storage.sql) forgetFacetStorage(this.ctx.storage.sql, name);
137
142
  } catch (e) {
138
143
  throw new Error(
139
144
  `fabric: facet '${name}' was evicted but its storage was not reclaimed — `