@nimbus-sh/platform 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +36 -0
  3. package/dist/diag-counters.d.ts +210 -0
  4. package/dist/diag-counters.d.ts.map +1 -0
  5. package/dist/diag-counters.js +170 -0
  6. package/dist/heap-estimate.d.ts +225 -0
  7. package/dist/heap-estimate.d.ts.map +1 -0
  8. package/dist/heap-estimate.js +185 -0
  9. package/dist/heavy-alloc-coord.d.ts +93 -0
  10. package/dist/heavy-alloc-coord.d.ts.map +1 -0
  11. package/dist/heavy-alloc-coord.js +181 -0
  12. package/dist/index.d.ts +19 -0
  13. package/dist/index.d.ts.map +1 -0
  14. package/dist/index.js +18 -0
  15. package/dist/install-phase.d.ts +47 -0
  16. package/dist/install-phase.d.ts.map +1 -0
  17. package/dist/install-phase.js +1 -0
  18. package/dist/limits.d.ts +83 -0
  19. package/dist/limits.d.ts.map +1 -0
  20. package/dist/limits.js +146 -0
  21. package/dist/oom-classify.d.ts +172 -0
  22. package/dist/oom-classify.d.ts.map +1 -0
  23. package/dist/oom-classify.js +336 -0
  24. package/dist/oom-discriminator.d.ts +176 -0
  25. package/dist/oom-discriminator.d.ts.map +1 -0
  26. package/dist/oom-discriminator.js +272 -0
  27. package/dist/rpc-dispose.d.ts +4 -0
  28. package/dist/rpc-dispose.d.ts.map +1 -0
  29. package/dist/rpc-dispose.js +30 -0
  30. package/dist/w7-frame.d.ts +107 -0
  31. package/dist/w7-frame.d.ts.map +1 -0
  32. package/dist/w7-frame.js +842 -0
  33. package/dist/weighted-credit-pool.d.ts +61 -0
  34. package/dist/weighted-credit-pool.d.ts.map +1 -0
  35. package/dist/weighted-credit-pool.js +196 -0
  36. package/package.json +57 -0
  37. package/src/diag-counters.ts +310 -0
  38. package/src/heap-estimate.ts +331 -0
  39. package/src/heavy-alloc-coord.ts +241 -0
  40. package/src/index.ts +19 -0
  41. package/src/install-phase.ts +56 -0
  42. package/src/limits.ts +165 -0
  43. package/src/oom-classify.ts +350 -0
  44. package/src/oom-discriminator.ts +405 -0
  45. package/src/rpc-dispose.ts +31 -0
  46. package/src/w7-frame.ts +1046 -0
  47. package/src/weighted-credit-pool.ts +244 -0
package/src/limits.ts ADDED
@@ -0,0 +1,165 @@
1
+ /**
2
+ * limits.ts — the measured Cloudflare platform limits Nimbus is built
3
+ * against.
4
+ *
5
+ * Every constant here is a fact about the platform — a hard ceiling, or a
6
+ * measured envelope proven safe against one — rather than a Nimbus policy
7
+ * choice. Policy constants (versions, defaults, feature bounds) live in
8
+ * `@nimbus-sh/core/constants.js`; this table is what they are derived from.
9
+ */
10
+
11
+ // ── Storage-transaction bounds ──────────────────────────────────────────
12
+ //
13
+ // The platform resets a Durable Object over what ONE TURN has outstanding
14
+ // in storage ("Internal error in Durable Object storage caused object to
15
+ // be reset"), not over what it eventually writes. These bounds keep any
16
+ // one transaction below that wall.
17
+
18
+ /**
19
+ * The storage/wire chunk quantum: bytes per content chunk in the SQLite
20
+ * VFS schema, and therefore the unit every transaction bound and streamed
21
+ * write frame is denominated in. SQLite's per-ROW value cap is 2 MB (key
22
+ * length included, measured single-value ceiling 2,199,981 B), so 64 KiB
23
+ * keeps a chunk row far under it while amortizing per-row overhead.
24
+ */
25
+ export const CHUNK_SIZE = 65_536;
26
+
27
+ export const MAX_TX_BLOB_BYTES = 1 * 1024 * 1024;
28
+ export const MAX_TX_LOGICAL_ROWS = 256;
29
+ export const MAX_TX_SQL_EXECS = 64;
30
+
31
+ // ── SQLite storage walls ────────────────────────────────────────────────
32
+
33
+ /**
34
+ * Storage per SQLite-backed Durable Object: 10 decimal GB documented
35
+ * (Workers Paid), shared by the root object, every facet beneath it, and
36
+ * every clone — a copy-on-write clone consumes its FULL logical bytes with
37
+ * no CoW credit. Probed window: 10.58e9 logical bytes fit, 11.6e9 failed —
38
+ * 10 GiB (10,737,418,240) falls INSIDE that window and is not a number to
39
+ * design to. At the wall, ordinary writes fail catchably as SQLITE_FULL
40
+ * ('database or disk is full') while reads and DELETEs keep working, so the
41
+ * recovery is to drain; a facet CLONE over the wall is an uncatchable reset
42
+ * that empties the destination, so clone admission is decided BEFORE the
43
+ * clone, with reserve.
44
+ */
45
+ export const DO_STORAGE_LIMIT_BYTES = 10_000_000_000;
46
+
47
+ /**
48
+ * Maximum SQL statement text per exec. Documented "100 KB"; read as binary
49
+ * KiB because the value is SQLite's compile-time SQLITE_MAX_SQL_LENGTH
50
+ * rather than a billing quantity (unverified reading — reached by generated
51
+ * statements, a batched multi-VALUES insert being the realistic breach).
52
+ */
53
+ export const SQLITE_MAX_STATEMENT_BYTES = 100 * 1024;
54
+
55
+ /**
56
+ * Maximum bound parameters per query. The easiest cap to hit accidentally:
57
+ * a batched insert of more than 100/columns rows in one statement breaches
58
+ * it.
59
+ */
60
+ export const SQLITE_MAX_BOUND_PARAMETERS = 100;
61
+
62
+ /**
63
+ * Maximum bytes of one string, BLOB, or table ROW — the bound is per ROW,
64
+ * key length included, not per value. The measured single-value ceiling is
65
+ * 2,199,981 bytes, ABOVE this constant: budgeting each value against the
66
+ * row bound is conservative and correct, budgeting a row against the value
67
+ * ceiling is not. Writes over it fail; reads and deletes keep working.
68
+ */
69
+ export const SQLITE_MAX_ROW_BYTES = 2_000_000;
70
+
71
+ // ── Init-gate hazard ────────────────────────────────────────────────────
72
+
73
+ /**
74
+ * A `blockConcurrencyWhile()` callback still pending this long is
75
+ * cancelled and the Durable Object is RESET — every event queued behind
76
+ * the gate dies with it. Proven by probe (reset observed at 31 s against a
77
+ * 31 s-busy neighbour; cold activations track a busy neighbour 1:1), and
78
+ * partyserver runs `onStart()` inside the gate, so ordinary traffic can
79
+ * reach this. Anything on that path must be bounded well below this;
80
+ * fabric defers async reconciliation off the gate entirely (`onColdStart`).
81
+ */
82
+ export const BLOCK_CONCURRENCY_CANCEL_MS = 30_000;
83
+
84
+ // ── WebSocket attachment bound ──────────────────────────────────────────
85
+
86
+ /**
87
+ * Bytes one hibernatable WebSocket attachment may serialize to. Verified in
88
+ * workerd source at the pinned version (v1.20260603.1,
89
+ * src/workerd/api/web-socket.h: `MAX_ATTACHMENT_SIZE = 1024 * 16`). The
90
+ * bound is on the SERIALIZED bytes — workerd re-serializes on every
91
+ * `serializeAttachment` call to check it — not on UTF-8 JSON text, so a
92
+ * JSON-length measurement is an approximation and only the platform's own
93
+ * refusal ("A WebSocket 'attachment' cannot be larger than 16384 bytes.")
94
+ * is the ceiling.
95
+ */
96
+ export const WS_ATTACHMENT_LIMIT_BYTES = 16_384;
97
+
98
+ // ── RPC payload envelope ────────────────────────────────────────────────
99
+
100
+ // Largest byte payload Nimbus sends through an ordinary Workers RPC value.
101
+ // The platform limit is 32 MiB; 28 MiB leaves room for structured-clone
102
+ // metadata and matches the proven on-demand facet transfer envelope.
103
+ export const MAX_RPC_SAFE_PAYLOAD_BYTES = 28 * 1024 * 1024;
104
+
105
+ // ── Supervisor heap budget [C'.1] ───────────────────────────────────────
106
+ //
107
+ // Three distinct memory regimes, not one shared pool:
108
+ //
109
+ // - Supervisor DO isolate: 128 MiB HARD platform ceiling ("Each isolate
110
+ // can consume up to 128 MB of memory" —
111
+ // https://developers.cloudflare.com/workers/platform/limits/).
112
+ // - Facets: ~208-256 MiB EACH, measured — memory independent of the
113
+ // supervisor and of each other (1,664 MiB live across 8 facets +
114
+ // parent) but CPU SHARED across the actor thread. See
115
+ // packages/fabric/src/process-fabric.ts.
116
+ // - Peer DOs: independent memory AND CPU budgets, measured 1.2 GiB
117
+ // live across 8 peers.
118
+ //
119
+ // Nimbus targets HALF of the supervisor's 128 MiB as a self-imposed soft
120
+ // ceiling so the supervisor always has runway when workerd LRU-evicts
121
+ // neighbours or AIR (Asynchronous Isolate Recreation) folds growing
122
+ // isolates.
123
+ //
124
+ // 64 MiB is a soft admission budget, not a measurement — the right value
125
+ // is unmeasured — and nothing enforces it directly.
126
+ // heap-estimate.ts sums the INSTRUMENTED contributors —
127
+ // the supervisor baseline, VFS LRU and in-flight writes, pre-bundle
128
+ // slices, streaming RPC buffers, and the prefetch-bundle build lease and
129
+ // cache gauge — which is a lower bound, not full coverage; allocation
130
+ // sites known to be missing are listed in HEAP_BLIND_SPOTS. Read a low
131
+ // percentOfCeiling accordingly.
132
+ export const SUPERVISOR_HEAP_CEILING_BYTES = 64 * 1024 * 1024;
133
+
134
+ // Shared allowance for transient allocations in the supervisor DO. With the
135
+ // VFS LRU shrunk to 8 MiB during an active reservation, 40 MiB of admitted
136
+ // payload plus the 9 MiB bundle baseline stays below the 64 MiB soft ceiling
137
+ // with 7 MiB left for metadata, structured-clone overhead, and runtime state.
138
+ // This is 31.25% of the platform's 128 MiB hard isolate ceiling.
139
+ export const SUPERVISOR_IN_FLIGHT_ALLOCATION_BUDGET_BYTES = 40 * 1024 * 1024;
140
+
141
+ // Reserved slice of that allowance for chunk-sized filesystem reads, so a
142
+ // read is never queued behind a multi-megabyte owner for a wait that has
143
+ // nothing to do with its own cost. A read only draws on this when the shared
144
+ // budget is already contended — which is exactly when the VFS LRU is shrunk
145
+ // to 8 MiB — so peak accounting is 40 + 1 admitted, 8 LRU, 9 bundle baseline:
146
+ // 58 MiB, still under the 64 MiB soft ceiling.
147
+ //
148
+ // 1 MiB is 16 concurrent READ_STREAM_CHUNK_BYTES reads. The point is not
149
+ // depth, it is that a read loop keeps moving while a heavy owner works
150
+ // instead of stopping dead for the duration.
151
+ export const SUPERVISOR_READ_RESERVE_BYTES = 1024 * 1024;
152
+
153
+ // Global bound on supervisor-resident write-stream credit, shared by every
154
+ // concurrent streamed bulk write into one DO. Part of the same supervisor
155
+ // budget arithmetic as the allocation budget above.
156
+ export const MAX_GLOBAL_WRITE_STREAM_CREDIT_BYTES = 8 * 1024 * 1024;
157
+
158
+ // ── Pre-bundle admission envelope ───────────────────────────────────────
159
+ //
160
+ // The proven-safe envelope for shipping module slices through a shared DO
161
+ // isolate: one slice at the RPC envelope at a time. Measured the hard way —
162
+ // concurrency=2 of max slices crashed Mossaic-scale installs on shared DO
163
+ // isolates. The heap estimator's pre-bundle lane models exactly this bound.
164
+ export const PRE_BUNDLE_SLICE_CAP_BYTES = MAX_RPC_SAFE_PAYLOAD_BYTES;
165
+ export const PRE_BUNDLE_CONCURRENCY = 1;
@@ -0,0 +1,350 @@
1
+ /**
2
+ * oom-classify.ts — discriminator for OOM-like errors at Nimbus
3
+ * boundaries.
4
+ *
5
+ * Why this exists
6
+ * ───────────────
7
+ * Nimbus has at least five distinct "the work failed" failure modes
8
+ * that all surface today as either a thrown JS Error or a console
9
+ * line:
10
+ *
11
+ * 1. SQLITE_NOMEM at the storage layer (per-DO SQLite cap, post-
12
+ * STOR/SPEC: Address SQLITE_NOMEM issues).
13
+ * 2. Isolate memory exhaustion → 'oom'.
14
+ * 3. Isolate CPU-budget exhaustion → 'cpu_exceeded'.
15
+ * 4. Structured-clone refusal (`Cannot deserialize cloned data`)
16
+ * between supervisor ↔ facet RPC (32 MiB cap).
17
+ * 5. RPC timeout (TimeoutError from facet-pool's per-task race).
18
+ *
19
+ * Plus a few platform-side terminations (subrequest cap, condemnation,
20
+ * hard eviction) that the user sees but Nimbus has no first-party
21
+ * signal for.
22
+ *
23
+ * Memory and CPU are separate buckets
24
+ * ───────────────────────────────────
25
+ * workerd models them as distinct trace outcomes — `exceededMemory` vs
26
+ * `exceededCpu` (EventOutcome, workerd/io/outcome.capnp) — and words the
27
+ * two message families so neither one's signature appears in the other:
28
+ *
29
+ * memory "Worker has exceeded memory limit."
30
+ * "Worker exceeded memory limit."
31
+ * "broken.exceededMemory; jsg.Error: Durable Object's isolate
32
+ * exceeded its memory limit …"
33
+ * "Memory limit exceeded" (RangeError)
34
+ * cpu "Worker exceeded CPU time limit."
35
+ * "Durable Object exceeded its CPU time limit and was reset."
36
+ * "Python Worker exceeded CPU time limit"
37
+ *
38
+ * Read a low 'cpu_exceeded' count carefully: neither condition reliably
39
+ * produces a message. Both are uncatchable inside the isolate that died,
40
+ * so the string is only observable by a CALLER across an RPC boundary,
41
+ * and Nimbus has repeatedly measured isolates vanishing with no throw at
42
+ * all. Absence of 'cpu_exceeded' entries is therefore NOT evidence that
43
+ * CPU was not the cause — confirm against `wrangler tail` either way.
44
+ *
45
+ * Without a classifier, every recordFailure() call has to stringify-
46
+ * match its own error site. With this module, we pin the regex once
47
+ * and reuse it everywhere.
48
+ *
49
+ * Inputs we accept
50
+ * ────────────────
51
+ * The classifier accepts:
52
+ * - Error instances (read e.message)
53
+ * - plain strings (use directly — for stderr-line classification at
54
+ * facet-exit boundaries where we don't have an Error)
55
+ * - anything else (return 'unknown')
56
+ *
57
+ * Output values
58
+ * ─────────────
59
+ * Same set used by `DiagFailure.cause` in oom-discriminator.ts. Keep
60
+ * the union narrow and additive — adding a new value is fine, but
61
+ * never re-purpose an existing one.
62
+ */
63
+
64
+ const OOM_CAUSES = [
65
+ 'sqlite_nomem',
66
+ 'sqlite_full',
67
+ 'oom',
68
+ 'cpu_exceeded',
69
+ 'clone_refused',
70
+ 'rpc_timeout',
71
+ 'subrequest_cap',
72
+ 'dynamic_worker_cap',
73
+ 'condemnation',
74
+ 'hard_evict',
75
+ 'unknown',
76
+ ] as const;
77
+
78
+ export type OomCause = typeof OOM_CAUSES[number];
79
+
80
+ export function isOomCause(input: unknown): input is OomCause {
81
+ return OOM_CAUSES.some((cause) => cause === input);
82
+ }
83
+
84
+ /**
85
+ * Classify an error or message string into an OomCause. Returns
86
+ * 'unknown' when no signature matches — callers should still record
87
+ * the message via DiagFailure.message so a human can later widen the
88
+ * classifier rules.
89
+ */
90
+ export function classifyError(input: unknown): OomCause {
91
+ const msg = readMessage(input);
92
+ if (msg.length === 0) return 'unknown';
93
+ return classifyMessage(msg);
94
+ }
95
+
96
+ /**
97
+ * Variant for callers that already have the message string. Prefer
98
+ * classifyError() at boundaries; this is exposed for cases where the
99
+ * message has already been extracted (e.g. truncated / sanitised).
100
+ */
101
+ export function classifyMessage(msg: string): OomCause {
102
+ // Use lower-case match for forgiveness; SQLITE_NOMEM is canonically
103
+ // upper but stderr can be either.
104
+ const m = msg.toLowerCase();
105
+
106
+ // SQLITE_FULL: the 10 GB storage wall (platform catalog do.storage.bytes,
107
+ // DO_STORAGE_LIMIT_BYTES). A DIFFERENT condition from NOMEM with the
108
+ // opposite remedy: reads and DELETEs keep working, so the recovery is to
109
+ // drain data — where NOMEM asks for a smaller transaction. agent-core maps
110
+ // this to the same generic code as a malformed statement, which is the
111
+ // defect this split exists to avoid.
112
+ if (m.includes('sqlite_full')) return 'sqlite_full';
113
+ if (m.includes('database or disk is full')) return 'sqlite_full';
114
+
115
+ // SQLITE_NOMEM signals
116
+ if (m.includes('sqlite_nomem')) return 'sqlite_nomem';
117
+ if (m.includes('out of memory')) return 'sqlite_nomem';
118
+
119
+ // Structured-clone refusal — a 32 MiB-cap cousin
120
+ if (m.includes('deserialize cloned data')) return 'clone_refused';
121
+ if (m.includes('could not be cloned')) return 'clone_refused';
122
+
123
+ // RPC timeout (Nimbus's TimeoutError from facet-pool)
124
+ if (m.includes('timeouterror')) return 'rpc_timeout';
125
+ if (/\btimed?\s*out\b/.test(m)) return 'rpc_timeout';
126
+
127
+ // Subrequest cap (Cloudflare platform)
128
+ if (m.includes('too many subrequests')) return 'subrequest_cap';
129
+
130
+ // Worker Loader concurrency cap (Cloudflare platform): a Durable Object
131
+ // admits ~5-6 concurrent dynamic workers, and loader-cache entries are
132
+ // never released — every distinct loader.get(id) permanently consumes a
133
+ // slot, so this cap recurs until the DO itself is replaced.
134
+ if (m.includes('too many concurrent dynamic workers')) return 'dynamic_worker_cap';
135
+
136
+ // Memory exhaustion. workerd trace outcome `exceededMemory`; in-band it
137
+ // arrives as a `broken.exceededMemory` actor break, a "…exceeded [its]
138
+ // memory limit" kill, or the "Memory limit exceeded" RangeError.
139
+ if (m.includes('exceededmemory')) return 'oom';
140
+ if (/exceeded (?:its )?memory limit/.test(m)) return 'oom';
141
+ if (m.includes('memory limit exceeded')) return 'oom';
142
+ if (m.includes('memory limit') && m.includes('reset')) return 'oom';
143
+
144
+ // CPU-budget exhaustion. A DIFFERENT platform condition from memory —
145
+ // workerd reports it as trace outcome `exceededCpu`. Never fold the two:
146
+ // a CPU kill recurs on the same input, a memory kill recurs on the same
147
+ // working-set size, and the remedies are opposites.
148
+ //
149
+ // `cpu time limit` covers both scopes, which differ only by the
150
+ // possessive: "Worker exceeded CPU time limit." and "Durable Object
151
+ // exceeded its CPU time limit and was reset."
152
+ //
153
+ // Wall-clock handler timeouts are deliberately NOT matched here.
154
+ // "Actor exceeded event execution time and was disconnected." is the
155
+ // caller-configurable setHibernatableWebSocketEventTimeout() bound and
156
+ // "Alarm exceeded its allowed execution time" is its alarm twin. Both
157
+ // are elapsed-time bounds a handler blocked on I/O trips without burning
158
+ // CPU, so bucketing them here would repeat the merge this split undoes.
159
+ if (m.includes('cpu time limit')) return 'cpu_exceeded';
160
+ if (m.includes('exceeded cpu limit')) return 'cpu_exceeded';
161
+
162
+ // Explicit eviction (per CF research §A.1.2)
163
+ if (m.includes('condemned')) return 'condemnation';
164
+ if (m.includes('hard evict')) return 'hard_evict';
165
+
166
+ return 'unknown';
167
+ }
168
+
169
+ /**
170
+ * Transient Durable Object reset — the object was reset by the platform
171
+ * mid-request for a reason unrelated to the request's own resource use:
172
+ * a code deploy rolling over, or a storage-subsystem cold-start hiccup.
173
+ * The in-flight request/RPC rejects, but the work itself never ran to a
174
+ * conclusion and is safe to re-attempt.
175
+ *
176
+ * Deliberately narrow: it must NOT match resource kills ("isolate exceeded
177
+ * its memory limit and was reset", "Durable Object exceeded its CPU time
178
+ * limit and was reset"), which classifyMessage() routes to 'oom' and
179
+ * 'cpu_exceeded' — those recur on retry and must surface, not loop. Note
180
+ * both end in "and was reset"; the checks below key on the CAUSE clause,
181
+ * never on the reset itself.
182
+ * Observed verbatim signatures:
183
+ * - "Durable Object reset because its code was updated."
184
+ * - "Internal error while starting up Durable Object storage caused
185
+ * object to be reset; reference = ..."
186
+ * - "Internal error in Durable Object storage caused object to be reset;
187
+ * reference = ..."
188
+ * - "Durable Object storage operation exceeded timeout which caused
189
+ * the object to be reset."
190
+ *
191
+ * The second and third are the SAME condition worded for two moments —
192
+ * startup versus a live write — and only the first was matched here, which is
193
+ * why an object reset mid-write still failed a whole install. Nimbus has
194
+ * measured the live-write wording twice: a 45.7 MB single-turn write reset the
195
+ * object once and the same write then succeeded 12/12 on retry
196
+ * (`vfs/facet-resident-store.ts`), and it reset a session DO mid-npm-install
197
+ * on 2026-08-10 (probe `agentic-cli/new/pi-official-installer`). Retrying is
198
+ * bounded, so the one case that is NOT transient — crossing the object's
199
+ * storage budget, which the platform also reports this way
200
+ * (`loaders/process-fabric.ts`) — recurs, exhausts the budget and surfaces.
201
+ */
202
+ export function isTransientDoReset(input: unknown): boolean {
203
+ const m = readMessage(input).toLowerCase();
204
+ if (m.length === 0) return false;
205
+ if (m.includes('reset because its code was updated')) return true;
206
+ if (m.includes('durable object storage caused object to be reset')) return true;
207
+ if (m.includes('storage operation') && m.includes('reset')) return true;
208
+ return false;
209
+ }
210
+
211
+ /**
212
+ * Workerd shed the call because the target Durable Object's input-gate
213
+ * queue was too deep or too old: "Durable Object is overloaded."
214
+ *
215
+ * Distinct from `isTransientDoReset` on purpose. The object is alive and
216
+ * the work never started, so the call is safe to re-attempt — but the
217
+ * cause is queue pressure, so the only useful retry is one that backs off
218
+ * long enough for the queue to drain. Callers pair this with a longer
219
+ * backoff than a reset retry uses.
220
+ */
221
+ export function isDoOverloaded(input: unknown): boolean {
222
+ return readMessage(input).toLowerCase().includes('durable object is overloaded');
223
+ }
224
+
225
+ /**
226
+ * How one failed Durable Object call relates to a retry. The taxonomy
227
+ * Proteus hand-wrote in `cf-backend/src/lib/do-rpc.ts:71-79` because the
228
+ * Agents SDK does not export its own, plus the `overloaded` class both
229
+ * consumers need: Cloudflare documents that errors carry `.retryable` and
230
+ * `.overloaded`, that a retryable error should be retried with backoff on a
231
+ * FRESH stub, and that an overloaded one must not be retried at all
232
+ * (developers.cloudflare.com/durable-objects/best-practices/error-handling).
233
+ */
234
+ export type DoCallClass =
235
+ /** A deploy replaced the isolate mid-call. */
236
+ | 'superseded_isolate'
237
+ /** The stub or storage connection dropped. */
238
+ | 'connection_lost'
239
+ /** Storage reset the object — cold start, live write, or a timeout. */
240
+ | 'storage_reset'
241
+ /** The runtime flagged the error `retryable` itself. */
242
+ | 'retryable_flag'
243
+ /** The object shed the call under queue pressure. NEVER retried: retrying
244
+ * an overloaded object is what overloaded it. */
245
+ | 'overloaded'
246
+ /** Everything else — the call ran and the answer is the error. */
247
+ | 'permanent';
248
+
249
+ /** The classes a fresh-stub retry may act on. `overloaded` is deliberately
250
+ * outside: the platform's own docs forbid retrying it. */
251
+ export function isRetryableDoCall(input: DoCallClass): boolean {
252
+ return input === 'superseded_isolate'
253
+ || input === 'connection_lost'
254
+ || input === 'storage_reset'
255
+ || input === 'retryable_flag';
256
+ }
257
+
258
+ /**
259
+ * Prose signatures of the transient classes, matched against the rendered
260
+ * cause chain — a wrapper error (Proteus measured its SqlError doing this)
261
+ * drops the `.retryable` property, so only the joined prose survives it.
262
+ *
263
+ * `storage_reset` carries the live-write wording on purpose. Proteus's
264
+ * mirror of the SDK matcher excludes it because a storage-budget breach is
265
+ * reported the same way; Nimbus measured the reset-mid-write case succeed
266
+ * 12/12 on retry ({@link isTransientDoReset}), and a bounded retry lets the
267
+ * budget-wall case recur and surface.
268
+ */
269
+ const DO_CALL_TRANSIENT: ReadonlyArray<readonly [DoCallClass, RegExp]> = [
270
+ ['superseded_isolate', /reset because its code was updated|this script has been upgraded/i],
271
+ ['connection_lost', /network connection lost/i],
272
+ ['storage_reset', /durable object storage caused object to be reset|storage operation exceeded timeout which caused the object to be reset/i],
273
+ ];
274
+
275
+ /**
276
+ * Classify one failed Durable Object call. Overloaded is checked first, per
277
+ * link, because it vetoes everything: an overloaded-and-retryable error is
278
+ * overloaded. The `.retryable` flag is also read per link — it is a
279
+ * property, not prose, so the rendered chain cannot carry it. The walk is
280
+ * cycle-guarded.
281
+ */
282
+ export function classifyDoCall(input: unknown): DoCallClass {
283
+ if (!(input instanceof Error)) return 'permanent';
284
+ const seen = new Set<Error>();
285
+ const messages: string[] = [];
286
+ let flagged = false;
287
+ let link: Error | null = input;
288
+ while (link !== null && !seen.has(link)) {
289
+ seen.add(link);
290
+ const overloaded = ('overloaded' in link && link.overloaded === true)
291
+ || isDoOverloaded(link.message);
292
+ if (overloaded) return 'overloaded';
293
+ if ('retryable' in link && link.retryable === true) flagged = true;
294
+ messages.push(link.message ?? '');
295
+ const cause: unknown = link.cause;
296
+ link = cause instanceof Error ? cause : null;
297
+ }
298
+ const chain = messages.join('; ');
299
+ for (const [transient, pattern] of DO_CALL_TRANSIENT) {
300
+ if (pattern.test(chain)) return transient;
301
+ }
302
+ if (flagged) return 'retryable_flag';
303
+ return 'permanent';
304
+ }
305
+
306
+ /**
307
+ * One line about a failure, in terms someone can act on.
308
+ *
309
+ * npm install's catch sites each wrote `e?.remoteMessage || e?.message ||
310
+ * String(e)`. That reduces a failure to a bare sentence, and when the platform
311
+ * declines to describe a rejected Durable Object call the sentence is the word
312
+ * `internal error` — which is how `resolver-fanout layer 2 failed: internal
313
+ * error` reached users for months carrying nothing at all. The message is
314
+ * often the least we know: the error's class, whether the description came
315
+ * from a remote isolate, and which condition it classifies as are all still in
316
+ * hand, and naming them costs nothing and invents nothing.
317
+ */
318
+ export function describeError(input: unknown): string {
319
+ const remote = typeof input === 'object' && input !== null
320
+ ? (input as { remoteMessage?: unknown }).remoteMessage
321
+ : undefined;
322
+ const message = (typeof remote === 'string' && remote.length > 0)
323
+ ? `${remote} (remote)`
324
+ : readMessage(input) || String(input);
325
+ const name = input instanceof Error && input.name && input.name !== 'Error'
326
+ ? `${input.name}: `
327
+ : '';
328
+ const cause = classifyError(input);
329
+ const suffix = cause !== 'unknown' ? ` [${cause}]`
330
+ : isTransientDoReset(input) ? ' [transient-do-reset]'
331
+ : isDoOverloaded(input) ? ' [do-overloaded]'
332
+ : '';
333
+ return `${name}${message}${suffix}`;
334
+ }
335
+
336
+ function readMessage(input: unknown): string {
337
+ if (input == null) return '';
338
+ if (typeof input === 'string') return input;
339
+ if (input instanceof Error) return input.message ?? '';
340
+ // Some thrown values are plain objects with .message
341
+ if (typeof input === 'object') {
342
+ const m = (input as { message?: unknown }).message;
343
+ if (typeof m === 'string') return m;
344
+ }
345
+ try {
346
+ return String(input);
347
+ } catch {
348
+ return '';
349
+ }
350
+ }