@serfab/cadre-core 1.18.0 → 1.19.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/dist/cadre-node.d.ts +65 -2
  2. package/dist/cadre-node.js +120 -4
  3. package/dist/cadre-node.js.map +1 -1
  4. package/dist/control-read-retry.d.ts +19 -16
  5. package/dist/control-read-retry.js +22 -54
  6. package/dist/control-read-retry.js.map +1 -1
  7. package/dist/control-retry.d.ts +50 -6
  8. package/dist/control-retry.js +53 -6
  9. package/dist/control-retry.js.map +1 -1
  10. package/dist/control-write-retry.d.ts +24 -16
  11. package/dist/control-write-retry.js +58 -30
  12. package/dist/control-write-retry.js.map +1 -1
  13. package/dist/edge-node/config.d.ts +8 -2
  14. package/dist/edge-node/config.js +5 -1
  15. package/dist/edge-node/config.js.map +1 -1
  16. package/dist/edge-node/edge-join.js +2 -0
  17. package/dist/edge-node/edge-join.js.map +1 -1
  18. package/dist/edge-node/edge-node.d.ts +10 -0
  19. package/dist/edge-node/edge-node.js +52 -8
  20. package/dist/edge-node/edge-node.js.map +1 -1
  21. package/dist/edge-node/options.d.ts +11 -2
  22. package/dist/edge-node/options.js +11 -4
  23. package/dist/edge-node/options.js.map +1 -1
  24. package/dist/edge-node/platform.d.ts +15 -3
  25. package/dist/edge-node/strands.js +3 -2
  26. package/dist/edge-node/strands.js.map +1 -1
  27. package/dist/index.d.ts +5 -3
  28. package/dist/index.js +7 -2
  29. package/dist/index.js.map +1 -1
  30. package/dist/peer-join-backfill.d.ts +8 -11
  31. package/dist/peer-join-backfill.js +22 -35
  32. package/dist/peer-join-backfill.js.map +1 -1
  33. package/dist/self-address-update.d.ts +13 -0
  34. package/dist/self-address-update.js +19 -0
  35. package/dist/self-address-update.js.map +1 -0
  36. package/dist/strand-database.d.ts +56 -0
  37. package/dist/strand-database.js +87 -0
  38. package/dist/strand-database.js.map +1 -1
  39. package/dist/strand-instance-manager.d.ts +67 -2
  40. package/dist/strand-instance-manager.js +105 -4
  41. package/dist/strand-instance-manager.js.map +1 -1
  42. package/dist/strand-membership-reconciler.d.ts +13 -1
  43. package/dist/strand-membership-reconciler.js +23 -14
  44. package/dist/strand-membership-reconciler.js.map +1 -1
  45. package/dist/strand-membership-writer.d.ts +16 -2
  46. package/dist/strand-membership-writer.js +54 -8
  47. package/dist/strand-membership-writer.js.map +1 -1
  48. package/dist/strand-offline.d.ts +131 -0
  49. package/dist/strand-offline.js +142 -0
  50. package/dist/strand-offline.js.map +1 -0
  51. package/dist/types.d.ts +35 -0
  52. package/dist/types.js.map +1 -1
  53. package/package.json +8 -8
  54. package/src/cadre-node.ts +11456 -11326
  55. package/src/control-read-retry.ts +22 -58
  56. package/src/control-retry.ts +54 -6
  57. package/src/control-write-retry.ts +58 -30
  58. package/src/edge-node/config.ts +11 -2
  59. package/src/edge-node/edge-join.ts +1 -0
  60. package/src/edge-node/edge-node.ts +55 -6
  61. package/src/edge-node/options.ts +20 -4
  62. package/src/edge-node/platform.ts +15 -3
  63. package/src/edge-node/strands.ts +3 -2
  64. package/src/index.ts +917 -899
  65. package/src/peer-join-backfill.ts +33 -43
  66. package/src/self-address-update.ts +19 -0
  67. package/src/strand-database.ts +466 -352
  68. package/src/strand-instance-manager.ts +2210 -2092
  69. package/src/strand-membership-reconciler.ts +29 -14
  70. package/src/strand-membership-writer.ts +61 -14
  71. package/src/strand-offline.ts +247 -0
  72. package/src/types.ts +2960 -2926
@@ -1,5 +1,5 @@
1
1
  import { BlockUnavailableError } from '@optimystic/db-core';
2
- import { causeChain, chainMessages, retryControlOperation } from './control-retry.js';
2
+ import { causeChain, chainMessages, reportsUnreachableCohort, retryControlOperation } from './control-retry.js';
3
3
  import type { ControlRetryOptions } from './control-retry.js';
4
4
  import { isUncommittedTransactorAggregate } from './control-write-retry.js';
5
5
 
@@ -21,16 +21,14 @@ import { isUncommittedTransactorAggregate } from './control-write-retry.js';
21
21
  * This is deliberately NOT "the write classifier minus its commit veto" — the retriable
22
22
  * set differs in both directions (see {@link isRetriableControlReadFailure}). The loop
23
23
  * lives in `control-retry.ts`, shared with the write policy; this module owns only the
24
- * read policy. And like the write side, the retry classifies by TEXT. That was forced when
25
- * it was written — optimystic's scan path used to rethrow `new Error('Query failed: ' +
26
- * message)` with no `cause` — and is no longer: `OptimysticVirtualTable` now rethrows
27
- * through `rewrapAsQueryError` (`quereus-plugin-optimystic/src/optimystic-module.ts`),
28
- * which keeps the typed `BlockUnavailableError` on `cause`, and Quereus wraps that as
24
+ * read policy. A block-unavailability failure is matched by TYPE: `OptimysticVirtualTable`
25
+ * rethrows a scan failure through `rewrapAsQueryError`
26
+ * (`quereus-plugin-optimystic/src/optimystic-module.ts`), which keeps the typed
27
+ * `BlockUnavailableError` on `cause`, and Quereus wraps that as
29
28
  * `Error during query on table '<T>': …`, again preserving `cause`
30
- * (`quereus/src/runtime/emit/scan.ts`). The text matchers still work because every wrap
31
- * embeds the inner message. Every matcher fails CLOSED — an upstream rewording stops the
32
- * retry engaging, it never makes it unsafe. {@link isCohortUnreachableRead}, which is not a
33
- * retry classifier, matches by type.
29
+ * (`quereus/src/runtime/emit/scan.ts`). The transactor's aggregate is matched by TEXT, since
30
+ * upstream raises it as a plain `Error`. Every matcher fails CLOSED — an upstream rewording or
31
+ * a lost `cause` stops the retry engaging, it never makes it unsafe.
34
32
  */
35
33
 
36
34
  /**
@@ -83,45 +81,6 @@ export const CONTROL_READ_RETRY_DELAYS_MS: readonly number[] = [100, 400];
83
81
  // eslint-disable-next-line no-restricted-syntax -- cuts off by design: a slow attempt is not retried, because the admission gate this budget is sized for has already failed open by then; see docs/cadre-consistency.md → "Deadlines Over Optimystic's Reads and Commits"
84
82
  export const CONTROL_READ_RETRY_BUDGET_MS = 1_500;
85
83
 
86
- /**
87
- * The block-unavailability reasons a repeat read can actually improve on, matched against
88
- * upstream's exact template `Block <id> is unavailable (<reason>): …`
89
- * (`BlockUnavailableError`, `db-core/src/network/struct.ts`):
90
- *
91
- * - **`peers-unreachable`** — part of the cohort answered and part could not be asked, and
92
- * OTHER coordinators are reachable, so asking one of them can still settle it. Upstream's
93
- * own definition invites the second attempt.
94
- * - **`cohort-unreachable`** — the marginal member of the set. Upstream says the answer
95
- * will not improve until this node's connectivity does — but during bring-up connectivity
96
- * DOES improve within a second (the boot-time failure shape recorded in
97
- * `tickets/blocked/control-read-over-fresh-edge-stream-resets`), and the cost of being
98
- * wrong is bounded by {@link CONTROL_READ_RETRY_BUDGET_MS}, which is the whole reason it
99
- * is safe to include.
100
- *
101
- * Deliberately NOT matched:
102
- *
103
- * - **`claimed-elsewhere`** — a cohort peer positively claims the block exists and nobody
104
- * could corroborate or acquire it. MEASURED not to clear: reissuing the same call every
105
- * second for 60 s returned the identical error every time (2026-08-20, recorded in the
106
- * fix ticket). Retrying spends the whole budget and fails anyway. The measured cause (a
107
- * block only one machine held) was fixed upstream by the solo-commit proof in
108
- * `@optimystic/db-p2p` 0.28.0 (`tickets/complete/block-held-by-only-one-machine-is-unreadable`),
109
- * so a `claimed-elsewhere` that surfaces now is a new defect — this retry must not hide it.
110
- * - **`unmaterializable`** — records are held locally but cannot be reassembled. A local
111
- * data problem; a second read reads the same records.
112
- * - **`Block <id> may be stale`** (`BlockPossiblyStaleError`) — about currency, not
113
- * existence: a cohort claim no reachable coordinator could confirm or refute will not
114
- * resolve inside a few hundred milliseconds of backoff.
115
- * - Anything unmatched. The default is no retry.
116
- */
117
- const RETRIABLE_BLOCK_UNAVAILABLE =
118
- /Block \S+ is unavailable \((?:peers-unreachable|cohort-unreachable)\)/;
119
-
120
- /** A block-unavailable read failure whose reason a repeat read can improve on. */
121
- function isRetriableBlockUnavailable(message: string): boolean {
122
- return RETRIABLE_BLOCK_UNAVAILABLE.test(message);
123
- }
124
-
125
84
  /**
126
85
  * Pacing overrides for {@link retryControlRead} — the read twin of
127
86
  * `ControlWriteRetryOptions`. Every field defaults to the shipped read policy
@@ -141,18 +100,20 @@ export type ControlReadRetryOptions = ControlRetryOptions;
141
100
  * `[block:` token, and a read can never produce the commit-phase `[blocks:` token, so
142
101
  * the write side's indeterminate-commit veto has nothing to veto here (a read commits
143
102
  * nothing and re-running it is always safe);
144
- * - the block-unavailability reasons in {@link RETRIABLE_BLOCK_UNAVAILABLE} (and only
145
- * those — see its comment for the measured exclusions).
103
+ * - a block read the cohort could not answer ({@link reportsUnreachableCohort}, the
104
+ * predicate the write classifier shares — see it for the reasons and the measured
105
+ * exclusions).
146
106
  *
147
- * Classifies by MESSAGE, walking the `cause` chain with the shared `chainMessages` (why
148
- * text, in the module comment). Anything that is not an `Error` is never retried.
107
+ * The aggregate is matched by message at any depth of the `cause` chain (`chainMessages`),
108
+ * the unavailability by type on the same chain. Anything that is not an `Error` is never
109
+ * retried, and anything unmatched is not retried either.
149
110
  */
150
111
  export function isRetriableControlReadFailure(error: unknown): boolean {
151
112
  if (!(error instanceof Error)) {
152
113
  return false;
153
114
  }
154
- return chainMessages(error).some(message =>
155
- isUncommittedTransactorAggregate(message) || isRetriableBlockUnavailable(message));
115
+ return reportsUnreachableCohort(causeChain(error))
116
+ || chainMessages(error).some(isUncommittedTransactorAggregate);
156
117
  }
157
118
 
158
119
  /**
@@ -165,9 +126,12 @@ export function isRetriableControlReadFailure(error: unknown): boolean {
165
126
  * never-received `Revocation` block as "no revocations known" (the reasoning is in the NOTE
166
127
  * there). Upstream names this the one reason a caller may treat permissively
167
128
  * (`db-core/src/transactor/network-transactor.ts`). The other reasons do NOT match:
168
- * `peers-unreachable` means part of the cohort answered, and `claimed-elsewhere` /
169
- * `unmaterializable` mean the block is known to exist, so an empty answer could hide a
170
- * real revocation.
129
+ * `peers-unreachable` means part of the cohort answered (or no coordinator did), and
130
+ * `claimed-elsewhere` / `unmaterializable` mean the block is known to exist, so an empty
131
+ * answer could hide a real revocation. NOTE: narrower than upstream's same-named
132
+ * `isCohortUnreachableRead` (`@optimystic/db-core`), which also matches `peers-unreachable`
133
+ * and is what the retry classifiers use ({@link reportsUnreachableCohort}); do not swap one
134
+ * for the other.
171
135
  *
172
136
  * Matched by TYPE, since the typed error survives upstream's rewraps (module comment).
173
137
  * Fails CLOSED: a non-`Error`, a text-only copy of the message, and an error built by a
@@ -1,4 +1,5 @@
1
1
  import debug from 'debug';
2
+ import { isCohortUnreachableRead } from '@optimystic/db-core';
2
3
  import { unwrapError } from '@quereus/quereus';
3
4
 
4
5
  const log = debug('sereus:cadre:control-db');
@@ -232,10 +233,10 @@ function notifyAbandoned(
232
233
  }
233
234
 
234
235
  /**
235
- * Every message in the failure's `cause` chain — the shared substrate both retry classifiers
236
- * match against. They match text (why, in `control-read-retry.ts`'s module comment); every
237
- * wrap on the way out of optimystic and Quereus embeds the inner message, so the text of
238
- * the deepest failure is visible at every level.
236
+ * Every message in the failure's `cause` chain — the substrate both retry classifiers match
237
+ * their text matchers against (the transactor's aggregate and the coordinator's shortfall are
238
+ * plain `Error`s, recognisable only by text); every wrap on the way out of optimystic and
239
+ * Quereus embeds the inner message, so the text of the deepest failure is visible at every level.
239
240
  *
240
241
  * `unwrapError` declares its `message` as `string`, but it follows `.cause` without checking
241
242
  * what that holds — a chain link that is not an `Error` (a stream rejected with a bare string
@@ -256,8 +257,8 @@ export function chainMessages(error: Error): string[] {
256
257
 
257
258
  /**
258
259
  * `error` and every `cause` below it, outermost first — the substrate for the classifiers that
259
- * match by TYPE rather than text (the read side's cohort-unreachable check, the write side's
260
- * possibly-stored veto). Stops at a non-`Error` link (it can carry no further `cause` worth
260
+ * match by TYPE rather than text ({@link reportsUnreachableCohort}, the read side's isolated-node
261
+ * check, the write side's possibly-stored veto). Stops at a non-`Error` link (it can carry no further `cause` worth
261
262
  * trusting) and at a repeat, so a cyclic chain terminates.
262
263
  */
263
264
  export function causeChain(error: unknown): Error[] {
@@ -270,6 +271,53 @@ export function causeChain(error: unknown): Error[] {
270
271
  return links;
271
272
  }
272
273
 
274
+ /**
275
+ * Does any link of the `cause` chain report a block read that failed because the cohort could not
276
+ * be asked — a `BlockUnavailableError` whose reason is `peers-unreachable` or `cohort-unreachable`?
277
+ * The one block-unavailability test both retry classifiers share, so the read and write policies
278
+ * cannot disagree on which reasons a repeat can improve on:
279
+ *
280
+ * - **`peers-unreachable`** — part of the cohort answered and part could not be asked, or no
281
+ * coordinator answered the read at all (`NetworkTransactor.get` flags such a block rather than
282
+ * failing the whole read). Another attempt can reach one that answers; upstream's own definition
283
+ * invites the second attempt.
284
+ * - **`cohort-unreachable`** — the marginal member of the set. Upstream says the answer will not
285
+ * improve until this node's connectivity does — but during bring-up connectivity DOES improve
286
+ * within a second (the boot-time failure shape recorded in
287
+ * `tickets/blocked/control-read-over-fresh-edge-stream-resets`), and each policy's elapsed budget
288
+ * bounds the cost of being wrong, which is the whole reason it is safe to include.
289
+ *
290
+ * Deliberately NOT matched:
291
+ *
292
+ * - **`claimed-elsewhere`** — a cohort peer positively claims the block exists and nobody could
293
+ * corroborate or acquire it. MEASURED not to clear: reissuing the same call every second for 60 s
294
+ * returned the identical error every time (2026-08-20, recorded in the fix ticket). Retrying spends
295
+ * the whole budget and fails anyway. The measured cause (a block only one machine held) was fixed
296
+ * upstream by the solo-commit proof in `@optimystic/db-p2p` 0.28.0
297
+ * (`tickets/complete/block-held-by-only-one-machine-is-unreadable`), so a `claimed-elsewhere` that
298
+ * surfaces now is a new defect — a retry must not hide it.
299
+ * - **`unmaterializable`** — records are held locally but cannot be reassembled. A local data
300
+ * problem; a second read reads the same records.
301
+ * - **`BlockPossiblyStaleError`** (`Block <id> may be stale`) — about currency, not existence: a
302
+ * cohort claim no reachable coordinator could confirm or refute will not resolve inside a few
303
+ * hundred milliseconds of backoff.
304
+ *
305
+ * The reason set is upstream's (`isCohortUnreachableRead`, `db-core/src/transaction/commit-failure-class.ts`),
306
+ * the rule its commit-failure sort and the plugin's offline reads use for "the cohort could not be
307
+ * reached". NOTE: a reason upstream adds to that set is therefore retried here with no change in
308
+ * this repo — the right default for a set defined as unreachability; if it ever grows, re-check it
309
+ * against the exclusions above.
310
+ *
311
+ * Matched by TYPE. Upstream builds this error only on the node doing the read, from the
312
+ * `unavailable` flag a get result carries (`TransactorSource.tryGet`, `NetworkTransactor.getStatus`),
313
+ * never from a peer's error, and every rewrap on the way to either funnel keeps `cause`. NOTE: an
314
+ * error built by a second loaded copy of `@optimystic/db-core`, or one rebuilt as a plain `Error`
315
+ * from its text, fails `instanceof` and is not retried — the safe side.
316
+ */
317
+ export function reportsUnreachableCohort(links: readonly Error[]): boolean {
318
+ return links.some(link => isCohortUnreachableRead(link));
319
+ }
320
+
273
321
  /**
274
322
  * Backoff for the retry that follows attempt `attemptNumber`: the matching base delay
275
323
  * (last entry repeats), jittered ±50%, then capped at the list's largest base so no single
@@ -1,6 +1,6 @@
1
1
  import { CoordinatorPartialCommitError, SyncRetryExhaustedError, TornActionError } from '@optimystic/db-core';
2
2
  import { ConcurrentModificationError, PartialCommitError } from '@optimystic/quereus-plugin-optimystic';
3
- import { causeChain, chainMessages, retryControlOperation } from './control-retry.js';
3
+ import { causeChain, chainMessages, reportsUnreachableCohort, retryControlOperation } from './control-retry.js';
4
4
  import type { ControlRetryOptions } from './control-retry.js';
5
5
 
6
6
  /**
@@ -159,11 +159,17 @@ const SINGLE_BLOCK_BATCH_TOKEN = '[block:';
159
159
  const COMMIT_BATCH_TOKEN = '[blocks:';
160
160
 
161
161
  /**
162
- * The cluster coordinator's super-majority shortfall (`db-p2p/src/repo/cluster-coordinator.ts`),
163
- * matched ONLY with a ZERO rejection count. The same message with a non-zero count means a
164
- * member actually voted no (that branch carries `membership-not-admitted` rejections), and
165
- * retrying it would re-present a spent signature to a cohort that already refused it. This one
166
- * is raised while collecting PROMISES, before any commit, so re-presenting it is safe.
162
+ * The cluster coordinator's super-majority shortfall (`PromiseShortfallError`, db-core), matched
163
+ * ONLY with a ZERO rejection count. The same message with a non-zero count means a member actually
164
+ * voted no (that branch carries `membership-not-admitted` rejections), and retrying it would
165
+ * re-present a spent signature to a cohort that already refused it. A pend's shortfall reaches this
166
+ * funnel as that error on its own, before any commit, so re-presenting it is safe; a commit's
167
+ * arrives inside the `[blocks:` aggregate, which {@link reportsIndeterminateCommit} vetoes (both
168
+ * seen live in `control-write-degraded-cohort-member`).
169
+ *
170
+ * NOTE: matched by TEXT although the error is typed. Upstream keeps this sentence byte for byte as a
171
+ * downstream contract, so the regex is stable; moving to `instanceof PromiseShortfallError` with
172
+ * `rejections === 0` is backlog `debt-control-write-retry-matches-the-shortfall-by-type`.
167
173
  *
168
174
  * This regex itself never matches a decisive rejection (`ValidatorRejectionError`, `Transaction
169
175
  * rejected by validators`) — that text carries no `Failed to get super-majority:` prefix. But the
@@ -183,7 +189,9 @@ const SUPER_MAJORITY_SHORTFALL_UNANSWERED =
183
189
  * re-present:
184
190
  *
185
191
  * - `get` (a block read, which a write body also performs) and `pend` (phase 1) fail before
186
- * anything commits, so the write is known not to have landed;
192
+ * anything commits, so the write is known not to have landed. `get` raises it only when a
193
+ * coordinator answered and left a block out; a block nobody answered for fails typed
194
+ * instead, which {@link matchesRetriableFailure} claims ({@link reportsUnreachableCohort});
187
195
  * - `commitBlocks` (phase 2) is REACHABLE here and NOT safe. `commitBlock` throws the
188
196
  * aggregate when a header/tail commit got NO response at all — precisely the transient
189
197
  * class this retry targets — and that escapes `commit()`, survives `TransactorSource`'s
@@ -206,10 +214,12 @@ const SUPER_MAJORITY_SHORTFALL_UNANSWERED =
206
214
  * NOTE: the discriminator is a formatting detail of another repo. If Optimystic ever reformats
207
215
  * those per-batch details, this fails CLOSED — the aggregate stops matching and control writes
208
216
  * simply stop being retried, silently losing the absorption rather than doing anything unsafe.
209
- * The guard against that is live: `control-write-degraded-cohort-member.integration.ts` asserts
210
- * this classifier against the real degraded-cohort failure object and asserts the `[block:`
211
- * token on the live aggregate; if it reddens there, an upstream reformat is the first thing to
212
- * check.
217
+ * The guard is `control-write-degraded-cohort-member.integration.ts`, which asserts the `[block:`
218
+ * token on any live aggregate a degraded case produces; if it reddens there, an upstream reformat
219
+ * is the first thing to check. NOTE: on Optimystic 1.20.0 none of its cases produced this form
220
+ * (2026-10-28: the injected reset failed the write's read typed, and a pend's shortfall arrives on
221
+ * its own), so the guard can pass without seeing it; if the form is ever needed live again, inject
222
+ * the reset at the pend's batch rather than the read's.
213
223
  *
214
224
  * NOTE: the discriminator is the prefix AND the token within ONE message — never relax it to
215
225
  * "contains `[block:`". Beyond the `get` and `pend` sites above, Optimystic renders `[block:` in
@@ -234,12 +244,14 @@ const SUPER_MAJORITY_SHORTFALL_UNANSWERED =
234
244
  * have left its log entry standing), and its collection sync has already spent ten retries on
235
245
  * the race. The torn-write rule beside it: a `TornActionError` is re-presented only when
236
246
  * `final` is true; any other torn write is declined.
237
- * - What remains rejectable at the promise phase is stale revision, block-unavailable,
238
- * membership-not-admitted, and a configured validator's refusal. Only the first is worth a
239
- * retry — and it is worth one: this loop re-runs the WHOLE write body, reads included
247
+ * - What remains rejectable at the promise phase is stale revision, membership-not-admitted, an
248
+ * expired transaction, and a configured validator's refusal (`ClusterMember.evaluatePromise`).
249
+ * A block whose revision a member cannot establish is approved, not refused: the member logs
250
+ * `validation-block-unavailable` and votes on (`judgeStaleRevisions`). Stale revision is worth
251
+ * a retry: this loop re-runs the WHOLE write body, reads included
240
252
  * (`ControlDatabase.withWriteLock`'s contract), so attempt 2 presents a pend against the
241
253
  * revision that made attempt 1 stale. A blanket "decline any chain reporting a validator
242
- * rejection" rule would remove that retry to save two wasted attempts on the other three.
254
+ * rejection" rule would remove that retry to save two wasted attempts on the others.
243
255
  *
244
256
  * Revisit if a re-presentation ever becomes expensive relative to what it buys — a rejection class
245
257
  * that costs a full cohort round-trip to re-present, or a write body whose reads are no longer
@@ -404,7 +416,9 @@ function isSelfCoordinationGraceRefusal(message: string): boolean {
404
416
  /**
405
417
  * The error messages that mean "the cohort did not answer AND nothing committed", and nothing
406
418
  * else. The observed one-shot failure was `registerSelf()` racing a connection still forming:
407
- * a read/pend-phase transactor aggregate with `cause=The stream has been reset`.
419
+ * a read/pend-phase transactor aggregate with `cause=The stream has been reset`. When none of
420
+ * the cohort answers the read at all, that failure arrives typed instead, and is claimed by
421
+ * type in {@link matchesRetriableFailure} ({@link reportsUnreachableCohort}), not here.
408
422
  *
409
423
  * Deliberately NOT matched: every constraint / authorization failure
410
424
  * (`CHECK constraint failed:`, `UNIQUE constraint failed:`) — those reach the same funnel
@@ -471,16 +485,18 @@ const RETRIABLE_SCHEMA_INIT_MATCHERS: readonly ((message: string) => boolean)[]
471
485
  * Two vetoes run over the whole chain before any matcher: {@link reportsPossiblyStoredWrite}
472
486
  * (by type — a spent sync budget, a non-final torn write, a partial commit) and
473
487
  * {@link reportsIndeterminateCommit} (by text — a commit-phase batch). A chain either veto flags,
474
- * at any level, is never retried on the strength of some other level looking transient. The one
475
- * typed failure claimed as retriable is a final torn write ({@link isFinalTornWrite}).
488
+ * at any level, is never retried on the strength of some other level looking transient. Two typed
489
+ * failures are claimed as retriable: a block read the cohort could not answer
490
+ * ({@link reportsUnreachableCohort}, shared with the read classifier) and a final torn write
491
+ * ({@link isFinalTornWrite}).
476
492
  *
477
493
  * NOTE: this depends on engine/transactor error TEXT. The messages producible without a
478
494
  * network are driven from the REAL engine in `control-formation-seat-budget.spec.ts`, so
479
- * a rewording reddens a spec rather than silently disabling the retry. The two RETRIABLE
480
- * messages need a real multi-node cluster to produce; the degraded-cohort scenario
481
- * (`control-write-degraded-cohort-member.integration.ts`) asserts this classifier against
482
- * both LIVE — the super-majority shortfall from a real silent member, and the transactor
483
- * aggregate from a real stream reset, which the retry there absorbs end to end.
495
+ * a rewording reddens a spec rather than silently disabling the retry. The RETRIABLE failures
496
+ * need a real multi-node cluster to produce; the degraded-cohort scenario
497
+ * (`control-write-degraded-cohort-member.integration.ts`) asserts this classifier against the
498
+ * super-majority shortfall from a real silent member, and absorbs a real stream reset end to end
499
+ * (on Optimystic 1.20.0 the reset fails the write's read as an unanswered block, claimed by type).
484
500
  */
485
501
  export function isRetriableControlWriteFailure(error: unknown): boolean {
486
502
  return matchesRetriableFailure(error, RETRIABLE_CONTROL_WRITE_MATCHERS);
@@ -511,12 +527,13 @@ export function isRetriableLostUpdateFailure(error: unknown): boolean {
511
527
  /**
512
528
  * Shared body of the three classifiers: walk the `cause` chain, let
513
529
  * {@link reportsPossiblyStoredWrite} and then {@link reportsIndeterminateCommit} veto it, then claim
514
- * a final torn write ({@link isFinalTornWrite}) or any link `claimsLink` names, then ask `matchers`.
515
- * The vetoes run BEFORE any claim for every policy — a chain reporting a possibly-stored write or a
516
- * commit-phase batch is never retried, however transient some other level looks, and a final torn
517
- * write or a lost update carried by a partial commit is still declined because a sibling collection
518
- * landed. `claimsLink` is the one typed claim a policy may add; it sits here beside the torn-write
519
- * claim so the vetoes stay in one place above both.
530
+ * a block read the cohort could not answer ({@link reportsUnreachableCohort}), a final torn write
531
+ * ({@link isFinalTornWrite}) or any link `claimsLink` names, then ask `matchers`. The vetoes run
532
+ * BEFORE any claim for every policy — a chain reporting a possibly-stored write or a commit-phase
533
+ * batch is never retried, however transient some other level looks, and a final torn write or a
534
+ * lost update carried by a partial commit is still declined because a sibling collection landed.
535
+ * `claimsLink` is the one typed claim a policy may add; it sits here beside the shared typed claims
536
+ * so the vetoes stay in one place above all of them.
520
537
  */
521
538
  function matchesRetriableFailure(
522
539
  error: unknown,
@@ -534,7 +551,18 @@ function matchesRetriableFailure(
534
551
  if (reportsIndeterminateCommit(messages)) {
535
552
  return false;
536
553
  }
537
- if (links.some(link => isFinalTornWrite(link) || claimsLink?.(link) === true)) {
554
+ // NOTE: an unanswered read can fail AFTER this write's log entry is stored: the refresh between
555
+ // commit attempts reads the collection's log ends, and finishing the write's own entry
556
+ // (`Collection.completeOwnEntry`) asks `getStatus` when no attempt is retained at the entry's
557
+ // revision (a forked lineage — both commit paths retain the failed attempt before refreshing).
558
+ // Either throw escapes the refresh loop unwrapped. Claimed anyway, because the `[block:`
559
+ // aggregate those same reads raise is already claimed by the text matcher, so the typed form
560
+ // adds no new case: the re-run is a new transaction whose body re-reads the blocks, so it diffs
561
+ // to nothing or fails on `UNIQUE` where the earlier write landed, and writes it once where it did
562
+ // not. A refresh that already saved a sibling collection arrives as
563
+ // `CoordinatorPartialCommitError`, which the veto above declines.
564
+ if (reportsUnreachableCohort(links)
565
+ || links.some(link => isFinalTornWrite(link) || claimsLink?.(link) === true)) {
538
566
  return true;
539
567
  }
540
568
  return messages.some(message => matchers.some(matches => matches(message)));
@@ -13,7 +13,7 @@ import type { BootstrapPeerStore } from '../bootstrap-peer-store.js';
13
13
  import type { EnrolledMachineStore } from '../enrolled-machine-store.js';
14
14
  import type { StrandNetworkStateStore } from '../strand-network-state.js';
15
15
  import type { TrustedOwnerStore } from '../trusted-owner-store.js';
16
- import type { IRawStorage, Libp2pTransports, NoiseCryptoInterface } from '@optimystic/db-p2p';
16
+ import type { IKVStore, IRawStorage, Libp2pTransports, NoiseCryptoInterface } from '@optimystic/db-p2p';
17
17
  import type { EdgeNodeOptions } from './options.js';
18
18
 
19
19
  /** What {@link buildEdgeNodeConfig} takes: one start's options plus what the platform built. */
@@ -31,6 +31,11 @@ export interface EdgeNodeConfigInputs extends EdgeNodeOptions {
31
31
  * segment — use it verbatim, do not parse it.
32
32
  */
33
33
  storageProvider: (scope: string) => IRawStorage;
34
+ /**
35
+ * A strand's offline journal (`EdgeNodeStorage.offlineJournal`). Absent: no offline mode, and
36
+ * {@link EdgeNodeOptions.strandOfflineMode} is ignored.
37
+ */
38
+ offlineJournal?: (scope: string) => Promise<IKVStore>;
34
39
  /** libp2p transport factories: what the node dials with, and what listens on `listenAddrs`. */
35
40
  transports: Libp2pTransports;
36
41
  /**
@@ -50,7 +55,8 @@ export interface EdgeNodeConfigInputs extends EdgeNodeOptions {
50
55
  * The edge node's config: transaction profile (Ring Zulu only, intermittent connectivity), no
51
56
  * listen address unless the platform names one, reachability through the configured relays (if
52
57
  * any) but never a hard requirement for one, every strand the control network lists, and no
53
- * automatic hibernation (the app's lifecycle runner hibernates on background). Schema signing is
58
+ * automatic hibernation (the app's lifecycle runner hibernates on background), and offline mode
59
+ * on strands when the platform keeps an offline journal. Schema signing is
54
60
  * left at cadre-core's fail-closed default; an app with unsigned schemas relaxes it in
55
61
  * `EdgeNodePlatform.configure`.
56
62
  */
@@ -102,6 +108,9 @@ export function buildEdgeNodeConfig(inputs: EdgeNodeConfigInputs): CadreNodeConf
102
108
  // gated by `denyDialPeer` and the inbound hooks. cadre-core hands this gater to strand nodes too.
103
109
  ...(inputs.allowPrivateDial ? { connectionGater: { denyDialMultiaddr: () => false } } : {}),
104
110
  },
111
+ ...(inputs.offlineJournal && {
112
+ strandOffline: { journal: inputs.offlineJournal, initialMode: inputs.strandOfflineMode ?? 'online' },
113
+ }),
105
114
  strandFilter: { mode: 'all' },
106
115
  hibernation: { enabled: false },
107
116
  trustedOwners: { store: inputs.trustedOwnerStore },
@@ -140,6 +140,7 @@ async function switchParty(
140
140
  };
141
141
  const noiseCryptoMode = options.noiseCryptoMode ?? leaving?.noiseCryptoMode;
142
142
  if (noiseCryptoMode !== undefined) joined.noiseCryptoMode = noiseCryptoMode;
143
+ if (leaving?.strandOfflineMode !== undefined) joined.strandOfflineMode = leaving.strandOfflineMode;
143
144
  if (formerPartyId !== undefined) joined.formerPartyId = formerPartyId;
144
145
  const movingFrom = founded ? await recordStrandMove(runtime, founded, credential) : undefined;
145
146
  try {
@@ -38,6 +38,7 @@ import type {
38
38
  RemoveOwnerOptions,
39
39
  } from '../types.js';
40
40
  import type { Libp2pTransports, NoiseCryptoInterface } from '@optimystic/db-p2p';
41
+ import type { StrandOfflineMode } from '../strand-offline.js';
41
42
  import { circuitRelayTransport } from '@libp2p/circuit-relay-v2';
42
43
  import { webSockets } from '@libp2p/websockets';
43
44
  import { buildEdgeNodeConfig, requireOwnerKeyLabel, runOwnerGenesis, wireJoinedOwnerKey, type OwnerGenesisResult } from './config.js';
@@ -167,6 +168,15 @@ export interface EdgeNode {
167
168
  * is built. `autoStart` is not cleared on the way, so a crash mid-restart still starts again.
168
169
  */
169
170
  restart(options: EdgeNodeOptions): Promise<CadreNode>;
171
+ /**
172
+ * Change the app's offline mode ({@link EdgeNodeOptions.strandOfflineMode}) without a restart:
173
+ * every strand the running node runs takes it at once (`CadreNode.setStrandOfflineMode`), so
174
+ * does each strand it starts later, and the saved start keeps it, so the next start, a
175
+ * background restart and a push wake start every strand in it. In turn with start, stop and
176
+ * join. Throws while no node runs, and on a platform with no offline journal
177
+ * (`EdgeNodeStorage.offlineJournal`).
178
+ */
179
+ setStrandOfflineMode(mode: StrandOfflineMode): Promise<void>;
170
180
  /**
171
181
  * Join the cadre `credential` names: this app's one node moves to that party, keeping its
172
182
  * identity, so the peer id an owner targeted is the one that redeems. A machine invitation
@@ -362,6 +372,8 @@ class EdgeNodeRuntime implements EdgeNode, JoinRuntime, OwnerGrantRuntime {
362
372
  */
363
373
  private transitions: Promise<unknown> = Promise.resolve();
364
374
  private currentStatus: EdgeNodeStatus = { state: 'stopped' };
375
+ /** The mode {@link setStrandOfflineMode} set on the running node, which it was not built with; null until then. */
376
+ private changedOfflineMode: { node: CadreNode; mode: StrandOfflineMode } | null = null;
365
377
  private lastUnfinishedRemoval: UnfinishedRemoval | null = null;
366
378
  private readonly subscriptions = new Set<Subscription>();
367
379
  private readonly statusListeners = new Set<(status: EdgeNodeStatus) => void>();
@@ -424,6 +436,19 @@ class EdgeNodeRuntime implements EdgeNode, JoinRuntime, OwnerGrantRuntime {
424
436
  });
425
437
  }
426
438
 
439
+ setStrandOfflineMode(mode: StrandOfflineMode): Promise<void> {
440
+ return this.inTurn(async () => {
441
+ const node = this.node;
442
+ const running = this.options;
443
+ if (!node || !running) throw new Error('Cannot set the offline mode: the edge node is not running');
444
+ if (!this.platform.storage.offlineJournal) throw new Error('Cannot set the offline mode: this platform keeps no offline journal');
445
+ for (const strandId of node.getStrands().keys()) node.setStrandOfflineMode(strandId, mode);
446
+ this.changedOfflineMode = { node, mode };
447
+ this.runningOptions = { ...running, strandOfflineMode: mode };
448
+ await this.saveRunningStart(this.runningOptions);
449
+ });
450
+ }
451
+
427
452
  joinCadre(credential: JoinCredential, options: JoinCadreOptions = {}): Promise<JoinCadreResult> {
428
453
  // A start called from here on runs after this join rather than joining a start it replaces.
429
454
  this.starting = null;
@@ -670,6 +695,7 @@ class EdgeNodeRuntime implements EdgeNode, JoinRuntime, OwnerGrantRuntime {
670
695
  this.current = node;
671
696
  for (const subscription of this.subscriptions) subscription.attach(node);
672
697
  this.watchRemoval(node);
698
+ this.followOfflineMode(node);
673
699
  await node.start();
674
700
  return { node, resolved };
675
701
  }
@@ -740,19 +766,22 @@ class EdgeNodeRuntime implements EdgeNode, JoinRuntime, OwnerGrantRuntime {
740
766
  * build's default does not change a device that already ran. Options naming no `origin`
741
767
  * take the saved start's `origin` and `formerPartyId` when it is for the same party, so a
742
768
  * party this app joined, started again from a hand-typed party id, does not run owner genesis.
769
+ * Options naming no `strandOfflineMode` take the saved start's, whatever its party: the mode is
770
+ * the app's choice ({@link setStrandOfflineMode}), and a start built from a form or from options
771
+ * held since before that choice would otherwise drop it.
743
772
  */
744
773
  private async resolveOptions(options: EdgeNodeOptions): Promise<EdgeNodeOptions> {
745
774
  requireOwnerKeyLabel(options.ownerKeyLabel);
746
775
  const resolved = { ...options };
747
776
  const defaultMode = this.platform.noiseCrypto?.defaultMode;
748
777
  if (resolved.noiseCryptoMode === undefined && defaultMode !== undefined) resolved.noiseCryptoMode = defaultMode;
749
- if (resolved.origin === undefined) {
750
- const saved = (await this.readSavedStart())?.options;
751
- if (saved?.partyId === resolved.partyId) {
752
- if (saved.origin !== undefined) resolved.origin = saved.origin;
753
- if (saved.formerPartyId !== undefined) resolved.formerPartyId = saved.formerPartyId;
754
- }
778
+ if (resolved.origin !== undefined && resolved.strandOfflineMode !== undefined) return resolved;
779
+ const saved = (await this.readSavedStart())?.options;
780
+ if (resolved.origin === undefined && saved?.partyId === resolved.partyId) {
781
+ if (saved.origin !== undefined) resolved.origin = saved.origin;
782
+ if (saved.formerPartyId !== undefined) resolved.formerPartyId = saved.formerPartyId;
755
783
  }
784
+ if (resolved.strandOfflineMode === undefined && saved?.strandOfflineMode !== undefined) resolved.strandOfflineMode = saved.strandOfflineMode;
756
785
  return resolved;
757
786
  }
758
787
 
@@ -770,6 +799,7 @@ class EdgeNodeRuntime implements EdgeNode, JoinRuntime, OwnerGrantRuntime {
770
799
  noiseCrypto: this.buildNoiseCrypto(options.noiseCryptoMode),
771
800
  keyStore: this.keyStore,
772
801
  storageProvider: (scope) => platform.storage.rawStorage(scope),
802
+ offlineJournal: platform.storage.offlineJournal?.bind(platform.storage),
773
803
  transports: (platform.transports ?? defaultTransports)(relayAddrs),
774
804
  listenAddrs: platform.listenAddrs?.(relayAddrs) ?? [],
775
805
  allowPrivateDial: platform.allowPrivateDial ?? true,
@@ -861,6 +891,24 @@ class EdgeNodeRuntime implements EdgeNode, JoinRuntime, OwnerGrantRuntime {
861
891
  });
862
892
  }
863
893
 
894
+ /**
895
+ * A strand that starts after {@link setStrandOfflineMode} changed the mode takes the new one:
896
+ * the node applies the mode it was built with (`strandOffline.initialMode`) to every launch.
897
+ */
898
+ // NOTE: this, and setStrandOfflineMode's loop, override a mode an app set on one strand with
899
+ // `CadreNode.setStrandOfflineMode`; if an app ever mixes per-strand modes with this call, keep the per-strand ones.
900
+ private followOfflineMode(node: CadreNode): void {
901
+ node.on('strand:started', ({ strandId }) => {
902
+ const changed = this.changedOfflineMode;
903
+ if (changed?.node !== node) return;
904
+ try {
905
+ node.setStrandOfflineMode(strandId, changed.mode);
906
+ } catch (err) {
907
+ console.warn(`[cadre-core/edge-node] strand ${strandId} started in the mode the node was built with, not ${changed.mode}:`, err);
908
+ }
909
+ });
910
+ }
911
+
864
912
  /**
865
913
  * In turn with start, stop and join: stop the removed party's node, point the saved start away
866
914
  * from that party, delete its local storage and report `removed`. A node of another party, started
@@ -944,6 +992,7 @@ class EdgeNodeRuntime implements EdgeNode, JoinRuntime, OwnerGrantRuntime {
944
992
  const stopping = this.current;
945
993
  this.current = null;
946
994
  this.runningOptions = null;
995
+ this.changedOfflineMode = null;
947
996
  try {
948
997
  if (stopping) {
949
998
  for (const subscription of this.subscriptions) subscription.detach(stopping);
@@ -9,6 +9,8 @@
9
9
  * start the node unattended.
10
10
  */
11
11
 
12
+ import type { StrandOfflineMode } from '../strand-offline.js';
13
+
12
14
  /**
13
15
  * How much of Noise's crypto the platform replaces (`EdgeNodePlatform.noiseCrypto`): none, the
14
16
  * per-frame hash and cipher, or those and the handshake's key exchange too. Named here, apart from
@@ -47,6 +49,13 @@ export interface EdgeNodeOptions {
47
49
  * stays on disk under its own names, for moving its strands and dissolving the old cadre.
48
50
  */
49
51
  formerPartyId?: string;
52
+ /**
53
+ * The offline mode every strand starts in (`CadreNodeConfig.strandOffline.initialMode`); absent,
54
+ * the saved start's, else `'online'`. Read only on a platform with an offline journal (`EdgeNodeStorage.offlineJournal`).
55
+ * `EdgeNode.setStrandOfflineMode` changes it while running, and saves it; `CadreNode.setStrandOfflineMode`
56
+ * changes one strand's mode until the process exits.
57
+ */
58
+ strandOfflineMode?: StrandOfflineMode;
50
59
  }
51
60
 
52
61
  /** Whether this app founded its party, or joined it by invitation. */
@@ -73,10 +82,11 @@ export interface SavedStart {
73
82
  * store, which every start reads.
74
83
  */
75
84
  export function startAfterRemoval(saved: SavedStart): SavedStart | undefined {
76
- const { partyId, formerPartyId, relayAddrs, noiseCryptoMode } = saved.options;
85
+ const { partyId, formerPartyId, relayAddrs, noiseCryptoMode, strandOfflineMode } = saved.options;
77
86
  if (formerPartyId === undefined || formerPartyId === partyId) return undefined;
78
87
  const options: EdgeNodeOptions = { partyId: formerPartyId, bootstrapAddrs: [], relayAddrs, origin: 'founded' };
79
88
  if (noiseCryptoMode !== undefined) options.noiseCryptoMode = noiseCryptoMode;
89
+ if (strandOfflineMode !== undefined) options.strandOfflineMode = strandOfflineMode;
80
90
  return { options, autoStart: false, ...(saved.writtenBy !== undefined && { writtenBy: saved.writtenBy }) };
81
91
  }
82
92
 
@@ -99,12 +109,13 @@ interface StartRecord {
99
109
  origin?: EdgeNodeOrigin;
100
110
  formerPartyId?: string;
101
111
  ownerKeyLabel?: string;
112
+ strandOfflineMode?: StrandOfflineMode;
102
113
  autoStart: boolean;
103
114
  writtenBy?: string;
104
115
  }
105
116
 
106
117
  export function serializeSavedStart(saved: SavedStart): string {
107
- const { partyId, bootstrapAddrs, relayAddrs, noiseCryptoMode, origin, formerPartyId, ownerKeyLabel } = saved.options;
118
+ const { partyId, bootstrapAddrs, relayAddrs, noiseCryptoMode, origin, formerPartyId, ownerKeyLabel, strandOfflineMode } = saved.options;
108
119
  const record: StartRecord = {
109
120
  version: RECORD_VERSION,
110
121
  partyId,
@@ -114,6 +125,7 @@ export function serializeSavedStart(saved: SavedStart): string {
114
125
  origin,
115
126
  formerPartyId,
116
127
  ownerKeyLabel,
128
+ strandOfflineMode,
117
129
  autoStart: saved.autoStart,
118
130
  writtenBy: saved.writtenBy,
119
131
  };
@@ -125,8 +137,9 @@ export function serializeSavedStart(saved: SavedStart): string {
125
137
  *
126
138
  * Only the party id is worth discarding a record over. A malformed optional field falls back to
127
139
  * its default — `[]` for an address list, the build default for the Noise mode, absent for
128
- * `origin`, `formerPartyId` and `ownerKeyLabel`, `false` for `autoStart` — so a damaged record still keeps the
129
- * app in its party, at worst needing the user to start the node by hand.
140
+ * `origin`, `formerPartyId`, `ownerKeyLabel` and `strandOfflineMode` (which is `'online'`), `false`
141
+ * for `autoStart` — so a damaged record still keeps the app in its party, at worst needing the
142
+ * user to start the node by hand.
130
143
  */
131
144
  export function parseSavedStart(text: string | undefined): SavedStart | undefined {
132
145
  if (text === undefined) return undefined;
@@ -148,6 +161,9 @@ export function parseSavedStart(text: string | undefined): SavedStart | undefine
148
161
  options.formerPartyId = record.formerPartyId;
149
162
  }
150
163
  if (typeof record.ownerKeyLabel === 'string') options.ownerKeyLabel = record.ownerKeyLabel;
164
+ if (record.strandOfflineMode === 'online' || record.strandOfflineMode === 'offline') {
165
+ options.strandOfflineMode = record.strandOfflineMode;
166
+ }
151
167
  const saved: SavedStart = { options, autoStart: record.autoStart === true };
152
168
  if (typeof record.writtenBy === 'string') saved.writtenBy = record.writtenBy;
153
169
  return saved;