@serfab/cadre-core 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/dist/cadre-node.d.ts +70 -34
- package/dist/cadre-node.js +197 -60
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-database.d.ts +24 -10
- package/dist/control-database.js +48 -16
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +4 -4
- package/dist/control-formation-recorder.js +6 -6
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-read-retry.d.ts +12 -0
- package/dist/control-read-retry.js +12 -0
- package/dist/control-read-retry.js.map +1 -1
- package/dist/control-write-retry.d.ts +5 -4
- package/dist/control-write-retry.js +24 -7
- package/dist/control-write-retry.js.map +1 -1
- package/dist/index.d.ts +5 -4
- package/dist/index.js +10 -5
- package/dist/index.js.map +1 -1
- package/dist/link-budget.d.ts +182 -0
- package/dist/link-budget.js +195 -0
- package/dist/link-budget.js.map +1 -0
- package/dist/membership-connection-gater.d.ts +18 -5
- package/dist/membership-connection-gater.js +50 -10
- package/dist/membership-connection-gater.js.map +1 -1
- package/dist/peer-dial.d.ts +36 -17
- package/dist/peer-dial.js +37 -17
- package/dist/peer-dial.js.map +1 -1
- package/dist/peer-join-backfill.d.ts +58 -4
- package/dist/peer-join-backfill.js +165 -20
- package/dist/peer-join-backfill.js.map +1 -1
- package/dist/relay-reservation.d.ts +59 -11
- package/dist/relay-reservation.js +169 -43
- package/dist/relay-reservation.js.map +1 -1
- package/dist/seed-bootstrap.js +2 -1
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/storage-scope.d.ts +50 -12
- package/dist/storage-scope.js +85 -12
- package/dist/storage-scope.js.map +1 -1
- package/dist/strand-first-sync-gate.d.ts +43 -5
- package/dist/strand-first-sync-gate.js +55 -7
- package/dist/strand-first-sync-gate.js.map +1 -1
- package/dist/strand-formation-manager.js +2 -1
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-id.d.ts +35 -0
- package/dist/strand-id.js +46 -0
- package/dist/strand-id.js.map +1 -0
- package/dist/strand-instance-manager.d.ts +27 -2
- package/dist/strand-instance-manager.js +75 -15
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-reconciler.d.ts +200 -45
- package/dist/strand-membership-reconciler.js +302 -86
- package/dist/strand-membership-reconciler.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +3 -2
- package/dist/strand-membership-writer.js +67 -90
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-network-config.d.ts +3 -3
- package/dist/strand-network-config.js +3 -3
- package/dist/strand-solicitation.js +2 -1
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-watcher.d.ts +45 -0
- package/dist/strand-watcher.js +74 -13
- package/dist/strand-watcher.js.map +1 -1
- package/dist/types.d.ts +190 -24
- package/dist/types.js +73 -4
- package/dist/types.js.map +1 -1
- package/package.json +8 -8
- package/src/cadre-node.ts +7445 -7302
- package/src/control-database.ts +48 -16
- package/src/control-formation-recorder.ts +6 -6
- package/src/control-read-retry.ts +214 -202
- package/src/control-write-retry.ts +24 -7
- package/src/index.ts +29 -2
- package/src/link-budget.ts +217 -0
- package/src/membership-connection-gater.ts +56 -11
- package/src/peer-dial.ts +39 -17
- package/src/peer-join-backfill.ts +192 -23
- package/src/relay-reservation.ts +1119 -951
- package/src/seed-bootstrap.ts +2 -1
- package/src/storage-scope.ts +163 -78
- package/src/strand-first-sync-gate.ts +55 -7
- package/src/strand-formation-manager.ts +647 -646
- package/src/strand-id.ts +48 -0
- package/src/strand-instance-manager.ts +1516 -1440
- package/src/strand-membership-reconciler.ts +334 -92
- package/src/strand-membership-writer.ts +1962 -1987
- package/src/strand-network-config.ts +250 -250
- package/src/strand-solicitation.ts +500 -499
- package/src/strand-watcher.ts +388 -325
- package/src/types.ts +192 -23
|
@@ -26,39 +26,93 @@
|
|
|
26
26
|
* that finds no member row AND no staged invitation is not an unfinished join but a wait
|
|
27
27
|
* for someone to admit this party — nothing this machine can do faster — so it re-arms on
|
|
28
28
|
* the flat poll interval and resets the ladder. The loop stops on the done state.
|
|
29
|
-
* - **Done state**: the party's member row is visible locally
|
|
30
|
-
*
|
|
31
|
-
* business, not this loop's; a resume rebuilds the reconciler and
|
|
32
|
-
* scratch (every write is idempotent, so re-running is free)
|
|
29
|
+
* - **Done state**: the party's member row is visible locally, this machine's own binding
|
|
30
|
+
* is in place, AND no invitation is still staged — the loop stops. A LATER revocation is
|
|
31
|
+
* the enforcer's business, not this loop's; a resume rebuilds the reconciler and
|
|
32
|
+
* re-verifies from scratch (every write is idempotent, so re-running is free), and a
|
|
33
|
+
* fresh invitation staged on a done loop re-arms it in place ({@link rearm}).
|
|
33
34
|
*
|
|
34
35
|
* ## One pass
|
|
35
36
|
*
|
|
36
|
-
* 1. **Self-revocation check.** If the enforcer flags this node's own party as revoked
|
|
37
|
-
* stop rather than fight — re-admission arrives (if ever) via
|
|
37
|
+
* 1. **Self-revocation check.** If the enforcer flags this node's own party as revoked and
|
|
38
|
+
* no invitation is staged, stop rather than fight — re-admission arrives (if ever) via
|
|
39
|
+
* a fresh formation, which re-arms the loop. With an invitation staged the loop reports
|
|
40
|
+
* the dead end (below) and carries on: the redemption is attempted like any other, and a
|
|
41
|
+
* manager's re-admission lets a later pass finish it.
|
|
38
42
|
* 2. **Ensure membership.** Member row visible → done with this step; if a staged
|
|
39
43
|
* invitation is still unspent, BURN it ({@link burnInvite} — the `ConsumedInvite` row
|
|
40
|
-
* alone) so the bearer credential cannot be spent by anyone else
|
|
41
|
-
*
|
|
44
|
+
* alone) so the bearer credential cannot be spent by anyone else. A burn refused by the
|
|
45
|
+
* cohort keeps the invitation staged for the next pass (it is the same write gate a
|
|
46
|
+
* redemption faces, and the local member row may be stale — see "Re-arming"); a dead
|
|
47
|
+
* invitation is dropped. Member row absent + staged invitation → {@link consumeInvite}
|
|
42
48
|
* under the party's own key, retried across passes until the `Invite` row has
|
|
43
49
|
* replicated here and the write commits. Absent with no invitation (founder rows not
|
|
44
50
|
* yet synced, or a machine whose party never joined) → keep waiting, escalating to one
|
|
45
51
|
* visible warning after {@link IDLE_PASSES_BEFORE_ESCALATION} passes, never throwing.
|
|
46
52
|
* 3. **Ensure the binding.** {@link registerMemberPeer} (insert-if-absent, restart-safe)
|
|
47
53
|
* with this node's own strand transport peer id — only after step 2 sees the member
|
|
48
|
-
* row locally (its deferred `MemberExists` reads the live local table).
|
|
54
|
+
* row locally (its deferred `MemberExists` reads the live local table). Done only once
|
|
55
|
+
* the binding is in place AND nothing is staged; a binding written while a burn keeps
|
|
56
|
+
* being refused leaves the loop live to settle the invitation later.
|
|
49
57
|
*
|
|
50
|
-
* ##
|
|
58
|
+
* ## Stops, and re-arming
|
|
59
|
+
*
|
|
60
|
+
* Two kinds of stop. A RE-ARMABLE stop is one a fresh invitation should reopen:
|
|
61
|
+
*
|
|
62
|
+
* - **Done** (above).
|
|
63
|
+
* - **Self-revoked with nothing staged** (step 1 above).
|
|
64
|
+
*
|
|
65
|
+
* A PERMANENT stop is one a fresh invitation changes nothing about — {@link rearm} is a
|
|
66
|
+
* no-op after it:
|
|
51
67
|
*
|
|
52
68
|
* - **Sealed strand**: `consumeInvite` rejected by `ConsumedInvite.NotSealed` — nobody
|
|
53
69
|
* can ever be admitted, so retrying forever is noise. Terminal-with-log.
|
|
54
|
-
* - **Self-revoked** (step 1 above).
|
|
55
70
|
* - **Undecodable party key**: nothing can be signed; loud log, stop.
|
|
71
|
+
* - **The public {@link stop}**: `StrandInstanceManager.releaseRuntime` and
|
|
72
|
+
* `clearOwnMemberPeerBinding` both call it and mean it — the latter stops the loop and
|
|
73
|
+
* awaits {@link settle} precisely so no pass can re-register the binding it is about to
|
|
74
|
+
* delete, and a re-arm that restarted the loop would undo that.
|
|
75
|
+
*
|
|
76
|
+
* `CadreNode.adoptFormationMembershipInvite` stages every fresh invitation and notifies
|
|
77
|
+
* the instance manager, which calls {@link rearm}: a done loop reopens and runs one pass
|
|
78
|
+
* at once; a running loop is merely kicked. This is what lets a party that was REMOVED
|
|
79
|
+
* from the strand, and then handed a new invitation, actually attempt it — the loop
|
|
80
|
+
* finished during its first join, long before the removal.
|
|
81
|
+
*
|
|
82
|
+
* On a removed party the local replica is usually stale: the cohort cut this machine off
|
|
83
|
+
* at about the moment the removal was written, so `isStrandMember` still answers true and
|
|
84
|
+
* a re-armed pass lands in the already-member arm. That is why the burn arm keeps a
|
|
85
|
+
* refused invitation staged and why the done state requires nothing staged — otherwise
|
|
86
|
+
* the fresh credential would be dropped on the first refused burn and the loop would latch
|
|
87
|
+
* done again, with the only trace a debug line.
|
|
56
88
|
*
|
|
57
89
|
* A DEAD invitation that is not terminal for the loop — expired, cancelled, or already
|
|
58
90
|
* consumed by someone else (`ConsumedInvite`'s primary key) — is dropped with a log and
|
|
59
91
|
* the loop keeps idling: a fresh formation stages a new invitation, and a manager-side
|
|
60
92
|
* admission (`addMemberByManager`) seats the member row without one.
|
|
61
93
|
*
|
|
94
|
+
* ## Reporting a blocked re-join
|
|
95
|
+
*
|
|
96
|
+
* Redeeming (or burning) writes into the strand, and the machines that would carry that
|
|
97
|
+
* write are exactly the ones the remaining members refuse once this party is removed. So
|
|
98
|
+
* the attempt is expected to fail, and the failure is REPORTED rather than left to the
|
|
99
|
+
* ladder: one `console.warn` plus the {@link StrandMembershipReconcilerDeps.onRejoinBlocked}
|
|
100
|
+
* callback (`CadreNode`'s `strand:rejoin-blocked` event), at most once per re-arm cycle.
|
|
101
|
+
* Two triggers:
|
|
102
|
+
*
|
|
103
|
+
* - **Confirmed** — a pass finds this party self-revoked AND an invitation staged. The node
|
|
104
|
+
* knows it was removed and is holding a credential it cannot spend.
|
|
105
|
+
* - **Probable** — {@link UNFINISHED_PASSES_BEFORE_ESCALATION} consecutive attempted passes
|
|
106
|
+
* left the invitation staged. The case that matters most in the field: a removed party
|
|
107
|
+
* that never learned it was removed (the removal did not replicate here before the cut).
|
|
108
|
+
* It is a suspicion, not a verdict — the invitation's `Invite` row may simply not have
|
|
109
|
+
* replicated here yet — and the warning names both causes.
|
|
110
|
+
*
|
|
111
|
+
* The remedy in either case is a remaining manager admitting this party's member key
|
|
112
|
+
* directly (`addMemberByManager`): that lifts the refusal, replication resumes, and the
|
|
113
|
+
* still-running loop sees the member row, burns the leftover credential and finishes by
|
|
114
|
+
* itself.
|
|
115
|
+
*
|
|
62
116
|
* ## Sharing the database with the app
|
|
63
117
|
*
|
|
64
118
|
* The strand `Database` this loop writes to is the one the app holds. Every write here
|
|
@@ -111,6 +165,17 @@ const log = debug('sereus:cadre:strand-membership-reconciler');
|
|
|
111
165
|
*/
|
|
112
166
|
export const IDLE_PASSES_BEFORE_ESCALATION = 10;
|
|
113
167
|
|
|
168
|
+
/**
|
|
169
|
+
* Consecutive passes that ran against a live database and ended with the staged
|
|
170
|
+
* invitation still unsettled — whatever refused it — before the loop reports a probable
|
|
171
|
+
* blocked re-join (see "Reporting a blocked re-join" in the module doc). The same order
|
|
172
|
+
* as {@link IDLE_PASSES_BEFORE_ESCALATION}: about three minutes at the default cadence
|
|
173
|
+
* (the ladder's five doubling rungs, then five passes at the 30 s cap). Counted by outcome
|
|
174
|
+
* rather than by classified failure because a cut-off machine can fail before any write is
|
|
175
|
+
* classified: its membership READ may already throw.
|
|
176
|
+
*/
|
|
177
|
+
export const UNFINISHED_PASSES_BEFORE_ESCALATION = 10;
|
|
178
|
+
|
|
114
179
|
/**
|
|
115
180
|
* First retry delay, ms, for a pass that left the join UNFINISHED — a `consumeInvite`
|
|
116
181
|
* whose `Strand.Invite` row has not replicated to this machine yet, a cohort briefly
|
|
@@ -159,25 +224,30 @@ export interface HalfCommittedJoin {
|
|
|
159
224
|
unsaved: readonly string[];
|
|
160
225
|
}
|
|
161
226
|
|
|
162
|
-
/** Where {@link classifyConsumeFailure} routes a failed `consumeInvite`. */
|
|
227
|
+
/** Where {@link classifyConsumeFailure} routes a failed `consumeInvite` or `burnInvite`. */
|
|
163
228
|
export type ConsumeFailure =
|
|
164
229
|
| HalfCommittedJoin
|
|
165
230
|
/** The app had a transaction open, so nothing was tried — keep the invitation, retry. */
|
|
166
231
|
| { kind: 'busy' }
|
|
167
|
-
/** The strand is sealed — terminal. */
|
|
232
|
+
/** The strand is sealed — terminal for a non-member; a member's leftover invitation is merely dead. */
|
|
168
233
|
| { kind: 'sealed' }
|
|
169
234
|
/** Expired, cancelled, or consumed by someone else — drop the invitation, keep waiting. */
|
|
170
235
|
| { kind: 'dead-invite' }
|
|
171
|
-
/**
|
|
236
|
+
/**
|
|
237
|
+
* Anything else — the `Invite` row has not replicated here yet, or the cohort refused the
|
|
238
|
+
* write (a removed party's machines are denied) — keep the invitation, retry.
|
|
239
|
+
*/
|
|
172
240
|
| { kind: 'retry' };
|
|
173
241
|
|
|
174
242
|
/**
|
|
175
|
-
* Route a `consumeInvite` rejection.
|
|
176
|
-
*
|
|
177
|
-
*
|
|
178
|
-
*
|
|
179
|
-
*
|
|
180
|
-
*
|
|
243
|
+
* Route a `consumeInvite` or `burnInvite` rejection. Both write the `Strand.ConsumedInvite`
|
|
244
|
+
* row under the same `NotExpired` / `NotCancelled` / `NotSealed` / primary-key constraints
|
|
245
|
+
* (a burn writes that row alone), so one classifier serves both. Typed checks run before any
|
|
246
|
+
* text check: a busy refusal (another transaction was open, so nothing ran) and a
|
|
247
|
+
* half-committed join are recognised by their error's type anywhere in the `cause` chain.
|
|
248
|
+
* The busy refusal carries Quereus's own refusal as its cause, and a half-committed join's
|
|
249
|
+
* message embeds collection names and the underlying failure's text, either of which a text
|
|
250
|
+
* matcher can misread. The sealed and dead-invitation checks then read the top-level message.
|
|
181
251
|
*
|
|
182
252
|
* A second loaded copy of `@optimystic/db-core` (or of the plugin) would fail `instanceof`; the
|
|
183
253
|
* half-commit then falls through the anchored texts to `retry`, and once the saved `ConsumedInvite`
|
|
@@ -234,8 +304,12 @@ function errorMessage(error: unknown): string {
|
|
|
234
304
|
export interface PendingMembershipInviteSource {
|
|
235
305
|
/** The staged invitation, or `undefined` when none is pending for this strand. */
|
|
236
306
|
get(): StrandMembershipInvite | undefined;
|
|
237
|
-
/**
|
|
238
|
-
|
|
307
|
+
/**
|
|
308
|
+
* Drop `settled` (spent, burned, or dead) — only if it is still the staged one. A
|
|
309
|
+
* re-formation can replace the entry between a pass's {@link get} and this call, and
|
|
310
|
+
* the fresh invitation it staged must survive the older one being settled.
|
|
311
|
+
*/
|
|
312
|
+
clear(settled: StrandMembershipInvite): void;
|
|
239
313
|
}
|
|
240
314
|
|
|
241
315
|
export interface StrandMembershipReconcilerDeps {
|
|
@@ -262,10 +336,17 @@ export interface StrandMembershipReconcilerDeps {
|
|
|
262
336
|
pendingInvite?: PendingMembershipInviteSource;
|
|
263
337
|
/**
|
|
264
338
|
* Whether the revocation enforcer currently flags THIS node's own party as revoked.
|
|
265
|
-
* When it does, the loop stops rather than fight the enforcer
|
|
266
|
-
*
|
|
339
|
+
* When it does and nothing is staged, the loop stops rather than fight the enforcer;
|
|
340
|
+
* with an invitation staged it reports the blocked re-join and keeps trying — see the
|
|
341
|
+
* module doc. Absent (enforcement disarmed) means "not known revoked".
|
|
267
342
|
*/
|
|
268
343
|
isSelfRevoked?: () => boolean;
|
|
344
|
+
/**
|
|
345
|
+
* Called when the loop reports a blocked re-join (see "Reporting a blocked re-join" in
|
|
346
|
+
* the module doc) — at most once per re-arm cycle, after the `console.warn`. `CadreNode`
|
|
347
|
+
* wires it to its `strand:rejoin-blocked` event.
|
|
348
|
+
*/
|
|
349
|
+
onRejoinBlocked?: () => void;
|
|
269
350
|
/**
|
|
270
351
|
* Timer seam for the retry ladder; omit for real (unref'd) timeouts. Timeouts rather
|
|
271
352
|
* than the enforcer's repeating interval because the delay changes per pass and the next
|
|
@@ -307,11 +388,21 @@ export class StrandMembershipReconciler {
|
|
|
307
388
|
private timer: unknown;
|
|
308
389
|
private started = false;
|
|
309
390
|
private stoppedFlag = false;
|
|
391
|
+
/**
|
|
392
|
+
* Whether the last stop was one {@link rearm} may not undo — see "Stops, and re-arming"
|
|
393
|
+
* in the module doc. Latched by the public {@link stop} and the permanent terminal
|
|
394
|
+
* states, never cleared.
|
|
395
|
+
*/
|
|
396
|
+
private permanentlyStopped = false;
|
|
310
397
|
private doneFlag = false;
|
|
311
398
|
/** Lazily decoded party keypair — see {@link StrandMembershipReconcilerDeps.partyMemberPrivateKey}. */
|
|
312
399
|
private keyPair: Ed25519KeyPair | undefined;
|
|
313
400
|
private idlePasses = 0;
|
|
314
401
|
private idleEscalated = false;
|
|
402
|
+
/** Consecutive attempted passes that left the invitation staged — see {@link UNFINISHED_PASSES_BEFORE_ESCALATION}. */
|
|
403
|
+
private unfinishedPasses = 0;
|
|
404
|
+
/** Whether the blocked re-join was already reported this re-arm cycle (one report, either trigger). */
|
|
405
|
+
private rejoinBlockedReported = false;
|
|
315
406
|
/**
|
|
316
407
|
* Whether the pass that just ran found nothing to act on — no member row and no staged
|
|
317
408
|
* invitation. That is a wait to be admitted, not an unfinished join, so it re-arms on the
|
|
@@ -335,12 +426,15 @@ export class StrandMembershipReconciler {
|
|
|
335
426
|
this.pollIntervalMs = config?.pollIntervalMs ?? DEFAULT_REVOCATION_POLL_INTERVAL_MS;
|
|
336
427
|
}
|
|
337
428
|
|
|
338
|
-
/**
|
|
429
|
+
/**
|
|
430
|
+
* True once member row + own binding were both confirmed with nothing staged and the
|
|
431
|
+
* loop stopped. Cleared again by {@link rearm}.
|
|
432
|
+
*/
|
|
339
433
|
get done(): boolean {
|
|
340
434
|
return this.doneFlag;
|
|
341
435
|
}
|
|
342
436
|
|
|
343
|
-
/** True
|
|
437
|
+
/** True while the loop is stopped — done, terminal, or externally stopped. */
|
|
344
438
|
get stopped(): boolean {
|
|
345
439
|
return this.stoppedFlag;
|
|
346
440
|
}
|
|
@@ -369,12 +463,24 @@ export class StrandMembershipReconciler {
|
|
|
369
463
|
await this.tail;
|
|
370
464
|
}
|
|
371
465
|
|
|
372
|
-
/**
|
|
466
|
+
/**
|
|
467
|
+
* Disarm the retry timer, for good: a later {@link rearm} is refused. A pass already in
|
|
468
|
+
* flight completes but writes idempotently — {@link settle} is what waits it out.
|
|
469
|
+
*/
|
|
373
470
|
stop(): void {
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
471
|
+
this.halt('stopped by the runtime', true);
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/**
|
|
475
|
+
* A fresh invitation was staged: reopen a loop that stopped on a re-armable state (done,
|
|
476
|
+
* or self-revoked with nothing staged) and run one pass at once; merely kick a loop that
|
|
477
|
+
* is still running; do nothing after a permanent stop. Never rejects, and serialized
|
|
478
|
+
* like {@link reconcile}: the reopen happens when the queued pass STARTS, after any pass
|
|
479
|
+
* in flight has settled, so a `done` that pass latches cannot swallow the re-arm — and a
|
|
480
|
+
* public {@link stop} that lands first still wins, because the queued pass re-checks it.
|
|
481
|
+
*/
|
|
482
|
+
rearm(): Promise<void> {
|
|
483
|
+
return this.enqueue(() => this.reopen() ? this.doPass() : Promise.resolve());
|
|
378
484
|
}
|
|
379
485
|
|
|
380
486
|
/**
|
|
@@ -385,13 +491,18 @@ export class StrandMembershipReconciler {
|
|
|
385
491
|
* published — REPLACES the pending timer rather than running alongside it.
|
|
386
492
|
*/
|
|
387
493
|
reconcile(): Promise<void> {
|
|
494
|
+
return this.enqueue(() => this.doPass());
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
/** Chain `pass` after every pass already queued; the tail never rejects. */
|
|
498
|
+
private enqueue(pass: () => Promise<void>): Promise<void> {
|
|
388
499
|
// The catch is what keeps the chain alive. `doPass` contains its own failures, but its
|
|
389
500
|
// `finally` calls into the injected scheduler, and a throw from there would leave `tail`
|
|
390
501
|
// REJECTED — after which every later `reconcile()` short-circuits on it and the join
|
|
391
|
-
// stalls for good, silently. Swallowing it here makes the "never rejects" contract
|
|
392
|
-
//
|
|
393
|
-
// than incidental.
|
|
394
|
-
const run = this.tail.then(
|
|
502
|
+
// stalls for good, silently. Swallowing it here makes the "never rejects" contract
|
|
503
|
+
// {@link reconcile}, {@link rearm}, {@link settle} and the `void`-ed call sites all rely
|
|
504
|
+
// on structural rather than incidental.
|
|
505
|
+
const run = this.tail.then(pass).catch((error) => {
|
|
395
506
|
log('[%s] reconcile pass threw outside its own handler — the chain continues: %o',
|
|
396
507
|
this.deps.label, error);
|
|
397
508
|
});
|
|
@@ -399,28 +510,68 @@ export class StrandMembershipReconciler {
|
|
|
399
510
|
return run;
|
|
400
511
|
}
|
|
401
512
|
|
|
513
|
+
/**
|
|
514
|
+
* The re-arm itself, run at the head of the queued pass: `false` refuses the pass (a
|
|
515
|
+
* permanent stop), a running loop is left as it is, and a re-armable stop is reopened
|
|
516
|
+
* with the idle and unfinished counters, both escalation latches and the retry ladder
|
|
517
|
+
* reset — a fresh invitation is a fresh cycle.
|
|
518
|
+
*/
|
|
519
|
+
private reopen(): boolean {
|
|
520
|
+
if (this.permanentlyStopped) {
|
|
521
|
+
log('[%s] re-arm refused — the loop is stopped for good', this.deps.label);
|
|
522
|
+
return false;
|
|
523
|
+
}
|
|
524
|
+
if (!this.stoppedFlag) {
|
|
525
|
+
// NOTE: a running loop keeps its counters and its once-per-cycle report latch, so a
|
|
526
|
+
// party whose loop already reported and is then re-formed AGAIN gets no second report
|
|
527
|
+
// for the replacement invitation. If that second report is ever wanted, key the latch
|
|
528
|
+
// on the staged invitation's key rather than on the re-arm.
|
|
529
|
+
log('[%s] fresh invitation staged on a running loop — kicking a pass', this.deps.label);
|
|
530
|
+
return true;
|
|
531
|
+
}
|
|
532
|
+
this.stoppedFlag = false;
|
|
533
|
+
this.doneFlag = false;
|
|
534
|
+
this.idlePasses = 0;
|
|
535
|
+
this.idleEscalated = false;
|
|
536
|
+
this.unfinishedPasses = 0;
|
|
537
|
+
this.rejoinBlockedReported = false;
|
|
538
|
+
this.retryDelayMs = undefined;
|
|
539
|
+
log('[%s] membership reconciler re-armed by a fresh invitation', this.deps.label);
|
|
540
|
+
return true;
|
|
541
|
+
}
|
|
542
|
+
|
|
402
543
|
/** One serialized pass; contains every failure (contract: never rejects). */
|
|
403
544
|
private async doPass(): Promise<void> {
|
|
404
545
|
if (this.stoppedFlag || this.doneFlag) return;
|
|
405
546
|
this.lastPassIdle = false;
|
|
406
547
|
try {
|
|
407
|
-
if (this.deps.isSelfRevoked?.() === true) {
|
|
408
|
-
// NOTE:
|
|
409
|
-
//
|
|
410
|
-
//
|
|
411
|
-
//
|
|
412
|
-
//
|
|
413
|
-
|
|
414
|
-
// its first join long before it was removed. Measured end to end in
|
|
415
|
-
// `strand-party-removal-via-formation-e2e.integration.ts` (test 2), tracked as
|
|
416
|
-
// `backlog/bug-removed-party-cannot-redeem-its-way-back`.
|
|
417
|
-
this.finish('this party is revoked from the strand — re-admission arrives (if ever) via a fresh formation');
|
|
548
|
+
if (this.deps.isSelfRevoked?.() === true && !this.stagedInviteBlockedBySelfRevocation()) {
|
|
549
|
+
// NOTE: the enforcer's flag lags a re-admission by up to one of its polls. If the
|
|
550
|
+
// pass that settled the last staged invitation then failed to write the binding, this
|
|
551
|
+
// stop lands with the binding missing until the next resume rebuilds the loop. A
|
|
552
|
+
// missing binding only mis-credits diversity today; if it ever gates admission, make
|
|
553
|
+
// this stop also require the binding to be in place.
|
|
554
|
+
this.finish('this party is revoked from the strand — re-admission arrives (if ever) via a fresh formation', false);
|
|
418
555
|
return;
|
|
419
556
|
}
|
|
420
557
|
const keyPair = this.resolveKeyPair();
|
|
421
558
|
if (!keyPair) return; // undecodable key already stopped the loop
|
|
422
559
|
const db = this.deps.getDatabase();
|
|
423
560
|
if (!db) return; // no live database this instant (quiesce race) — next tick decides
|
|
561
|
+
await this.attempt(db, keyPair);
|
|
562
|
+
} catch (error) {
|
|
563
|
+
log('[%s] reconcile pass failed — retrying next tick: %o', this.deps.label, error);
|
|
564
|
+
} finally {
|
|
565
|
+
this.scheduleNext();
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
/**
|
|
570
|
+
* Steps 2 and 3 against a live database, with the outcome counted toward the probable
|
|
571
|
+
* blocked re-join report whether the attempt returned or threw.
|
|
572
|
+
*/
|
|
573
|
+
private async attempt(db: Database, keyPair: Ed25519KeyPair): Promise<void> {
|
|
574
|
+
try {
|
|
424
575
|
if (!(await this.ensureMembership(db, keyPair))) return;
|
|
425
576
|
// NOTE: the redemption (`Member` + `ConsumedInvite`) and the binding below are two
|
|
426
577
|
// SEPARATE commits — measured at 27 and 18 `/cluster` streams on 2026-09-17.
|
|
@@ -433,10 +584,8 @@ export class StrandMembershipReconciler {
|
|
|
433
584
|
// cost ever shows up in a measurement, try the single transaction and measure against the
|
|
434
585
|
// 27 + 18 baseline.
|
|
435
586
|
await this.ensureBinding(db, keyPair);
|
|
436
|
-
} catch (error) {
|
|
437
|
-
log('[%s] reconcile pass failed — retrying next tick: %o', this.deps.label, error);
|
|
438
587
|
} finally {
|
|
439
|
-
this.
|
|
588
|
+
this.noteAttemptOutcome();
|
|
440
589
|
}
|
|
441
590
|
}
|
|
442
591
|
|
|
@@ -503,59 +652,84 @@ export class StrandMembershipReconciler {
|
|
|
503
652
|
memberKey: keyPair.publicKeyB64,
|
|
504
653
|
}, OWN_TRANSACTION);
|
|
505
654
|
} catch (error) {
|
|
506
|
-
this.handleConsumeFailure(error);
|
|
655
|
+
this.handleConsumeFailure(error, invite);
|
|
507
656
|
return false;
|
|
508
657
|
}
|
|
509
|
-
this.deps.pendingInvite?.clear();
|
|
658
|
+
this.deps.pendingInvite?.clear(invite);
|
|
510
659
|
log('[%s] redeemed the staged membership invitation — Member row seated under this party\'s key', this.deps.label);
|
|
511
660
|
return true;
|
|
512
661
|
}
|
|
513
662
|
|
|
514
663
|
/**
|
|
515
|
-
* The already-member arm: spend a still-staged invitation so nobody else can.
|
|
516
|
-
*
|
|
517
|
-
*
|
|
518
|
-
* present
|
|
519
|
-
*
|
|
520
|
-
*
|
|
521
|
-
*
|
|
664
|
+
* The already-member arm: spend a still-staged invitation so nobody else can. The stage
|
|
665
|
+
* is cleared once the burn lands or the invitation is known dead (already spent,
|
|
666
|
+
* cancelled, expired, or the strand sealed around a party that is still a member) — with
|
|
667
|
+
* the member row present a dead credential has no further local use, and keeping it
|
|
668
|
+
* staged would leave `getPendingMembershipInvite` lying. A refused burn keeps it staged:
|
|
669
|
+
* a busy refusal tried nothing, and a write the cohort refused is the removed-party shape
|
|
670
|
+
* ("Stops, and re-arming" in the module doc), where the local member row is stale and the
|
|
671
|
+
* credential is the one thing a later manager admission lets this loop settle.
|
|
522
672
|
*
|
|
523
|
-
* NOTE:
|
|
524
|
-
*
|
|
525
|
-
*
|
|
526
|
-
*
|
|
527
|
-
*
|
|
528
|
-
*
|
|
529
|
-
* `feat-strand-member-allowlist-admission` lands.
|
|
673
|
+
* NOTE: this REPLACES an earlier accepted tradeoff that never retried a burn which failed
|
|
674
|
+
* for a transient reason and dropped the invitation instead. That decision was made when
|
|
675
|
+
* this arm could only ever see an invitation staged before the first join finished, so a
|
|
676
|
+
* dropped credential cost nothing but a spendable bearer token. {@link rearm} makes the
|
|
677
|
+
* arm reachable with a freshly issued invitation the app wants redeemed, and dropping
|
|
678
|
+
* that one on the first refused write is the silent failure this loop exists to avoid.
|
|
530
679
|
*/
|
|
531
680
|
private async burnLeftoverInvite(db: Database, keyPair: Ed25519KeyPair): Promise<void> {
|
|
532
|
-
const
|
|
533
|
-
|
|
534
|
-
if (!pending || !invite) return;
|
|
681
|
+
const invite = this.deps.pendingInvite?.get();
|
|
682
|
+
if (!invite) return;
|
|
535
683
|
try {
|
|
536
684
|
await burnInvite(db, {
|
|
537
685
|
inviteKey: invite.inviteKey,
|
|
538
686
|
invitePrivateKey: invite.invitePrivateKey,
|
|
539
687
|
memberKey: keyPair.publicKeyB64,
|
|
540
688
|
}, OWN_TRANSACTION);
|
|
541
|
-
log('[%s] burned the leftover invitation (member row already present)', this.deps.label);
|
|
542
689
|
} catch (error) {
|
|
543
|
-
|
|
690
|
+
this.handleBurnFailure(error, invite);
|
|
691
|
+
return;
|
|
692
|
+
}
|
|
693
|
+
this.deps.pendingInvite?.clear(invite);
|
|
694
|
+
log('[%s] burned the leftover invitation (member row already present)', this.deps.label);
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
/**
|
|
698
|
+
* Act on a `burnInvite` rejection as {@link classifyConsumeFailure} routes it. Unlike a
|
|
699
|
+
* redemption, a sealed strand is not terminal here — the member row is present, so this
|
|
700
|
+
* party is a member of the sealed strand and only its leftover credential is dead — and a
|
|
701
|
+
* half-committed burn (one collection, so a report of it means the commit's durability
|
|
702
|
+
* was in doubt) is retried: the next pass either lands it or fails on the row's primary
|
|
703
|
+
* key, which drops it as dead.
|
|
704
|
+
*/
|
|
705
|
+
private handleBurnFailure(error: unknown, invite: StrandMembershipInvite): void {
|
|
706
|
+
const failure = classifyConsumeFailure(error);
|
|
707
|
+
switch (failure.kind) {
|
|
708
|
+
case 'busy':
|
|
544
709
|
log('[%s] burning the leftover invitation deferred — the app has a transaction open; it stays staged '
|
|
545
710
|
+ 'for the next pass', this.deps.label);
|
|
546
711
|
return;
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
712
|
+
case 'sealed':
|
|
713
|
+
case 'dead-invite':
|
|
714
|
+
log('[%s] the leftover invitation is dead (already spent, cancelled, expired, or the strand sealed) — '
|
|
715
|
+
+ 'dropping it: %s', this.deps.label, errorMessage(error));
|
|
716
|
+
this.deps.pendingInvite?.clear(invite);
|
|
717
|
+
return;
|
|
718
|
+
case 'half-committed':
|
|
719
|
+
case 'retry':
|
|
720
|
+
log('[%s] burning the leftover invitation was refused — it stays staged and is retried next tick '
|
|
721
|
+
+ '(the cohort refusing this machine, or a transient write failure): %s', this.deps.label, errorMessage(error));
|
|
722
|
+
return;
|
|
550
723
|
}
|
|
551
|
-
pending.clear();
|
|
552
724
|
}
|
|
553
725
|
|
|
554
726
|
/**
|
|
555
727
|
* Step 3 of the pass: write this machine's own `MemberPeer` binding (insert-if-absent)
|
|
556
|
-
* and,
|
|
557
|
-
* (a quiesce racing the pass) defers to the next tick; a
|
|
558
|
-
* included — is contained by the pass's outer catch and
|
|
728
|
+
* and, once it is in place with nothing staged, latch the done state and stop the loop.
|
|
729
|
+
* A missing transport peer id (a quiesce racing the pass) defers to the next tick; a
|
|
730
|
+
* write failure — a busy refusal included — is contained by the pass's outer catch and
|
|
731
|
+
* retried. A binding written while an invitation is still staged (its burn keeps being
|
|
732
|
+
* refused) is not done: the loop stays live so a later pass can settle the credential.
|
|
559
733
|
*/
|
|
560
734
|
private async ensureBinding(db: Database, keyPair: Ed25519KeyPair): Promise<void> {
|
|
561
735
|
const peerId = this.deps.getOwnPeerId();
|
|
@@ -564,8 +738,12 @@ export class StrandMembershipReconciler {
|
|
|
564
738
|
return;
|
|
565
739
|
}
|
|
566
740
|
await registerMemberPeer(db, { memberKeyPair: keyPair, peerId }, OWN_TRANSACTION);
|
|
741
|
+
if (this.deps.pendingInvite?.get()) {
|
|
742
|
+
log('[%s] own MemberPeer binding is in place but an invitation is still staged — not done yet', this.deps.label);
|
|
743
|
+
return;
|
|
744
|
+
}
|
|
567
745
|
this.doneFlag = true;
|
|
568
|
-
this.finish('member row and own MemberPeer binding are both in place');
|
|
746
|
+
this.finish('member row and own MemberPeer binding are both in place', false);
|
|
569
747
|
}
|
|
570
748
|
|
|
571
749
|
/**
|
|
@@ -573,7 +751,7 @@ export class StrandMembershipReconciler {
|
|
|
573
751
|
* the invitation and retry; half-committed → warn and drop; sealed → terminal; dead
|
|
574
752
|
* invitation → drop; anything else → retry.
|
|
575
753
|
*/
|
|
576
|
-
private handleConsumeFailure(error: unknown): void {
|
|
754
|
+
private handleConsumeFailure(error: unknown, invite: StrandMembershipInvite): void {
|
|
577
755
|
const failure = classifyConsumeFailure(error);
|
|
578
756
|
switch (failure.kind) {
|
|
579
757
|
case 'busy':
|
|
@@ -581,12 +759,12 @@ export class StrandMembershipReconciler {
|
|
|
581
759
|
+ 'database; retrying next tick', this.deps.label);
|
|
582
760
|
return;
|
|
583
761
|
case 'half-committed':
|
|
584
|
-
this.reportHalfCommittedJoin(failure, error);
|
|
762
|
+
this.reportHalfCommittedJoin(failure, error, invite);
|
|
585
763
|
return;
|
|
586
764
|
case 'sealed':
|
|
587
765
|
// The staged credential is dead with the seal; drop it so the cache stays honest.
|
|
588
|
-
this.deps.pendingInvite?.clear();
|
|
589
|
-
this.finish('the strand is sealed — nobody can ever be admitted, so the staged invitation is dead');
|
|
766
|
+
this.deps.pendingInvite?.clear(invite);
|
|
767
|
+
this.finish('the strand is sealed — nobody can ever be admitted, so the staged invitation is dead', true);
|
|
590
768
|
return;
|
|
591
769
|
case 'dead-invite':
|
|
592
770
|
// NOTE: accepted tradeoff — dropping a dead invitation does not mark the pass idle,
|
|
@@ -596,15 +774,67 @@ export class StrandMembershipReconciler {
|
|
|
596
774
|
// source of dead invitations ever repeats per pass.
|
|
597
775
|
log('[%s] the staged invitation is dead (expired, cancelled, or consumed elsewhere) — dropping it; '
|
|
598
776
|
+ 'a fresh formation stages a new one: %s', this.deps.label, errorMessage(error));
|
|
599
|
-
this.deps.pendingInvite?.clear();
|
|
777
|
+
this.deps.pendingInvite?.clear(invite);
|
|
600
778
|
return;
|
|
601
779
|
case 'retry':
|
|
602
|
-
log('[%s] consumeInvite failed — retrying next tick (Invite row not yet replicated here,
|
|
603
|
-
+ 'write failure): %s', this.deps.label, errorMessage(error));
|
|
780
|
+
log('[%s] consumeInvite failed — retrying next tick (Invite row not yet replicated here, the cohort '
|
|
781
|
+
+ 'refusing this machine, or a transient write failure): %s', this.deps.label, errorMessage(error));
|
|
604
782
|
return;
|
|
605
783
|
}
|
|
606
784
|
}
|
|
607
785
|
|
|
786
|
+
/**
|
|
787
|
+
* Count an attempted pass that left the invitation staged (whatever stopped it — a
|
|
788
|
+
* refused or busy write, or a read that threw on a cut-off machine); one that settled
|
|
789
|
+
* it, or finished, ends the streak. At {@link UNFINISHED_PASSES_BEFORE_ESCALATION}
|
|
790
|
+
* report the PROBABLE blocked re-join.
|
|
791
|
+
*/
|
|
792
|
+
private noteAttemptOutcome(): void {
|
|
793
|
+
if (this.doneFlag || !this.deps.pendingInvite?.get()) {
|
|
794
|
+
this.unfinishedPasses = 0;
|
|
795
|
+
return;
|
|
796
|
+
}
|
|
797
|
+
this.unfinishedPasses += 1;
|
|
798
|
+
if (this.unfinishedPasses >= UNFINISHED_PASSES_BEFORE_ESCALATION) {
|
|
799
|
+
this.reportRejoinBlocked(
|
|
800
|
+
`${this.unfinishedPasses} membership reconcile passes have left the staged membership invitation `
|
|
801
|
+
+ 'unsettled. Either this party was removed from the strand and the remaining members are refusing its '
|
|
802
|
+
+ 'machines, or the invitation\'s Strand.Invite row has not replicated here yet.'
|
|
803
|
+
);
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
/**
|
|
808
|
+
* Step 1's CONFIRMED trigger: self-revoked with an invitation staged. Reports (once per
|
|
809
|
+
* re-arm cycle) and answers `true` so the pass carries on — the redemption is attempted
|
|
810
|
+
* and refused like any other write, and a manager's re-admission lets a later pass finish.
|
|
811
|
+
* `false` (nothing staged) means step 1 stops the loop as before.
|
|
812
|
+
*/
|
|
813
|
+
private stagedInviteBlockedBySelfRevocation(): boolean {
|
|
814
|
+
if (!this.deps.pendingInvite?.get()) return false;
|
|
815
|
+
this.reportRejoinBlocked(
|
|
816
|
+
'this party is revoked from the strand and holds a staged membership invitation it cannot spend: the '
|
|
817
|
+
+ 'remaining members refuse this machine\'s strand writes, so the redemption cannot land.'
|
|
818
|
+
);
|
|
819
|
+
return true;
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
/**
|
|
823
|
+
* ONE `console.warn` and one callback per re-arm cycle, whichever trigger fires first —
|
|
824
|
+
* see "Reporting a blocked re-join" in the module doc. `cause` is the trigger's own
|
|
825
|
+
* sentence; the remedy and the loop's posture are the same for both.
|
|
826
|
+
*/
|
|
827
|
+
private reportRejoinBlocked(cause: string): void {
|
|
828
|
+
if (this.rejoinBlockedReported) return;
|
|
829
|
+
this.rejoinBlockedReported = true;
|
|
830
|
+
console.warn(
|
|
831
|
+
`[sereus] strand ${this.deps.label}: ${cause} A fresh invitation cannot re-admit a removed party by itself — `
|
|
832
|
+
+ 'a remaining manager must admit this party\'s member key directly (addMemberByManager). The membership '
|
|
833
|
+
+ 'loop keeps retrying quietly and completes the join once a Member row for this party appears.'
|
|
834
|
+
);
|
|
835
|
+
this.deps.onRejoinBlocked?.();
|
|
836
|
+
}
|
|
837
|
+
|
|
608
838
|
/**
|
|
609
839
|
* A redemption only part of which was saved (see "Half-committed join" in the module doc):
|
|
610
840
|
* ONE visible warning naming both halves, and the staged invitation dropped. The pass is not
|
|
@@ -621,7 +851,7 @@ export class StrandMembershipReconciler {
|
|
|
621
851
|
* path. Revisit if a `Member`-saved half-commit is ever observed, or when
|
|
622
852
|
* `blocked/strand-half-committed-join-recovery` is decided.
|
|
623
853
|
*/
|
|
624
|
-
private reportHalfCommittedJoin(failure: HalfCommittedJoin, error: unknown): void {
|
|
854
|
+
private reportHalfCommittedJoin(failure: HalfCommittedJoin, error: unknown, invite: StrandMembershipInvite): void {
|
|
625
855
|
console.warn(
|
|
626
856
|
`[sereus] strand ${this.deps.label}: redeeming the staged membership invitation was only partly saved. `
|
|
627
857
|
+ `Saved: [${failure.saved.join(', ')}]. Not saved: [${failure.unsaved.join(', ')}]. `
|
|
@@ -631,7 +861,7 @@ export class StrandMembershipReconciler {
|
|
|
631
861
|
+ 'The membership loop keeps checking and completes the join once a Member row for this party appears.'
|
|
632
862
|
);
|
|
633
863
|
log('[%s] half-committed redemption, full error: %o', this.deps.label, error);
|
|
634
|
-
this.deps.pendingInvite?.clear();
|
|
864
|
+
this.deps.pendingInvite?.clear(invite);
|
|
635
865
|
// The idle passes that follow are the wait this warning already explained; the escalation's
|
|
636
866
|
// "waiting on the founder rows to replicate" would be a second, misleading warning.
|
|
637
867
|
this.idleEscalated = true;
|
|
@@ -643,7 +873,7 @@ export class StrandMembershipReconciler {
|
|
|
643
873
|
try {
|
|
644
874
|
this.keyPair = strandMemberKeyPair(this.deps.partyMemberPrivateKey);
|
|
645
875
|
} catch (error) {
|
|
646
|
-
this.finish(`the party membership key does not decode — nothing can be signed (${errorMessage(error)})
|
|
876
|
+
this.finish(`the party membership key does not decode — nothing can be signed (${errorMessage(error)})`, true);
|
|
647
877
|
return undefined;
|
|
648
878
|
}
|
|
649
879
|
return this.keyPair;
|
|
@@ -672,9 +902,21 @@ export class StrandMembershipReconciler {
|
|
|
672
902
|
}
|
|
673
903
|
}
|
|
674
904
|
|
|
675
|
-
/**
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
905
|
+
/**
|
|
906
|
+
* Stop with a reason — the terminal and done paths' shared exit. `permanent` says whether
|
|
907
|
+
* a fresh invitation may reopen the loop ("Stops, and re-arming" in the module doc).
|
|
908
|
+
*/
|
|
909
|
+
private finish(reason: string, permanent: boolean): void {
|
|
910
|
+
this.halt(`membership reconciliation stopping: ${reason}`, permanent);
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
/** Latch the stop and disarm the timer; a permanent latch sticks even on an already-stopped loop. */
|
|
914
|
+
private halt(reason: string, permanent: boolean): void {
|
|
915
|
+
if (permanent) this.permanentlyStopped = true;
|
|
916
|
+
if (this.stoppedFlag) return;
|
|
917
|
+
this.stoppedFlag = true;
|
|
918
|
+
this.clearTimer();
|
|
919
|
+
log('[%s] membership reconciler stopped%s: %s%s', this.deps.label, this.doneFlag ? ' (done)' : '', reason,
|
|
920
|
+
permanent ? '' : ' — a fresh invitation re-arms it');
|
|
679
921
|
}
|
|
680
922
|
}
|