@serfab/cadre-core 1.4.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/dist/cadre-node.d.ts +38 -15
  2. package/dist/cadre-node.js +77 -29
  3. package/dist/cadre-node.js.map +1 -1
  4. package/dist/control-database.d.ts +24 -10
  5. package/dist/control-database.js +48 -16
  6. package/dist/control-database.js.map +1 -1
  7. package/dist/control-read-retry.d.ts +12 -0
  8. package/dist/control-read-retry.js +12 -0
  9. package/dist/control-read-retry.js.map +1 -1
  10. package/dist/control-write-retry.d.ts +5 -4
  11. package/dist/control-write-retry.js +24 -7
  12. package/dist/control-write-retry.js.map +1 -1
  13. package/dist/index.d.ts +3 -2
  14. package/dist/index.js +5 -2
  15. package/dist/index.js.map +1 -1
  16. package/dist/link-budget.d.ts +182 -0
  17. package/dist/link-budget.js +195 -0
  18. package/dist/link-budget.js.map +1 -0
  19. package/dist/peer-dial.d.ts +36 -17
  20. package/dist/peer-dial.js +37 -17
  21. package/dist/peer-dial.js.map +1 -1
  22. package/dist/peer-join-backfill.d.ts +58 -4
  23. package/dist/peer-join-backfill.js +165 -20
  24. package/dist/peer-join-backfill.js.map +1 -1
  25. package/dist/relay-reservation.d.ts +59 -11
  26. package/dist/relay-reservation.js +169 -43
  27. package/dist/relay-reservation.js.map +1 -1
  28. package/dist/seed-bootstrap.js +2 -1
  29. package/dist/seed-bootstrap.js.map +1 -1
  30. package/dist/strand-first-sync-gate.d.ts +43 -5
  31. package/dist/strand-first-sync-gate.js +55 -7
  32. package/dist/strand-first-sync-gate.js.map +1 -1
  33. package/dist/strand-instance-manager.d.ts +27 -2
  34. package/dist/strand-instance-manager.js +56 -13
  35. package/dist/strand-instance-manager.js.map +1 -1
  36. package/dist/strand-membership-reconciler.d.ts +200 -45
  37. package/dist/strand-membership-reconciler.js +302 -86
  38. package/dist/strand-membership-reconciler.js.map +1 -1
  39. package/dist/strand-membership-writer.d.ts +3 -2
  40. package/dist/strand-membership-writer.js +67 -90
  41. package/dist/strand-membership-writer.js.map +1 -1
  42. package/dist/strand-network-config.d.ts +2 -2
  43. package/dist/strand-network-config.js +2 -2
  44. package/dist/types.d.ts +96 -9
  45. package/dist/types.js +14 -4
  46. package/dist/types.js.map +1 -1
  47. package/package.json +8 -8
  48. package/src/cadre-node.ts +7445 -7397
  49. package/src/control-database.ts +48 -16
  50. package/src/control-read-retry.ts +214 -202
  51. package/src/control-write-retry.ts +24 -7
  52. package/src/index.ts +581 -563
  53. package/src/link-budget.ts +217 -0
  54. package/src/peer-dial.ts +39 -17
  55. package/src/peer-join-backfill.ts +192 -23
  56. package/src/relay-reservation.ts +1119 -951
  57. package/src/seed-bootstrap.ts +2 -1
  58. package/src/strand-first-sync-gate.ts +55 -7
  59. package/src/strand-instance-manager.ts +1516 -1458
  60. package/src/strand-membership-reconciler.ts +334 -92
  61. package/src/strand-membership-writer.ts +1962 -1987
  62. package/src/strand-network-config.ts +2 -2
  63. package/src/types.ts +95 -8
@@ -26,39 +26,93 @@
26
26
  * that finds no member row AND no staged invitation is not an unfinished join but a wait
27
27
  * for someone to admit this party — nothing this machine can do faster — so it re-arms on
28
28
  * the flat poll interval and resets the ladder. The loop stops on the done state.
29
- * - **Done state**: the party's member row is visible locally AND this machine's own
30
- * binding is in place — the loop stops for good. A LATER revocation is the enforcer's
31
- * business, not this loop's; a resume rebuilds the reconciler and re-verifies from
32
- * scratch (every write is idempotent, so re-running is free).
29
+ * - **Done state**: the party's member row is visible locally, this machine's own binding
30
+ * is in place, AND no invitation is still staged — the loop stops. A LATER revocation is
31
+ * the enforcer's business, not this loop's; a resume rebuilds the reconciler and
32
+ * re-verifies from scratch (every write is idempotent, so re-running is free), and a
33
+ * fresh invitation staged on a done loop re-arms it in place ({@link rearm}).
33
34
  *
34
35
  * ## One pass
35
36
  *
36
- * 1. **Self-revocation check.** If the enforcer flags this node's own party as revoked,
37
- * stop rather than fight — re-admission arrives (if ever) via a fresh formation.
37
+ * 1. **Self-revocation check.** If the enforcer flags this node's own party as revoked and
38
+ * no invitation is staged, stop rather than fight — re-admission arrives (if ever) via
39
+ * a fresh formation, which re-arms the loop. With an invitation staged the loop reports
40
+ * the dead end (below) and carries on: the redemption is attempted like any other, and a
41
+ * manager's re-admission lets a later pass finish it.
38
42
  * 2. **Ensure membership.** Member row visible → done with this step; if a staged
39
43
  * invitation is still unspent, BURN it ({@link burnInvite} — the `ConsumedInvite` row
40
- * alone) so the bearer credential cannot be spent by anyone else, ignoring burn
41
- * failures beyond a log. Member row absent + staged invitation → {@link consumeInvite}
44
+ * alone) so the bearer credential cannot be spent by anyone else. A burn refused by the
45
+ * cohort keeps the invitation staged for the next pass (it is the same write gate a
46
+ * redemption faces, and the local member row may be stale — see "Re-arming"); a dead
47
+ * invitation is dropped. Member row absent + staged invitation → {@link consumeInvite}
42
48
  * under the party's own key, retried across passes until the `Invite` row has
43
49
  * replicated here and the write commits. Absent with no invitation (founder rows not
44
50
  * yet synced, or a machine whose party never joined) → keep waiting, escalating to one
45
51
  * visible warning after {@link IDLE_PASSES_BEFORE_ESCALATION} passes, never throwing.
46
52
  * 3. **Ensure the binding.** {@link registerMemberPeer} (insert-if-absent, restart-safe)
47
53
  * with this node's own strand transport peer id — only after step 2 sees the member
48
- * row locally (its deferred `MemberExists` reads the live local table).
54
+ * row locally (its deferred `MemberExists` reads the live local table). Done only once
55
+ * the binding is in place AND nothing is staged; a binding written while a burn keeps
56
+ * being refused leaves the loop live to settle the invitation later.
49
57
  *
50
- * ## Terminal states (stop without done)
58
+ * ## Stops, and re-arming
59
+ *
60
+ * Two kinds of stop. A RE-ARMABLE stop is one a fresh invitation should reopen:
61
+ *
62
+ * - **Done** (above).
63
+ * - **Self-revoked with nothing staged** (step 1 above).
64
+ *
65
+ * A PERMANENT stop is one a fresh invitation changes nothing about — {@link rearm} is a
66
+ * no-op after it:
51
67
  *
52
68
  * - **Sealed strand**: `consumeInvite` rejected by `ConsumedInvite.NotSealed` — nobody
53
69
  * can ever be admitted, so retrying forever is noise. Terminal-with-log.
54
- * - **Self-revoked** (step 1 above).
55
70
  * - **Undecodable party key**: nothing can be signed; loud log, stop.
71
+ * - **The public {@link stop}**: `StrandInstanceManager.releaseRuntime` and
72
+ * `clearOwnMemberPeerBinding` both call it and mean it — the latter stops the loop and
73
+ * awaits {@link settle} precisely so no pass can re-register the binding it is about to
74
+ * delete, and a re-arm that restarted the loop would undo that.
75
+ *
76
+ * `CadreNode.adoptFormationMembershipInvite` stages every fresh invitation and notifies
77
+ * the instance manager, which calls {@link rearm}: a done loop reopens and runs one pass
78
+ * at once; a running loop is merely kicked. This is what lets a party that was REMOVED
79
+ * from the strand, and then handed a new invitation, actually attempt it — the loop
80
+ * finished during its first join, long before the removal.
81
+ *
82
+ * On a removed party the local replica is usually stale: the cohort cut this machine off
83
+ * at about the moment the removal was written, so `isStrandMember` still answers true and
84
+ * a re-armed pass lands in the already-member arm. That is why the burn arm keeps a
85
+ * refused invitation staged and why the done state requires nothing staged — otherwise
86
+ * the fresh credential would be dropped on the first refused burn and the loop would latch
87
+ * done again, with the only trace a debug line.
56
88
  *
57
89
  * A DEAD invitation that is not terminal for the loop — expired, cancelled, or already
58
90
  * consumed by someone else (`ConsumedInvite`'s primary key) — is dropped with a log and
59
91
  * the loop keeps idling: a fresh formation stages a new invitation, and a manager-side
60
92
  * admission (`addMemberByManager`) seats the member row without one.
61
93
  *
94
+ * ## Reporting a blocked re-join
95
+ *
96
+ * Redeeming (or burning) writes into the strand, and the machines that would carry that
97
+ * write are exactly the ones the remaining members refuse once this party is removed. So
98
+ * the attempt is expected to fail, and the failure is REPORTED rather than left to the
99
+ * ladder: one `console.warn` plus the {@link StrandMembershipReconcilerDeps.onRejoinBlocked}
100
+ * callback (`CadreNode`'s `strand:rejoin-blocked` event), at most once per re-arm cycle.
101
+ * Two triggers:
102
+ *
103
+ * - **Confirmed** — a pass finds this party self-revoked AND an invitation staged. The node
104
+ * knows it was removed and is holding a credential it cannot spend.
105
+ * - **Probable** — {@link UNFINISHED_PASSES_BEFORE_ESCALATION} consecutive attempted passes
106
+ * left the invitation staged. The case that matters most in the field: a removed party
107
+ * that never learned it was removed (the removal did not replicate here before the cut).
108
+ * It is a suspicion, not a verdict — the invitation's `Invite` row may simply not have
109
+ * replicated here yet — and the warning names both causes.
110
+ *
111
+ * The remedy in either case is a remaining manager admitting this party's member key
112
+ * directly (`addMemberByManager`): that lifts the refusal, replication resumes, and the
113
+ * still-running loop sees the member row, burns the leftover credential and finishes by
114
+ * itself.
115
+ *
62
116
  * ## Sharing the database with the app
63
117
  *
64
118
  * The strand `Database` this loop writes to is the one the app holds. Every write here
@@ -111,6 +165,17 @@ const log = debug('sereus:cadre:strand-membership-reconciler');
111
165
  */
112
166
  export const IDLE_PASSES_BEFORE_ESCALATION = 10;
113
167
 
168
+ /**
169
+ * Consecutive passes that ran against a live database and ended with the staged
170
+ * invitation still unsettled — whatever refused it — before the loop reports a probable
171
+ * blocked re-join (see "Reporting a blocked re-join" in the module doc). The same order
172
+ * as {@link IDLE_PASSES_BEFORE_ESCALATION}: about three minutes at the default cadence
173
+ * (the ladder's five doubling rungs, then five passes at the 30 s cap). Counted by outcome
174
+ * rather than by classified failure because a cut-off machine can fail before any write is
175
+ * classified: its membership READ may already throw.
176
+ */
177
+ export const UNFINISHED_PASSES_BEFORE_ESCALATION = 10;
178
+
114
179
  /**
115
180
  * First retry delay, ms, for a pass that left the join UNFINISHED — a `consumeInvite`
116
181
  * whose `Strand.Invite` row has not replicated to this machine yet, a cohort briefly
@@ -159,25 +224,30 @@ export interface HalfCommittedJoin {
159
224
  unsaved: readonly string[];
160
225
  }
161
226
 
162
- /** Where {@link classifyConsumeFailure} routes a failed `consumeInvite`. */
227
+ /** Where {@link classifyConsumeFailure} routes a failed `consumeInvite` or `burnInvite`. */
163
228
  export type ConsumeFailure =
164
229
  | HalfCommittedJoin
165
230
  /** The app had a transaction open, so nothing was tried — keep the invitation, retry. */
166
231
  | { kind: 'busy' }
167
- /** The strand is sealed — terminal. */
232
+ /** The strand is sealed — terminal for a non-member; a member's leftover invitation is merely dead. */
168
233
  | { kind: 'sealed' }
169
234
  /** Expired, cancelled, or consumed by someone else — drop the invitation, keep waiting. */
170
235
  | { kind: 'dead-invite' }
171
- /** Anything else — most often the `Invite` row has not replicated here yet — retry. */
236
+ /**
237
+ * Anything else — the `Invite` row has not replicated here yet, or the cohort refused the
238
+ * write (a removed party's machines are denied) — keep the invitation, retry.
239
+ */
172
240
  | { kind: 'retry' };
173
241
 
174
242
  /**
175
- * Route a `consumeInvite` rejection. Typed checks run before any text check: a busy refusal
176
- * (another transaction was open, so nothing ran) and a half-committed join are recognised by
177
- * their error's type anywhere in the `cause` chain. The busy refusal carries Quereus's own
178
- * refusal as its cause, and a half-committed join's message embeds collection names and the
179
- * underlying failure's text, either of which a text matcher can misread. The sealed and
180
- * dead-invitation checks then read the top-level message.
243
+ * Route a `consumeInvite` or `burnInvite` rejection. Both write the `Strand.ConsumedInvite`
244
+ * row under the same `NotExpired` / `NotCancelled` / `NotSealed` / primary-key constraints
245
+ * (a burn writes that row alone), so one classifier serves both. Typed checks run before any
246
+ * text check: a busy refusal (another transaction was open, so nothing ran) and a
247
+ * half-committed join are recognised by their error's type anywhere in the `cause` chain.
248
+ * The busy refusal carries Quereus's own refusal as its cause, and a half-committed join's
249
+ * message embeds collection names and the underlying failure's text, either of which a text
250
+ * matcher can misread. The sealed and dead-invitation checks then read the top-level message.
181
251
  *
182
252
  * A second loaded copy of `@optimystic/db-core` (or of the plugin) would fail `instanceof`; the
183
253
  * half-commit then falls through the anchored texts to `retry`, and once the saved `ConsumedInvite`
@@ -234,8 +304,12 @@ function errorMessage(error: unknown): string {
234
304
  export interface PendingMembershipInviteSource {
235
305
  /** The staged invitation, or `undefined` when none is pending for this strand. */
236
306
  get(): StrandMembershipInvite | undefined;
237
- /** Drop the staged invitation (spent, burned, or dead). */
238
- clear(): void;
307
+ /**
308
+ * Drop `settled` (spent, burned, or dead) — only if it is still the staged one. A
309
+ * re-formation can replace the entry between a pass's {@link get} and this call, and
310
+ * the fresh invitation it staged must survive the older one being settled.
311
+ */
312
+ clear(settled: StrandMembershipInvite): void;
239
313
  }
240
314
 
241
315
  export interface StrandMembershipReconcilerDeps {
@@ -262,10 +336,17 @@ export interface StrandMembershipReconcilerDeps {
262
336
  pendingInvite?: PendingMembershipInviteSource;
263
337
  /**
264
338
  * Whether the revocation enforcer currently flags THIS node's own party as revoked.
265
- * When it does, the loop stops rather than fight the enforcer — see the module doc.
266
- * Absent (enforcement disarmed) means "not known revoked".
339
+ * When it does and nothing is staged, the loop stops rather than fight the enforcer;
340
+ * with an invitation staged it reports the blocked re-join and keeps trying — see the
341
+ * module doc. Absent (enforcement disarmed) means "not known revoked".
267
342
  */
268
343
  isSelfRevoked?: () => boolean;
344
+ /**
345
+ * Called when the loop reports a blocked re-join (see "Reporting a blocked re-join" in
346
+ * the module doc) — at most once per re-arm cycle, after the `console.warn`. `CadreNode`
347
+ * wires it to its `strand:rejoin-blocked` event.
348
+ */
349
+ onRejoinBlocked?: () => void;
269
350
  /**
270
351
  * Timer seam for the retry ladder; omit for real (unref'd) timeouts. Timeouts rather
271
352
  * than the enforcer's repeating interval because the delay changes per pass and the next
@@ -307,11 +388,21 @@ export class StrandMembershipReconciler {
307
388
  private timer: unknown;
308
389
  private started = false;
309
390
  private stoppedFlag = false;
391
+ /**
392
+ * Whether the last stop was one {@link rearm} may not undo — see "Stops, and re-arming"
393
+ * in the module doc. Latched by the public {@link stop} and the permanent terminal
394
+ * states, never cleared.
395
+ */
396
+ private permanentlyStopped = false;
310
397
  private doneFlag = false;
311
398
  /** Lazily decoded party keypair — see {@link StrandMembershipReconcilerDeps.partyMemberPrivateKey}. */
312
399
  private keyPair: Ed25519KeyPair | undefined;
313
400
  private idlePasses = 0;
314
401
  private idleEscalated = false;
402
+ /** Consecutive attempted passes that left the invitation staged — see {@link UNFINISHED_PASSES_BEFORE_ESCALATION}. */
403
+ private unfinishedPasses = 0;
404
+ /** Whether the blocked re-join was already reported this re-arm cycle (one report, either trigger). */
405
+ private rejoinBlockedReported = false;
315
406
  /**
316
407
  * Whether the pass that just ran found nothing to act on — no member row and no staged
317
408
  * invitation. That is a wait to be admitted, not an unfinished join, so it re-arms on the
@@ -335,12 +426,15 @@ export class StrandMembershipReconciler {
335
426
  this.pollIntervalMs = config?.pollIntervalMs ?? DEFAULT_REVOCATION_POLL_INTERVAL_MS;
336
427
  }
337
428
 
338
- /** True once member row + own binding were both confirmed and the loop stopped. */
429
+ /**
430
+ * True once member row + own binding were both confirmed with nothing staged and the
431
+ * loop stopped. Cleared again by {@link rearm}.
432
+ */
339
433
  get done(): boolean {
340
434
  return this.doneFlag;
341
435
  }
342
436
 
343
- /** True once the loop has stopped — done, terminal, or externally stopped. */
437
+ /** True while the loop is stopped — done, terminal, or externally stopped. */
344
438
  get stopped(): boolean {
345
439
  return this.stoppedFlag;
346
440
  }
@@ -369,12 +463,24 @@ export class StrandMembershipReconciler {
369
463
  await this.tail;
370
464
  }
371
465
 
372
- /** Disarm the retry timer; a pass already in flight completes but writes idempotently. */
466
+ /**
467
+ * Disarm the retry timer, for good: a later {@link rearm} is refused. A pass already in
468
+ * flight completes but writes idempotently — {@link settle} is what waits it out.
469
+ */
373
470
  stop(): void {
374
- if (this.stoppedFlag) return;
375
- this.stoppedFlag = true;
376
- this.clearTimer();
377
- log('[%s] membership reconciler stopped%s', this.deps.label, this.doneFlag ? ' (done)' : '');
471
+ this.halt('stopped by the runtime', true);
472
+ }
473
+
474
+ /**
475
+ * A fresh invitation was staged: reopen a loop that stopped on a re-armable state (done,
476
+ * or self-revoked with nothing staged) and run one pass at once; merely kick a loop that
477
+ * is still running; do nothing after a permanent stop. Never rejects, and serialized
478
+ * like {@link reconcile}: the reopen happens when the queued pass STARTS, after any pass
479
+ * in flight has settled, so a `done` that pass latches cannot swallow the re-arm — and a
480
+ * public {@link stop} that lands first still wins, because the queued pass re-checks it.
481
+ */
482
+ rearm(): Promise<void> {
483
+ return this.enqueue(() => this.reopen() ? this.doPass() : Promise.resolve());
378
484
  }
379
485
 
380
486
  /**
@@ -385,13 +491,18 @@ export class StrandMembershipReconciler {
385
491
  * published — REPLACES the pending timer rather than running alongside it.
386
492
  */
387
493
  reconcile(): Promise<void> {
494
+ return this.enqueue(() => this.doPass());
495
+ }
496
+
497
+ /** Chain `pass` after every pass already queued; the tail never rejects. */
498
+ private enqueue(pass: () => Promise<void>): Promise<void> {
388
499
  // The catch is what keeps the chain alive. `doPass` contains its own failures, but its
389
500
  // `finally` calls into the injected scheduler, and a throw from there would leave `tail`
390
501
  // REJECTED — after which every later `reconcile()` short-circuits on it and the join
391
- // stalls for good, silently. Swallowing it here makes the "never rejects" contract this
392
- // method, {@link settle} and the two `void`-ed call sites all rely on structural rather
393
- // than incidental.
394
- const run = this.tail.then(() => this.doPass()).catch((error) => {
502
+ // stalls for good, silently. Swallowing it here makes the "never rejects" contract
503
+ // {@link reconcile}, {@link rearm}, {@link settle} and the `void`-ed call sites all rely
504
+ // on structural rather than incidental.
505
+ const run = this.tail.then(pass).catch((error) => {
395
506
  log('[%s] reconcile pass threw outside its own handler — the chain continues: %o',
396
507
  this.deps.label, error);
397
508
  });
@@ -399,28 +510,68 @@ export class StrandMembershipReconciler {
399
510
  return run;
400
511
  }
401
512
 
513
+ /**
514
+ * The re-arm itself, run at the head of the queued pass: `false` refuses the pass (a
515
+ * permanent stop), a running loop is left as it is, and a re-armable stop is reopened
516
+ * with the idle and unfinished counters, both escalation latches and the retry ladder
517
+ * reset — a fresh invitation is a fresh cycle.
518
+ */
519
+ private reopen(): boolean {
520
+ if (this.permanentlyStopped) {
521
+ log('[%s] re-arm refused — the loop is stopped for good', this.deps.label);
522
+ return false;
523
+ }
524
+ if (!this.stoppedFlag) {
525
+ // NOTE: a running loop keeps its counters and its once-per-cycle report latch, so a
526
+ // party whose loop already reported and is then re-formed AGAIN gets no second report
527
+ // for the replacement invitation. If that second report is ever wanted, key the latch
528
+ // on the staged invitation's key rather than on the re-arm.
529
+ log('[%s] fresh invitation staged on a running loop — kicking a pass', this.deps.label);
530
+ return true;
531
+ }
532
+ this.stoppedFlag = false;
533
+ this.doneFlag = false;
534
+ this.idlePasses = 0;
535
+ this.idleEscalated = false;
536
+ this.unfinishedPasses = 0;
537
+ this.rejoinBlockedReported = false;
538
+ this.retryDelayMs = undefined;
539
+ log('[%s] membership reconciler re-armed by a fresh invitation', this.deps.label);
540
+ return true;
541
+ }
542
+
402
543
  /** One serialized pass; contains every failure (contract: never rejects). */
403
544
  private async doPass(): Promise<void> {
404
545
  if (this.stoppedFlag || this.doneFlag) return;
405
546
  this.lastPassIdle = false;
406
547
  try {
407
- if (this.deps.isSelfRevoked?.() === true) {
408
- // NOTE: "arrives via a fresh formation" is aspirational, not current behaviour.
409
- // A fresh formation stages a new invitation (`adoptFormationMembershipInvite`)
410
- // but does NOT re-arm this loop — `finish` latches `stoppedFlag` and `start()`
411
- // early-returns on it, so only a strand relaunch (quiesce → resume, or a process
412
- // restart) builds a reconciler that would redeem it. The same latch applies on the
413
- // `done` path below, which is the case a REMOVED party actually hits: it finished
414
- // its first join long before it was removed. Measured end to end in
415
- // `strand-party-removal-via-formation-e2e.integration.ts` (test 2), tracked as
416
- // `backlog/bug-removed-party-cannot-redeem-its-way-back`.
417
- this.finish('this party is revoked from the strand — re-admission arrives (if ever) via a fresh formation');
548
+ if (this.deps.isSelfRevoked?.() === true && !this.stagedInviteBlockedBySelfRevocation()) {
549
+ // NOTE: the enforcer's flag lags a re-admission by up to one of its polls. If the
550
+ // pass that settled the last staged invitation then failed to write the binding, this
551
+ // stop lands with the binding missing until the next resume rebuilds the loop. A
552
+ // missing binding only mis-credits diversity today; if it ever gates admission, make
553
+ // this stop also require the binding to be in place.
554
+ this.finish('this party is revoked from the strand — re-admission arrives (if ever) via a fresh formation', false);
418
555
  return;
419
556
  }
420
557
  const keyPair = this.resolveKeyPair();
421
558
  if (!keyPair) return; // undecodable key already stopped the loop
422
559
  const db = this.deps.getDatabase();
423
560
  if (!db) return; // no live database this instant (quiesce race) — next tick decides
561
+ await this.attempt(db, keyPair);
562
+ } catch (error) {
563
+ log('[%s] reconcile pass failed — retrying next tick: %o', this.deps.label, error);
564
+ } finally {
565
+ this.scheduleNext();
566
+ }
567
+ }
568
+
569
+ /**
570
+ * Steps 2 and 3 against a live database, with the outcome counted toward the probable
571
+ * blocked re-join report whether the attempt returned or threw.
572
+ */
573
+ private async attempt(db: Database, keyPair: Ed25519KeyPair): Promise<void> {
574
+ try {
424
575
  if (!(await this.ensureMembership(db, keyPair))) return;
425
576
  // NOTE: the redemption (`Member` + `ConsumedInvite`) and the binding below are two
426
577
  // SEPARATE commits — measured at 27 and 18 `/cluster` streams on 2026-09-17.
@@ -433,10 +584,8 @@ export class StrandMembershipReconciler {
433
584
  // cost ever shows up in a measurement, try the single transaction and measure against the
434
585
  // 27 + 18 baseline.
435
586
  await this.ensureBinding(db, keyPair);
436
- } catch (error) {
437
- log('[%s] reconcile pass failed — retrying next tick: %o', this.deps.label, error);
438
587
  } finally {
439
- this.scheduleNext();
588
+ this.noteAttemptOutcome();
440
589
  }
441
590
  }
442
591
 
@@ -503,59 +652,84 @@ export class StrandMembershipReconciler {
503
652
  memberKey: keyPair.publicKeyB64,
504
653
  }, OWN_TRANSACTION);
505
654
  } catch (error) {
506
- this.handleConsumeFailure(error);
655
+ this.handleConsumeFailure(error, invite);
507
656
  return false;
508
657
  }
509
- this.deps.pendingInvite?.clear();
658
+ this.deps.pendingInvite?.clear(invite);
510
659
  log('[%s] redeemed the staged membership invitation — Member row seated under this party\'s key', this.deps.label);
511
660
  return true;
512
661
  }
513
662
 
514
663
  /**
515
- * The already-member arm: spend a still-staged invitation so nobody else can. Burn
516
- * failures (already spent, cancelled, expired — or a racing seal) are logged and
517
- * otherwise ignored, and the stage is cleared either way: with the member row
518
- * present the invitation has no further local use, and keeping a possibly-dead
519
- * credential staged would leave `getPendingMembershipInvite` lying. The one exception
520
- * is a busy refusal (the app had a transaction open): nothing was tried, so the
521
- * invitation stays staged and the next pass burns it.
664
+ * The already-member arm: spend a still-staged invitation so nobody else can. The stage
665
+ * is cleared once the burn lands or the invitation is known dead (already spent,
666
+ * cancelled, expired, or the strand sealed around a party that is still a member) — with
667
+ * the member row present a dead credential has no further local use, and keeping it
668
+ * staged would leave `getPendingMembershipInvite` lying. A refused burn keeps it staged:
669
+ * a busy refusal tried nothing, and a write the cohort refused is the removed-party shape
670
+ * ("Stops, and re-arming" in the module doc), where the local member row is stale and the
671
+ * credential is the one thing a later manager admission lets this loop settle.
522
672
  *
523
- * NOTE: accepted tradeoff — a burn that failed for a TRANSIENT reason is never
524
- * retried, so that bearer credential stays spendable until it expires. Weighed and
525
- * kept because it lands in the same state the strand already documents as normal
526
- * ("removal does not cancel an unspent invitation", docs/strands.md) and retrying
527
- * would mean keeping a dead-or-alive credential staged indefinitely. Revisit if
528
- * unspent invitations ever become a real admission risk — which is when
529
- * `feat-strand-member-allowlist-admission` lands.
673
+ * NOTE: this REPLACES an earlier accepted tradeoff that never retried a burn which failed
674
+ * for a transient reason and dropped the invitation instead. That decision was made when
675
+ * this arm could only ever see an invitation staged before the first join finished, so a
676
+ * dropped credential cost nothing but a spendable bearer token. {@link rearm} makes the
677
+ * arm reachable with a freshly issued invitation the app wants redeemed, and dropping
678
+ * that one on the first refused write is the silent failure this loop exists to avoid.
530
679
  */
531
680
  private async burnLeftoverInvite(db: Database, keyPair: Ed25519KeyPair): Promise<void> {
532
- const pending = this.deps.pendingInvite;
533
- const invite = pending?.get();
534
- if (!pending || !invite) return;
681
+ const invite = this.deps.pendingInvite?.get();
682
+ if (!invite) return;
535
683
  try {
536
684
  await burnInvite(db, {
537
685
  inviteKey: invite.inviteKey,
538
686
  invitePrivateKey: invite.invitePrivateKey,
539
687
  memberKey: keyPair.publicKeyB64,
540
688
  }, OWN_TRANSACTION);
541
- log('[%s] burned the leftover invitation (member row already present)', this.deps.label);
542
689
  } catch (error) {
543
- if (isTransactionBusy(error)) {
690
+ this.handleBurnFailure(error, invite);
691
+ return;
692
+ }
693
+ this.deps.pendingInvite?.clear(invite);
694
+ log('[%s] burned the leftover invitation (member row already present)', this.deps.label);
695
+ }
696
+
697
+ /**
698
+ * Act on a `burnInvite` rejection as {@link classifyConsumeFailure} routes it. Unlike a
699
+ * redemption, a sealed strand is not terminal here — the member row is present, so this
700
+ * party is a member of the sealed strand and only its leftover credential is dead — and a
701
+ * half-committed burn (one collection, so a report of it means the commit's durability
702
+ * was in doubt) is retried: the next pass either lands it or fails on the row's primary
703
+ * key, which drops it as dead.
704
+ */
705
+ private handleBurnFailure(error: unknown, invite: StrandMembershipInvite): void {
706
+ const failure = classifyConsumeFailure(error);
707
+ switch (failure.kind) {
708
+ case 'busy':
544
709
  log('[%s] burning the leftover invitation deferred — the app has a transaction open; it stays staged '
545
710
  + 'for the next pass', this.deps.label);
546
711
  return;
547
- }
548
- log('[%s] burning the leftover invitation failed (already spent, cancelled, or expired) — dropping it: %o',
549
- this.deps.label, error);
712
+ case 'sealed':
713
+ case 'dead-invite':
714
+ log('[%s] the leftover invitation is dead (already spent, cancelled, expired, or the strand sealed) — '
715
+ + 'dropping it: %s', this.deps.label, errorMessage(error));
716
+ this.deps.pendingInvite?.clear(invite);
717
+ return;
718
+ case 'half-committed':
719
+ case 'retry':
720
+ log('[%s] burning the leftover invitation was refused — it stays staged and is retried next tick '
721
+ + '(the cohort refusing this machine, or a transient write failure): %s', this.deps.label, errorMessage(error));
722
+ return;
550
723
  }
551
- pending.clear();
552
724
  }
553
725
 
554
726
  /**
555
727
  * Step 3 of the pass: write this machine's own `MemberPeer` binding (insert-if-absent)
556
- * and, on success, latch the done state and stop the loop. A missing transport peer id
557
- * (a quiesce racing the pass) defers to the next tick; a write failure — a busy refusal
558
- * included — is contained by the pass's outer catch and retried.
728
+ * and, once it is in place with nothing staged, latch the done state and stop the loop.
729
+ * A missing transport peer id (a quiesce racing the pass) defers to the next tick; a
730
+ * write failure — a busy refusal included — is contained by the pass's outer catch and
731
+ * retried. A binding written while an invitation is still staged (its burn keeps being
732
+ * refused) is not done: the loop stays live so a later pass can settle the credential.
559
733
  */
560
734
  private async ensureBinding(db: Database, keyPair: Ed25519KeyPair): Promise<void> {
561
735
  const peerId = this.deps.getOwnPeerId();
@@ -564,8 +738,12 @@ export class StrandMembershipReconciler {
564
738
  return;
565
739
  }
566
740
  await registerMemberPeer(db, { memberKeyPair: keyPair, peerId }, OWN_TRANSACTION);
741
+ if (this.deps.pendingInvite?.get()) {
742
+ log('[%s] own MemberPeer binding is in place but an invitation is still staged — not done yet', this.deps.label);
743
+ return;
744
+ }
567
745
  this.doneFlag = true;
568
- this.finish('member row and own MemberPeer binding are both in place');
746
+ this.finish('member row and own MemberPeer binding are both in place', false);
569
747
  }
570
748
 
571
749
  /**
@@ -573,7 +751,7 @@ export class StrandMembershipReconciler {
573
751
  * the invitation and retry; half-committed → warn and drop; sealed → terminal; dead
574
752
  * invitation → drop; anything else → retry.
575
753
  */
576
- private handleConsumeFailure(error: unknown): void {
754
+ private handleConsumeFailure(error: unknown, invite: StrandMembershipInvite): void {
577
755
  const failure = classifyConsumeFailure(error);
578
756
  switch (failure.kind) {
579
757
  case 'busy':
@@ -581,12 +759,12 @@ export class StrandMembershipReconciler {
581
759
  + 'database; retrying next tick', this.deps.label);
582
760
  return;
583
761
  case 'half-committed':
584
- this.reportHalfCommittedJoin(failure, error);
762
+ this.reportHalfCommittedJoin(failure, error, invite);
585
763
  return;
586
764
  case 'sealed':
587
765
  // The staged credential is dead with the seal; drop it so the cache stays honest.
588
- this.deps.pendingInvite?.clear();
589
- this.finish('the strand is sealed — nobody can ever be admitted, so the staged invitation is dead');
766
+ this.deps.pendingInvite?.clear(invite);
767
+ this.finish('the strand is sealed — nobody can ever be admitted, so the staged invitation is dead', true);
590
768
  return;
591
769
  case 'dead-invite':
592
770
  // NOTE: accepted tradeoff — dropping a dead invitation does not mark the pass idle,
@@ -596,15 +774,67 @@ export class StrandMembershipReconciler {
596
774
  // source of dead invitations ever repeats per pass.
597
775
  log('[%s] the staged invitation is dead (expired, cancelled, or consumed elsewhere) — dropping it; '
598
776
  + 'a fresh formation stages a new one: %s', this.deps.label, errorMessage(error));
599
- this.deps.pendingInvite?.clear();
777
+ this.deps.pendingInvite?.clear(invite);
600
778
  return;
601
779
  case 'retry':
602
- log('[%s] consumeInvite failed — retrying next tick (Invite row not yet replicated here, or a transient '
603
- + 'write failure): %s', this.deps.label, errorMessage(error));
780
+ log('[%s] consumeInvite failed — retrying next tick (Invite row not yet replicated here, the cohort '
781
+ + 'refusing this machine, or a transient write failure): %s', this.deps.label, errorMessage(error));
604
782
  return;
605
783
  }
606
784
  }
607
785
 
786
+ /**
787
+ * Count an attempted pass that left the invitation staged (whatever stopped it — a
788
+ * refused or busy write, or a read that threw on a cut-off machine); one that settled
789
+ * it, or finished, ends the streak. At {@link UNFINISHED_PASSES_BEFORE_ESCALATION}
790
+ * report the PROBABLE blocked re-join.
791
+ */
792
+ private noteAttemptOutcome(): void {
793
+ if (this.doneFlag || !this.deps.pendingInvite?.get()) {
794
+ this.unfinishedPasses = 0;
795
+ return;
796
+ }
797
+ this.unfinishedPasses += 1;
798
+ if (this.unfinishedPasses >= UNFINISHED_PASSES_BEFORE_ESCALATION) {
799
+ this.reportRejoinBlocked(
800
+ `${this.unfinishedPasses} membership reconcile passes have left the staged membership invitation `
801
+ + 'unsettled. Either this party was removed from the strand and the remaining members are refusing its '
802
+ + 'machines, or the invitation\'s Strand.Invite row has not replicated here yet.'
803
+ );
804
+ }
805
+ }
806
+
807
+ /**
808
+ * Step 1's CONFIRMED trigger: self-revoked with an invitation staged. Reports (once per
809
+ * re-arm cycle) and answers `true` so the pass carries on — the redemption is attempted
810
+ * and refused like any other write, and a manager's re-admission lets a later pass finish.
811
+ * `false` (nothing staged) means step 1 stops the loop as before.
812
+ */
813
+ private stagedInviteBlockedBySelfRevocation(): boolean {
814
+ if (!this.deps.pendingInvite?.get()) return false;
815
+ this.reportRejoinBlocked(
816
+ 'this party is revoked from the strand and holds a staged membership invitation it cannot spend: the '
817
+ + 'remaining members refuse this machine\'s strand writes, so the redemption cannot land.'
818
+ );
819
+ return true;
820
+ }
821
+
822
+ /**
823
+ * ONE `console.warn` and one callback per re-arm cycle, whichever trigger fires first —
824
+ * see "Reporting a blocked re-join" in the module doc. `cause` is the trigger's own
825
+ * sentence; the remedy and the loop's posture are the same for both.
826
+ */
827
+ private reportRejoinBlocked(cause: string): void {
828
+ if (this.rejoinBlockedReported) return;
829
+ this.rejoinBlockedReported = true;
830
+ console.warn(
831
+ `[sereus] strand ${this.deps.label}: ${cause} A fresh invitation cannot re-admit a removed party by itself — `
832
+ + 'a remaining manager must admit this party\'s member key directly (addMemberByManager). The membership '
833
+ + 'loop keeps retrying quietly and completes the join once a Member row for this party appears.'
834
+ );
835
+ this.deps.onRejoinBlocked?.();
836
+ }
837
+
608
838
  /**
609
839
  * A redemption only part of which was saved (see "Half-committed join" in the module doc):
610
840
  * ONE visible warning naming both halves, and the staged invitation dropped. The pass is not
@@ -621,7 +851,7 @@ export class StrandMembershipReconciler {
621
851
  * path. Revisit if a `Member`-saved half-commit is ever observed, or when
622
852
  * `blocked/strand-half-committed-join-recovery` is decided.
623
853
  */
624
- private reportHalfCommittedJoin(failure: HalfCommittedJoin, error: unknown): void {
854
+ private reportHalfCommittedJoin(failure: HalfCommittedJoin, error: unknown, invite: StrandMembershipInvite): void {
625
855
  console.warn(
626
856
  `[sereus] strand ${this.deps.label}: redeeming the staged membership invitation was only partly saved. `
627
857
  + `Saved: [${failure.saved.join(', ')}]. Not saved: [${failure.unsaved.join(', ')}]. `
@@ -631,7 +861,7 @@ export class StrandMembershipReconciler {
631
861
  + 'The membership loop keeps checking and completes the join once a Member row for this party appears.'
632
862
  );
633
863
  log('[%s] half-committed redemption, full error: %o', this.deps.label, error);
634
- this.deps.pendingInvite?.clear();
864
+ this.deps.pendingInvite?.clear(invite);
635
865
  // The idle passes that follow are the wait this warning already explained; the escalation's
636
866
  // "waiting on the founder rows to replicate" would be a second, misleading warning.
637
867
  this.idleEscalated = true;
@@ -643,7 +873,7 @@ export class StrandMembershipReconciler {
643
873
  try {
644
874
  this.keyPair = strandMemberKeyPair(this.deps.partyMemberPrivateKey);
645
875
  } catch (error) {
646
- this.finish(`the party membership key does not decode — nothing can be signed (${errorMessage(error)})`);
876
+ this.finish(`the party membership key does not decode — nothing can be signed (${errorMessage(error)})`, true);
647
877
  return undefined;
648
878
  }
649
879
  return this.keyPair;
@@ -672,9 +902,21 @@ export class StrandMembershipReconciler {
672
902
  }
673
903
  }
674
904
 
675
- /** Stop with a reason — the terminal and done paths' shared exit. */
676
- private finish(reason: string): void {
677
- log('[%s] membership reconciliation stopping: %s', this.deps.label, reason);
678
- this.stop();
905
+ /**
906
+ * Stop with a reason — the terminal and done paths' shared exit. `permanent` says whether
907
+ * a fresh invitation may reopen the loop ("Stops, and re-arming" in the module doc).
908
+ */
909
+ private finish(reason: string, permanent: boolean): void {
910
+ this.halt(`membership reconciliation stopping: ${reason}`, permanent);
911
+ }
912
+
913
+ /** Latch the stop and disarm the timer; a permanent latch sticks even on an already-stopped loop. */
914
+ private halt(reason: string, permanent: boolean): void {
915
+ if (permanent) this.permanentlyStopped = true;
916
+ if (this.stoppedFlag) return;
917
+ this.stoppedFlag = true;
918
+ this.clearTimer();
919
+ log('[%s] membership reconciler stopped%s: %s%s', this.deps.label, this.doneFlag ? ' (done)' : '', reason,
920
+ permanent ? '' : ' — a fresh invitation re-arms it');
679
921
  }
680
922
  }