@serfab/cadre-core 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/README.md +255 -255
  2. package/dist/announce-addrs.d.ts +49 -0
  3. package/dist/announce-addrs.js +78 -0
  4. package/dist/announce-addrs.js.map +1 -0
  5. package/dist/cadre-node.d.ts +484 -58
  6. package/dist/cadre-node.js +953 -182
  7. package/dist/cadre-node.js.map +1 -1
  8. package/dist/control-cohort.d.ts +26 -0
  9. package/dist/control-cohort.js +26 -0
  10. package/dist/control-cohort.js.map +1 -1
  11. package/dist/control-database.d.ts +222 -125
  12. package/dist/control-database.js +356 -266
  13. package/dist/control-database.js.map +1 -1
  14. package/dist/control-formation-recorder.d.ts +8 -7
  15. package/dist/control-formation-recorder.js +8 -7
  16. package/dist/control-formation-recorder.js.map +1 -1
  17. package/dist/control-read-retry.d.ts +103 -0
  18. package/dist/control-read-retry.js +145 -0
  19. package/dist/control-read-retry.js.map +1 -0
  20. package/dist/control-retry.d.ts +101 -0
  21. package/dist/control-retry.js +108 -0
  22. package/dist/control-retry.js.map +1 -0
  23. package/dist/control-schema.d.ts +1 -1
  24. package/dist/control-schema.js +805 -774
  25. package/dist/control-schema.js.map +1 -1
  26. package/dist/control-write-retry.d.ts +65 -31
  27. package/dist/control-write-retry.js +33 -103
  28. package/dist/control-write-retry.js.map +1 -1
  29. package/dist/delegate-admission.d.ts +17 -15
  30. package/dist/delegate-admission.js +17 -15
  31. package/dist/delegate-admission.js.map +1 -1
  32. package/dist/index.d.ts +7 -5
  33. package/dist/index.js +18 -9
  34. package/dist/index.js.map +1 -1
  35. package/dist/membership-connection-gater.d.ts +211 -25
  36. package/dist/membership-connection-gater.js +309 -29
  37. package/dist/membership-connection-gater.js.map +1 -1
  38. package/dist/peer-addr-book.d.ts +92 -0
  39. package/dist/peer-addr-book.js +220 -0
  40. package/dist/peer-addr-book.js.map +1 -0
  41. package/dist/peer-record.d.ts +26 -0
  42. package/dist/peer-record.js +32 -0
  43. package/dist/peer-record.js.map +1 -1
  44. package/dist/relay-addrs.d.ts +76 -26
  45. package/dist/relay-addrs.js +115 -27
  46. package/dist/relay-addrs.js.map +1 -1
  47. package/dist/relay-reservation.d.ts +20 -15
  48. package/dist/relay-reservation.js +18 -14
  49. package/dist/relay-reservation.js.map +1 -1
  50. package/dist/seed-bootstrap.d.ts +7 -0
  51. package/dist/seed-bootstrap.js +15 -13
  52. package/dist/seed-bootstrap.js.map +1 -1
  53. package/dist/strand-addr-protocol.js +16 -0
  54. package/dist/strand-addr-protocol.js.map +1 -1
  55. package/dist/strand-backfill.js +13 -4
  56. package/dist/strand-backfill.js.map +1 -1
  57. package/dist/strand-cohort.d.ts +4 -37
  58. package/dist/strand-cohort.js +4 -15
  59. package/dist/strand-cohort.js.map +1 -1
  60. package/dist/strand-database.d.ts +11 -16
  61. package/dist/strand-database.js +19 -7
  62. package/dist/strand-database.js.map +1 -1
  63. package/dist/strand-formation-manager.d.ts +18 -17
  64. package/dist/strand-formation-manager.js +25 -27
  65. package/dist/strand-formation-manager.js.map +1 -1
  66. package/dist/strand-formation-protocol.d.ts +7 -7
  67. package/dist/strand-formation-protocol.js +8 -8
  68. package/dist/strand-formation-protocol.js.map +1 -1
  69. package/dist/strand-instance-manager.d.ts +53 -21
  70. package/dist/strand-instance-manager.js +128 -37
  71. package/dist/strand-instance-manager.js.map +1 -1
  72. package/dist/strand-membership-writer.d.ts +113 -19
  73. package/dist/strand-membership-writer.js +219 -51
  74. package/dist/strand-membership-writer.js.map +1 -1
  75. package/dist/strand-solicitation.d.ts +3 -3
  76. package/dist/strand-solicitation.js +4 -5
  77. package/dist/strand-solicitation.js.map +1 -1
  78. package/dist/strand-wake-protocol.d.ts +47 -2
  79. package/dist/strand-wake-protocol.js +76 -6
  80. package/dist/strand-wake-protocol.js.map +1 -1
  81. package/dist/types.d.ts +107 -54
  82. package/dist/types.js.map +1 -1
  83. package/package.json +8 -7
  84. package/src/announce-addrs.ts +93 -0
  85. package/src/arachnode-stub.ts +159 -159
  86. package/src/bootstrap-peer-store-file.ts +44 -44
  87. package/src/bootstrap-peer-store.ts +172 -172
  88. package/src/cadre-node.ts +1026 -194
  89. package/src/canonical-json.ts +30 -30
  90. package/src/control-authorization.ts +114 -114
  91. package/src/control-cohort.ts +130 -103
  92. package/src/control-database.ts +376 -298
  93. package/src/control-formation-recorder.ts +305 -304
  94. package/src/control-read-retry.ts +167 -0
  95. package/src/control-retry.ts +190 -0
  96. package/src/control-schema.ts +817 -786
  97. package/src/control-stream.ts +205 -205
  98. package/src/control-write-retry.ts +383 -473
  99. package/src/delegate-admission.ts +304 -302
  100. package/src/device-token.ts +94 -94
  101. package/src/diagnostics/connection-path.ts +277 -277
  102. package/src/diagnostics/webrtc-turn-tracker.ts +222 -222
  103. package/src/ed25519-key.ts +134 -134
  104. package/src/enrollment.ts +199 -199
  105. package/src/file-durable-slot.ts +61 -61
  106. package/src/formation-approval.ts +604 -604
  107. package/src/fs-atomic.ts +107 -107
  108. package/src/hibernation-manager.ts +437 -437
  109. package/src/identity-key.ts +109 -109
  110. package/src/index.ts +435 -406
  111. package/src/key-store-file.ts +132 -132
  112. package/src/key-store.ts +99 -99
  113. package/src/membership-connection-gater.ts +368 -39
  114. package/src/node-local-snapshot.ts +260 -260
  115. package/src/peer-addr-book.ts +250 -0
  116. package/src/peer-authorization.ts +259 -259
  117. package/src/peer-record.ts +197 -164
  118. package/src/push-credentials.ts +84 -84
  119. package/src/push-node.ts +64 -64
  120. package/src/push-notifier-apns.ts +247 -247
  121. package/src/push-notifier-fcm.ts +220 -220
  122. package/src/push-notifier-shared.ts +28 -28
  123. package/src/push-notifier.ts +52 -52
  124. package/src/relay-addrs.ts +134 -27
  125. package/src/relay-reservation.ts +22 -17
  126. package/src/seed-bootstrap.ts +15 -15
  127. package/src/seed-trust-policy.ts +140 -140
  128. package/src/strand-addr-protocol.ts +458 -441
  129. package/src/strand-backfill.ts +28 -5
  130. package/src/strand-cohort.ts +38 -77
  131. package/src/strand-database.ts +202 -204
  132. package/src/strand-formation-manager.ts +26 -28
  133. package/src/strand-formation-protocol.ts +692 -692
  134. package/src/strand-instance-manager.ts +636 -553
  135. package/src/strand-member-key.ts +39 -39
  136. package/src/strand-member-registry.ts +182 -182
  137. package/src/strand-membership-writer.ts +1627 -1430
  138. package/src/strand-solicitation.ts +471 -472
  139. package/src/strand-transport-key.ts +58 -58
  140. package/src/strand-wake-payload.ts +32 -32
  141. package/src/strand-wake-protocol.ts +400 -314
  142. package/src/strand-watcher.ts +314 -314
  143. package/src/trusted-owner-store-file.ts +41 -41
  144. package/src/trusted-owner-store.ts +171 -171
  145. package/src/types.ts +1354 -1303
@@ -1,6 +1,6 @@
1
1
  import debug from 'debug';
2
2
  import { toString as uint8ArrayToString } from 'uint8arrays';
3
- import { Database, registerPlugin, unwrapError } from '@quereus/quereus';
3
+ import { Database, registerPlugin } from '@quereus/quereus';
4
4
  import cryptoPlugin from '@optimystic/quereus-plugin-crypto/plugin';
5
5
  import optimysticPlugin from '@optimystic/quereus-plugin-optimystic/plugin';
6
6
  import { digest, randomBytes } from '@optimystic/quereus-plugin-crypto';
@@ -9,6 +9,7 @@ import { canonicalDatetime } from './canonical-datetime.js';
9
9
  import { controlAuthorizationFields, CONTROL_TABLES } from './control-authorization.js';
10
10
  import { requireEd25519PublicKeyB64 } from './ed25519-key.js';
11
11
  import { retryControlWrite, SCHEMA_INIT_RETRY_POLICY } from './control-write-retry.js';
12
+ import { retryControlRead } from './control-read-retry.js';
12
13
  const log = debug('sereus:cadre:control-db');
13
14
  const timing = debug('sereus:cadre:timing');
14
15
  /**
@@ -73,94 +74,30 @@ export class FormationAbortedError extends Error {
73
74
  }
74
75
  }
75
76
  /**
76
- * A redemption could not be given a `UseNumber` within the invite's seat budget: the use
77
- * number the writer would have to take is past `FormationInvite.TotalUses`, so the
78
- * invitation is spent.
77
+ * The invitation's seat budget is already spent: the count of recorded `FormationUsage` rows
78
+ * for the token has reached `FormationInvite.TotalUses`, so this redemption cannot be given a
79
+ * seat.
79
80
  *
80
- * Raised inside {@link ControlDatabase.redeemInvitation} / {@link ControlDatabase.recordFormationUsage}
81
- * on EVERY attempt, including the first: two redemptions of one token racing on the SAME node
82
- * serialize behind the local write queue, so the loser's first attempt already takes a use
83
- * number past the seat budget — `StrandFormationManager.validateToken`'s `isTokenUsed` check
84
- * cannot catch this, since it ran before either redemption wrote anything. Without this the
85
- * write would fail as a generic `CHECK constraint failed: Authorized`, which the manager
86
- * reports as a retryable `Formation conflict, retry` — telling the joiner to retry something
87
- * that can never succeed.
81
+ * Raised off the committed-count check ({@link ControlDatabase.assertSeatRemains}) ahead of
82
+ * the write, inside {@link ControlDatabase.redeemInvitation} /
83
+ * {@link ControlDatabase.recordFormationUsage} — and off nothing else. It exists because
84
+ * without it the refusal surfaces as a generic `CHECK constraint failed: Authorized` from the
85
+ * schema's own count-based cap clause, which the manager reports as a retryable
86
+ * `Formation conflict, retry` — telling the joiner to retry something that can never succeed.
87
+ * Two redemptions of one token racing on the SAME node serialize behind the local write
88
+ * queue, so the loser reads the winner's committed row and is refused here by name —
89
+ * `StrandFormationManager.validateToken`'s `isTokenUsed` check cannot catch that, since it
90
+ * ran before either redemption wrote anything.
88
91
  */
89
92
  export class InvitationExhaustedError extends Error {
90
- constructor(token, useNumber, totalUses) {
91
- super(`Invitation ${token} is exhausted: use #${useNumber} is past its limit of ${totalUses} use(s)`);
93
+ constructor(token, usesRecorded, totalUses) {
94
+ super(`Invitation ${token} is exhausted: ${usesRecorded} of ${totalUses} use(s) already recorded`);
92
95
  this.token = token;
93
- this.useNumber = useNumber;
96
+ this.usesRecorded = usesRecorded;
94
97
  this.totalUses = totalUses;
95
98
  this.name = 'InvitationExhaustedError';
96
99
  }
97
100
  }
98
- /**
99
- * Attempts allowed per redemption: the first, plus two retries of a use number another
100
- * writer took. Bounded so sustained contention terminates in a clean rejection rather than
101
- * spinning; two retries is enough for the realistic case (a handful of nodes redeeming one
102
- * multi-use token at once) without holding the write lock hostage.
103
- *
104
- * NOTE: the attempts fire back-to-back, with no backoff and no jitter — each one only takes
105
- * and releases the write lock. Fine while cross-node contention on a single token is a
106
- * handful of writers; if one hot multi-use token ever draws sustained concurrent redemption
107
- * (a broadcast invite, a bulk onboarding run), all three attempts can burn against the same
108
- * competitor and the caller gets the engine's error. Add a short randomised delay between
109
- * attempts then — NOT a longer attempt budget, which would hold the write queue.
110
- */
111
- const USE_NUMBER_ATTEMPTS = 3;
112
- /**
113
- * The two error messages that mean "another writer took the `UseNumber` we picked", and
114
- * nothing else.
115
- *
116
- * - the `(Token, UseNumber)` primary-key collision, which the optimystic vtab reports on the
117
- * insert itself (`uniqueConstraintMessage` renders the columns table-qualified, no schema
118
- * prefix);
119
- * - `CHECK constraint failed: Monotonic` — `FormationUsage.Monotonic` contains a subquery, so
120
- * Quereus defers it to commit where it reads `committed.FormationUsage`. A concurrent row
121
- * that our insert's key probe did not see still trips it there. Same meaning, different
122
- * surface.
123
- *
124
- * Deliberately NOT matched: `FormationUsage.UsageStampId`'s `unique` (a repeated nonce is
125
- * replay, never a race — retrying it would re-present a spent approval), and every named
126
- * CHECK other than `Monotonic`. `Authorized` in particular renders with the same
127
- * `CHECK constraint failed:` prefix, so the constraint NAME is anchored with `\b` exactly as
128
- * `expectConstraintFailure` does rather than matched by prefix.
129
- *
130
- * NOTE: a CHECK failure names only the CONSTRAINT, never its table, and `Monotonic` is a
131
- * generic enough name that a second table could plausibly claim it. Today it is unique across
132
- * `schemas/control.qsql`, so the pattern cannot mis-fire; if another table ever declares a
133
- * `Monotonic` constraint, a failure of THAT constraint would be read here as a lost use-number
134
- * race and pointlessly retried up to {@link USE_NUMBER_ATTEMPTS} times. Rename the other
135
- * constraint (cheapest) rather than trying to recover the table from the message.
136
- */
137
- const LOST_USE_NUMBER_PATTERNS = [
138
- /UNIQUE constraint failed: FormationUsage\.Token, FormationUsage\.UseNumber/i,
139
- /CHECK constraint failed: Monotonic\b/,
140
- ];
141
- /**
142
- * Did this write fail purely because another writer took the `UseNumber` it picked — i.e. is
143
- * re-presenting the SAME approval under the next free use number the right response?
144
- *
145
- * Classifies by MESSAGE, walking the `cause` chain with Quereus' own {@link unwrapError}:
146
- * neither surface can be recognised by type. The deferred `Monotonic` failure is thrown as a
147
- * bare `QuereusError`, not a `ConstraintError` (see
148
- * `quereus/src/runtime/deferred-constraint-queue.ts`), so an `instanceof ConstraintError` gate
149
- * would silently drop the branch this retry exists for. Messages also carry an
150
- * ` (at line N, column M)` suffix and arrive wrapped when the write ran inside a transaction,
151
- * so the match is a substring test rather than an equality one.
152
- *
153
- * NOTE: this is the one piece of the retry that depends on engine error TEXT. Its own test
154
- * (`control-formation-use-number-retry.spec.ts`) produces all three messages from the real
155
- * engine rather than from string literals, so a reworded storage-layer error fails that spec
156
- * instead of silently disabling the retry.
157
- */
158
- export function isLostUseNumberRace(error) {
159
- if (!(error instanceof Error)) {
160
- return false;
161
- }
162
- return unwrapError(error).some(({ message }) => LOST_USE_NUMBER_PATTERNS.some(pattern => pattern.test(message)));
163
- }
164
101
  /**
165
102
  * Parse a stored Quereus `datetime` value into epoch milliseconds.
166
103
  *
@@ -312,6 +249,13 @@ export class ControlDatabase {
312
249
  * having to restate the policy's attempts or classifier.
313
250
  */
314
251
  this.controlWriteRetryPacing = {};
252
+ /**
253
+ * Pacing seams for {@link readRows}'s retry — the read twin of
254
+ * {@link controlWriteRetryPacing}, with the same contract: production leaves this
255
+ * empty, specs inject a recorded `sleep` / fake `now` so no test waits out a real
256
+ * backoff. Applied OVER the read policy's defaults but UNDER the call site's `label`.
257
+ */
258
+ this.controlReadRetryPacing = {};
315
259
  this.config = config;
316
260
  }
317
261
  /**
@@ -377,6 +321,20 @@ export class ControlDatabase {
377
321
  timing('[controlDb] hydrate: %dms (tables=%d, indexes=%d)', Math.round(performance.now() - t0), hydrated.tables, hydrated.indexes);
378
322
  log('Hydrated control catalog (tables=%d, indexes=%d)', hydrated.tables, hydrated.indexes);
379
323
  // Load and execute the schema
324
+ // NOTE: this is where a slow launch is felt. Duration is (raw-storage operations
325
+ // issued) × per-operation storage latency: ~1ms/op on an idle machine, but
326
+ // 50-90ms/op on a loaded disk or a phone's flash under launch contention. A cold
327
+ // start now reaches the backend 172 times (8 tables + 1 index, 21 distinct blocks
328
+ // — dominated by its 130 genuine writes), a warm restart 52, because cadre-core
329
+ // wraps every embedder storage in `@optimystic/db-p2p`'s write-through cache
330
+ // (@serfab/quereus-plugin-sereus's cached-storage.ts). Uncached the same start issued ~2000 operations — the
331
+ // upstream re-read amplification measured in
332
+ // tickets/complete/optimystic-block-read-amplification-on-control-start.md (and
333
+ // its follow-up ...-schema-catalog-reread-per-write-blows-storage-budgets.md),
334
+ // which also rules out the retry policy and cluster deadlines. The counts are
335
+ // pinned by packages/cadre-core/test/control-start-storage-op-budget.spec.ts; if
336
+ // they grow — e.g. the cache leaves this path — that spec fails before anyone
337
+ // has to debug a launch.
380
338
  t0 = performance.now();
381
339
  await this.loadSchema();
382
340
  timing('[controlDb] loadSchema: %dms', Math.round(performance.now() - t0));
@@ -443,16 +401,129 @@ export class ControlDatabase {
443
401
  // also what lets schema init absorb optimystic's self-coordination grace refusal, which
444
402
  // is unsafe for control writes in general. Why, on
445
403
  // `RETRIABLE_SCHEMA_INIT_MATCHERS` in `control-write-retry.ts`.
446
- await this.lockedWithRetry(() => this.db.exec(schemaContent), SCHEMA_INIT_RETRY_POLICY);
404
+ await this.lockedWithRetry(() => this.db.exec(schemaContent), SCHEMA_INIT_RETRY_POLICY, 'schema-init');
447
405
  log('Schema loaded and executed');
448
406
  }
407
+ /**
408
+ * One UNRETRIED drain of a control read to rows — every read in this class bottoms out
409
+ * here (almost all via {@link readRows}, which adds the transient-failure retry), and
410
+ * this is the ONE place the committed-read opt-in is spelled.
411
+ *
412
+ * Reached only through {@link readRows}, which drops straight here for the call sites
413
+ * that pass `retry: false` — reads issued from INSIDE a locked write body, where a
414
+ * backoff sleep would hold the write lock and the write funnel re-runs the read anyway
415
+ * (see the NOTE on {@link readRows}).
416
+ *
417
+ * A control read normally queues on the database's exec mutex, so it does not answer
418
+ * until whatever statement holds that mutex finishes. When a control WRITE is parked
419
+ * against an unresponsive cohort member that is minutes, and a read that only wants
420
+ * the last committed state waits the whole time for no reason
421
+ * (`complete/control-reads-blocked-by-stalled-write`).
422
+ *
423
+ * Quereus's remedy is `readConcurrency: 'committed'` — a PER-CALL opt-in
424
+ * (`Statement.tryRouteConcurrent` refuses anything that did not ask for it by name)
425
+ * that runs an eligible read off the mutex against each table's last committed state.
426
+ * Routing is best-effort and never an error: an ineligible statement falls back to
427
+ * the serialized path silently — any side-effecting node, any table-valued function
428
+ * call (pure ones included), any table whose module does not declare
429
+ * `readCommittedSnapshot`, or an EXPLICIT transaction being open on this database.
430
+ *
431
+ * **It is asked for only while a write is in flight, and that condition is the whole
432
+ * point.** The opt-in does not merely move the read off the mutex: it connects the
433
+ * table with `_readCommitted`, and the optimystic vtab serves such a connection from
434
+ * a pinned pre-transaction snapshot that NEVER refreshes from the network, where an
435
+ * ordinary read calls `update()` on the collection first. Taking that path
436
+ * unconditionally therefore trades "reads block behind a stalled write" for "reads
437
+ * stop seeing what other machines wrote" — measured, not theorised: with every read
438
+ * opted in, `control-write-degraded-cohort-member` could no longer observe a sibling
439
+ * publishing its own `CadrePeer` row and failed at suite setup on two runs out of two.
440
+ *
441
+ * `getAutocommit()` reports the whole `Database` rather than this call (the same
442
+ * property {@link assertCommitBoundary} leans on), which is exactly what is wanted:
443
+ * false means some writer's transaction is open, which is the only situation where a
444
+ * read would have blocked. The check races that writer harmlessly in both directions
445
+ * — losing it either reads normally (today's behaviour) or reads committed state.
446
+ *
447
+ * What a committed read gives up is observing a write that has not finished
448
+ * committing, and that is safe for every caller here: an AWAITED write is committed by
449
+ * the time it returns, and a read inside {@link inTransaction} is disqualified by the
450
+ * open explicit transaction and stays serialized, so it still sees its own
451
+ * transaction's rows. A read after an UNAWAITED write on this database would be wrong
452
+ * — no such caller exists, and {@link withWriteLock} is what keeps it that way.
453
+ */
454
+ async readRowsOnce(sql, params) {
455
+ let iterator;
456
+ if (this.db.getAutocommit()) {
457
+ iterator = this.db.eval(sql, params);
458
+ }
459
+ else {
460
+ log('read-eval: a write is in flight, asking for a committed read: %s', sql);
461
+ iterator = this.db.eval(sql, params, { readConcurrency: 'committed' });
462
+ }
463
+ const rows = [];
464
+ for await (const row of iterator) {
465
+ rows.push(row);
466
+ }
467
+ return rows;
468
+ }
469
+ /**
470
+ * {@link readRowsOnce} plus the bounded transient-failure retry
471
+ * ({@link retryControlRead}) — the funnel every UNLOCKED control read goes through, so a
472
+ * read that failed because the cluster could not be asked properly (a stream reset
473
+ * mid-scan, a partially unreachable cohort) is re-presented a moment later instead of
474
+ * surfacing to every reader.
475
+ *
476
+ * MATERIALIZES the read rather than returning a lazy iterator, and that shape is what
477
+ * makes the retry possible at all: the failure happens during ITERATION, not at call
478
+ * time, and a half-consumed iterator cannot be retried without re-yielding rows the
479
+ * caller already saw. Materializing is safe here — every caller either drains to an
480
+ * array anyway or takes the first row of a statement that yields at most one (a
481
+ * primary-key lookup or a `count(1)`); nothing streams.
482
+ *
483
+ * Each ATTEMPT is a fresh {@link readRowsOnce} call, so the committed-read opt-in
484
+ * (`getAutocommit()`) is re-evaluated per attempt — a write can finish between attempts
485
+ * and change the right answer.
486
+ *
487
+ * The retry's budget ({@link CONTROL_READ_RETRY_BUDGET_MS}, 1.5 s) is deliberately far
488
+ * under the write funnel's: the tightest caller deadline over a control read is the
489
+ * inbound admission gate's 2 s FAIL-OPEN timeout, and a read that outlives it spends its
490
+ * retries after the gate has already admitted (see `control-read-retry.ts`).
491
+ *
492
+ * `label` names the read in the retry loop's `Control read [<label>] …` debug lines and
493
+ * nothing else; several reads are in flight concurrently in a real party, so an
494
+ * unlabelled line cannot be attributed — keep new call sites labelled.
495
+ *
496
+ * `retry: false` drops straight to {@link readRowsOnce}: reads issued from INSIDE a
497
+ * locked write body must not retry on their own. `retryControlWrite`'s contract is that
498
+ * backoff sleeps happen with NO lock held; a read retrying its own backoff inside a
499
+ * locked body would sleep holding the write lock (stalling every other local writer),
500
+ * and the write funnel already re-runs the body's reads when it re-runs the body. See
501
+ * {@link queryStampId} / {@link assertSeatRemains} / {@link queryCadrePeers} for the
502
+ * per-call opt-outs.
503
+ *
504
+ * NOTE: nothing structural enforces that opt-out — it is per-call because an unlocked
505
+ * read runs CONCURRENTLY with a locked body, so `this`-state cannot tell the two apart,
506
+ * and there is no async-context primitive available on every target platform (browser,
507
+ * React Native) to carry the answer. Reads reached through a CALLBACK invoked under the
508
+ * lock are the easy miss: `notifyMembershipChanged` runs its listener with the lock
509
+ * held, and its membership read is opted out at {@link queryCadrePeers}. If another
510
+ * under-lock callback seam is ever added, audit its read graph the same way.
511
+ */
512
+ readRows(sql, params, label, retry = true) {
513
+ if (!retry) {
514
+ return this.readRowsOnce(sql, params);
515
+ }
516
+ // The label lands LAST so the call site's name survives the spec-injected pacing —
517
+ // it changes log attribution only, never behaviour (same layering as lockedWithRetry).
518
+ return retryControlRead(() => this.readRowsOnce(sql, params), { ...this.controlReadRetryPacing, label });
519
+ }
449
520
  /**
450
521
  * Query all strands from the control database
451
522
  */
452
523
  async queryStrands() {
453
524
  this.ensureInitialized();
454
525
  const results = [];
455
- for await (const row of this.db.eval('select Id, MemberPrivateKey, Type from CadreControl.Strand')) {
526
+ for (const row of await this.readRows('select Id, MemberPrivateKey, Type from CadreControl.Strand', undefined, 'strands')) {
456
527
  results.push({
457
528
  Id: row.Id,
458
529
  MemberPrivateKey: row.MemberPrivateKey,
@@ -469,7 +540,7 @@ export class ControlDatabase {
469
540
  */
470
541
  async queryStrand(strandId) {
471
542
  this.ensureInitialized();
472
- for await (const row of this.db.eval('select Id, MemberPrivateKey, Type from CadreControl.Strand where Id = ?', [strandId])) {
543
+ for (const row of await this.readRows('select Id, MemberPrivateKey, Type from CadreControl.Strand where Id = ?', [strandId], 'strand')) {
473
544
  return {
474
545
  Id: row.Id,
475
546
  MemberPrivateKey: row.MemberPrivateKey,
@@ -494,7 +565,7 @@ export class ControlDatabase {
494
565
  if (!CONTROL_TABLE_SET.has(table)) {
495
566
  throw new Error(`Unknown CadreControl table: ${table}`);
496
567
  }
497
- for await (const row of this.db.eval(`select count(1) as Count from CadreControl.${table}`)) {
568
+ for (const row of await this.readRows(`select count(1) as Count from CadreControl.${table}`, undefined, 'count-rows')) {
498
569
  return row.Count ?? 0;
499
570
  }
500
571
  return 0;
@@ -512,7 +583,7 @@ export class ControlDatabase {
512
583
  */
513
584
  async hasOwnerKey() {
514
585
  this.ensureInitialized();
515
- for await (const row of this.db.eval('select count(1) as Count from CadreControl.OwnerKey')) {
586
+ for (const row of await this.readRows('select count(1) as Count from CadreControl.OwnerKey', undefined, 'owner-key-exists')) {
516
587
  return row.Count > 0;
517
588
  }
518
589
  return false;
@@ -543,7 +614,7 @@ export class ControlDatabase {
543
614
  async getOwnerKeys() {
544
615
  this.ensureInitialized();
545
616
  const keys = new Set();
546
- for await (const row of this.db.eval('select Key from CadreControl.OwnerKey')) {
617
+ for (const row of await this.readRows('select Key from CadreControl.OwnerKey', undefined, 'owner-keys')) {
547
618
  keys.add(row.Key);
548
619
  }
549
620
  return keys;
@@ -564,12 +635,29 @@ export class ControlDatabase {
564
635
  * membership-gate refresh path; control tables are a handful of rows today, so this
565
636
  * is cheap — if they ever grow, fold the exclusion into one statement or cache the
566
637
  * retired-stamp set.
638
+ *
639
+ * NOTE: this filter is HAND-MIRRORED by the gate suites' fake control databases
640
+ * (`test/membership-gate-helpers.ts` and `test/cadre-node-authorized-surface.spec.ts`),
641
+ * which feed the node rows directly. Nothing enforces that the copies stay true, so a
642
+ * change to the filter's SHAPE here (what it keys on, which table it consults) must be
643
+ * carried into both, or those suites keep passing against a contract the database no
644
+ * longer honours. Fine while the filter is one `Set.has` on `StampId`; if it grows a
645
+ * second dimension, give the fakes a shared fixture or a contract test instead.
646
+ *
647
+ * `retry: false` is passed by exactly one caller —
648
+ * `CadreNode.refreshAuthorizedControlPeers`, the membership-gate snapshot refresh —
649
+ * because {@link notifyMembershipChanged} invokes that listener with the write lock
650
+ * HELD, and a retrying read there would sleep its backoff holding the lock (see the
651
+ * NOTE on {@link readRows}). It costs nothing: that refresh is best-effort, keeps the
652
+ * previous snapshot on failure, and is re-driven by the next write and by the timed
653
+ * reconcile. BOTH of this method's reads take the flag — retrying either one under the
654
+ * lock is the same defect.
567
655
  */
568
- async queryCadrePeers() {
656
+ async queryCadrePeers(retry = true) {
569
657
  this.ensureInitialized();
570
- const revoked = await this.queryRevokedStamps('CadrePeer');
658
+ const revoked = await this.queryRevokedStamps('CadrePeer', retry);
571
659
  const rows = [];
572
- for await (const row of this.db.eval('select PeerId, Multiaddr, StampId, VouchOwner, VouchSig from CadreControl.CadrePeer')) {
660
+ for (const row of await this.readRows('select PeerId, Multiaddr, StampId, VouchOwner, VouchSig from CadreControl.CadrePeer', undefined, 'cadre-peers', retry)) {
573
661
  const stampId = row.StampId ?? null;
574
662
  if (stampId !== null && revoked.has(stampId)) {
575
663
  continue;
@@ -596,7 +684,7 @@ export class ControlDatabase {
596
684
  async queryValidationKeys() {
597
685
  this.ensureInitialized();
598
686
  const keys = [];
599
- for await (const row of this.db.eval('select Key from CadreControl.ValidationKey')) {
687
+ for (const row of await this.readRows('select Key from CadreControl.ValidationKey', undefined, 'validation-keys')) {
600
688
  keys.push(row.Key);
601
689
  }
602
690
  return keys.sort();
@@ -615,11 +703,21 @@ export class ControlDatabase {
615
703
  * `table` and its {@link GUARDED_KEY_COLUMN} column are interpolated into the SQL; both
616
704
  * come from closed literal unions — no caller-supplied string can reach the statement
617
705
  * (same injection-surface discipline as {@link countRows}'s `CONTROL_TABLE_SET` guard).
706
+ *
707
+ * NOTE: `retry: false` is passed by exactly the call sites that run INSIDE a locked
708
+ * write body — {@link deleteGuardedRow} and {@link insertCadrePeer}'s insert-if-absent
709
+ * guard. A read retrying there would sleep its backoff HOLDING the write lock
710
+ * (`retryControlWrite`'s contract is that backoff sleeps happen with no lock held), and
711
+ * the write funnel already re-runs the body's reads when it re-runs the body. The
712
+ * unlocked callers — the public stamp readers, {@link reauthorizeCadrePeer} and
713
+ * {@link reapRevokedRow}, whose stamp reads run BEFORE their locks are taken — keep the
714
+ * default retried path. Explicit per call rather than an ambient flag: unlocked reads
715
+ * run concurrently with a locked body, so `this`-state cannot tell the two apart.
618
716
  */
619
- async queryStampId(table, keyValue) {
717
+ async queryStampId(table, keyValue, retry = true) {
620
718
  this.ensureInitialized();
621
719
  const sql = `select StampId from CadreControl.${table} where ${GUARDED_KEY_COLUMN[table]} = ?`;
622
- for await (const row of this.db.eval(sql, [keyValue])) {
720
+ for (const row of await this.readRows(sql, [keyValue], `stamp-${table}`, retry)) {
623
721
  return row.StampId ?? null;
624
722
  }
625
723
  return null;
@@ -664,11 +762,14 @@ export class ControlDatabase {
664
762
  * NOTE: re-reads the whole retired set on every call, and the caller runs per inbound
665
763
  * gate request while the table only ever grows. Cheap today (a cadre removes peers
666
764
  * rarely); if removals ever become routine, cache the set and invalidate it on write.
765
+ *
766
+ * `retry: false` is forwarded by {@link queryCadrePeers} for the one caller that reads
767
+ * under the write lock — see that method's note.
667
768
  */
668
- async queryRevokedStamps(tableName) {
769
+ async queryRevokedStamps(tableName, retry = true) {
669
770
  this.ensureInitialized();
670
771
  const stamps = new Set();
671
- for await (const row of this.db.eval('select StampId from CadreControl.Revocation where TableName = ?', [tableName])) {
772
+ for (const row of await this.readRows('select StampId from CadreControl.Revocation where TableName = ?', [tableName], 'revoked-stamps', retry)) {
672
773
  stamps.add(row.StampId);
673
774
  }
674
775
  return stamps;
@@ -684,7 +785,7 @@ export class ControlDatabase {
684
785
  async queryRevocations() {
685
786
  this.ensureInitialized();
686
787
  const rows = [];
687
- for await (const row of this.db.eval('select TableName, RowKey, StampId, ReissuedAt from CadreControl.Revocation')) {
788
+ for (const row of await this.readRows('select TableName, RowKey, StampId, ReissuedAt from CadreControl.Revocation', undefined, 'revocations')) {
688
789
  rows.push({
689
790
  tableName: row.TableName,
690
791
  rowKey: row.RowKey,
@@ -705,7 +806,7 @@ export class ControlDatabase {
705
806
  * {@link queryRevokedStamps} rather than inlining the SQL, so tests can interpose on
706
807
  * that seam (see {@link queryCadrePeers}).
707
808
  *
708
- * Missing/legacy column values are coalesced
809
+ * Missing/null column values are coalesced
709
810
  * to their empty form (`''` key/sig, `[]` addrs, `0` stamp) so the caller's
710
811
  * verify/freshness gates uniformly reject an unpublished or malformed row.
711
812
  * The split `addrs` re-join to the exact stored `Multiaddr` (split-on-`,` is
@@ -715,7 +816,7 @@ export class ControlDatabase {
715
816
  async queryPeerRecord(peerId) {
716
817
  this.ensureInitialized();
717
818
  const revoked = await this.queryRevokedStamps('CadrePeer');
718
- for await (const row of this.db.eval('select PeerId, PublicKey, Multiaddr, UpdatedAt, Sig, StampId from CadreControl.CadrePeer where PeerId = ?', [peerId])) {
819
+ for (const row of await this.readRows('select PeerId, PublicKey, Multiaddr, UpdatedAt, Sig, StampId from CadreControl.CadrePeer where PeerId = ?', [peerId], 'peer-record')) {
719
820
  const stampId = row.StampId ?? null;
720
821
  if (stampId !== null && revoked.has(stampId)) {
721
822
  return null;
@@ -762,13 +863,13 @@ export class ControlDatabase {
762
863
  with context OwnerKey = null, Signature = ?
763
864
  set Multiaddr = ?, UpdatedAt = ?, Sig = ?
764
865
  where PeerId = ?
765
- `, [record.sig, multiaddr, record.updatedAt, record.sig, record.peerId]);
866
+ `, [record.sig, multiaddr, record.updatedAt, record.sig, record.peerId], 'self-record-update');
766
867
  log('Self peer record updated: %s (updatedAt=%d)', record.peerId, record.updatedAt);
767
868
  }
768
869
  /**
769
870
  * Read a single peer's device push token (the full `DeviceToken` row) by PeerId.
770
871
  *
771
- * Returns null when no row exists. Missing/legacy column values are coalesced to
872
+ * Returns null when no row exists. Missing/null column values are coalesced to
772
873
  * their empty form (`''` token/sig, `0` stamp) so the caller's verify/freshness
773
874
  * gates uniformly reject an unpublished or malformed row. `platform` is returned
774
875
  * verbatim (the resolver validates it against {@link PushPlatform} and re-verifies
@@ -780,7 +881,7 @@ export class ControlDatabase {
780
881
  */
781
882
  async queryDeviceToken(peerId) {
782
883
  this.ensureInitialized();
783
- for await (const row of this.db.eval('select PeerId, Platform, Token, UpdatedAt, Sig, StampId from CadreControl.DeviceToken where PeerId = ?', [peerId])) {
884
+ for (const row of await this.readRows('select PeerId, Platform, Token, UpdatedAt, Sig, StampId from CadreControl.DeviceToken where PeerId = ?', [peerId], 'device-token')) {
784
885
  return {
785
886
  peerId: row.PeerId,
786
887
  platform: (row.Platform ?? ''),
@@ -810,7 +911,7 @@ export class ControlDatabase {
810
911
  with context OwnerKey = null, Signature = ?
811
912
  set Platform = ?, Token = ?, UpdatedAt = ?, Sig = ?
812
913
  where PeerId = ?
813
- `, [record.sig, record.platform, record.token, record.updatedAt, record.sig, record.peerId]);
914
+ `, [record.sig, record.platform, record.token, record.updatedAt, record.sig, record.peerId], 'device-token-update');
814
915
  log('Self device token updated: %s (platform=%s, updatedAt=%d)', record.peerId, record.platform, record.updatedAt);
815
916
  }
816
917
  /**
@@ -832,7 +933,7 @@ export class ControlDatabase {
832
933
  insert into CadreControl.OwnerKey (Key, StampId)
833
934
  with context OwnerKey = null, Signature = null
834
935
  values (?, ?)
835
- `, [key, stampId]);
936
+ `, [key, stampId], 'owner-key-insert');
836
937
  log('Owner key inserted');
837
938
  }
838
939
  /**
@@ -865,7 +966,7 @@ export class ControlDatabase {
865
966
  insert into CadreControl.Strand (Id, Type, MemberPrivateKey, StampId)
866
967
  with context OwnerKey = ?, Signature = ?
867
968
  values (?, ?, ?, ?)
868
- `, [ownerKey, signature, strandId, type, memberPrivateKey ?? null, stampId]);
969
+ `, [ownerKey, signature, strandId, type, memberPrivateKey ?? null, stampId], 'strand-insert');
869
970
  log('Strand inserted: %s', strandId);
870
971
  }
871
972
  /**
@@ -887,7 +988,7 @@ export class ControlDatabase {
887
988
  * when a row was actually removed.
888
989
  */
889
990
  deleteStrand(strandId, ownerKey, signMessage) {
890
- return this.lockedWithRetry(() => this.deleteGuardedRow('Strand', strandId, ownerKey, signMessage));
991
+ return this.lockedWithRetry(() => this.deleteGuardedRow('Strand', strandId, ownerKey, signMessage), {}, 'strand-delete');
891
992
  }
892
993
  /**
893
994
  * Insert a validation key into the control database using an owner signature.
@@ -914,7 +1015,7 @@ export class ControlDatabase {
914
1015
  insert into CadreControl.ValidationKey (Key, StampId)
915
1016
  with context OwnerKey = ?, Signature = ?
916
1017
  values (?, ?)
917
- `, [ownerKey, signature, key, stampId]);
1018
+ `, [ownerKey, signature, key, stampId], 'validation-key-insert');
918
1019
  log('Validation key inserted: %s', key);
919
1020
  }
920
1021
  /**
@@ -934,7 +1035,7 @@ export class ControlDatabase {
934
1035
  * when a row was actually removed.
935
1036
  */
936
1037
  deleteValidationKey(key, ownerKey, signMessage) {
937
- return this.lockedWithRetry(() => this.deleteGuardedRow('ValidationKey', key, ownerKey, signMessage));
1038
+ return this.lockedWithRetry(() => this.deleteGuardedRow('ValidationKey', key, ownerKey, signMessage), {}, 'validation-key-delete');
938
1039
  }
939
1040
  /**
940
1041
  * Owner-vouched INSERT of one `CadrePeer` membership row, notifying the membership
@@ -980,7 +1081,8 @@ export class ControlDatabase {
980
1081
  const stampId = generateStampId(row.peerId);
981
1082
  const signature = signMessage(buildAuthorizationMessage('CadreControl.CadrePeer', 'vouch', [row.peerId, stampId]));
982
1083
  return await this.mutateCadrePeer('peer-insert', async () => {
983
- if (await this.queryStampId('CadrePeer', row.peerId) !== null) {
1084
+ // retry: false — this guard runs inside the locked write body (see queryStampId's NOTE).
1085
+ if (await this.queryStampId('CadrePeer', row.peerId, false) !== null) {
984
1086
  log('CadrePeer row already present for %s; insert skipped (already a member)', row.peerId);
985
1087
  return false;
986
1088
  }
@@ -1102,7 +1204,7 @@ export class ControlDatabase {
1102
1204
  * when a row was actually removed.
1103
1205
  */
1104
1206
  deleteDeviceToken(peerId, ownerKey, signMessage) {
1105
- return this.lockedWithRetry(() => this.deleteGuardedRow('DeviceToken', peerId, ownerKey, signMessage));
1207
+ return this.lockedWithRetry(() => this.deleteGuardedRow('DeviceToken', peerId, ownerKey, signMessage), {}, 'device-token-delete');
1106
1208
  }
1107
1209
  /**
1108
1210
  * Owner-signed delete of one guarded row plus the `Revocation` tombstone retiring its
@@ -1139,7 +1241,8 @@ export class ControlDatabase {
1139
1241
  */
1140
1242
  async deleteGuardedRow(table, keyValue, ownerKey, signMessage) {
1141
1243
  this.ensureInitialized();
1142
- const stampId = await this.queryStampId(table, keyValue);
1244
+ // retry: false — this runs inside the locked write body (see queryStampId's NOTE).
1245
+ const stampId = await this.queryStampId(table, keyValue, false);
1143
1246
  if (stampId === null) {
1144
1247
  log('delete %s: no row for %s (already absent)', table, keyValue);
1145
1248
  return false;
@@ -1259,7 +1362,7 @@ export class ControlDatabase {
1259
1362
  await this.mutateCadrePeer('peer-reap', () => this.db.exec(sql, [rowKey, stampId]));
1260
1363
  }
1261
1364
  else {
1262
- await this.execWrite(sql, [rowKey, stampId]);
1365
+ await this.execWrite(sql, [rowKey, stampId], `reap-${table}`);
1263
1366
  }
1264
1367
  // A row leaving the control plane without an owner signature at the write site
1265
1368
  // should be visible in a log.
@@ -1411,7 +1514,7 @@ export class ControlDatabase {
1411
1514
  });
1412
1515
  log('Reissued %d revocation tombstone(s) at %d', executed, reissuedAt);
1413
1516
  return executed;
1414
- });
1517
+ }, {}, 'revocation-reissue');
1415
1518
  }
1416
1519
  /**
1417
1520
  * Run `body` between `beginTransaction` and `commit`, rolling back on failure.
@@ -1514,7 +1617,7 @@ export class ControlDatabase {
1514
1617
  this.assertCommitBoundary(reason, 'on return from');
1515
1618
  await this.notifyMembershipChanged(reason);
1516
1619
  return result;
1517
- });
1620
+ }, {}, reason);
1518
1621
  }
1519
1622
  /**
1520
1623
  * Run one local write under the database-wide write lock, serializing it against
@@ -1526,7 +1629,8 @@ export class ControlDatabase {
1526
1629
  * {@link assertCommitBoundary} (any `CadrePeer` path) or silently join each other's
1527
1630
  * open transaction — a torn commit if either side rolls back. This lock closes that
1528
1631
  * class: every public write method of this class runs its statement(s) under it, and
1529
- * `SeedBootstrapService` wraps its direct `db.exec` writes in it too. The race is
1632
+ * `SeedBootstrapService` reaches it through {@link execWrite} rather than touching the
1633
+ * `Database` directly, so it is covered too. The race is
1530
1634
  * real, not theoretical: the background self-record publish on a control connection
1531
1635
  * opening (`CadreNode.drainPendingControlReplication` → `registerSelf`) collided with
1532
1636
  * a foreground `authorizePeer` — both `CadrePeer` inserts — and tripped the boundary
@@ -1569,10 +1673,8 @@ export class ControlDatabase {
1569
1673
  *
1570
1674
  * The retry wraps the LOCK, it does not live inside it: each attempt takes and
1571
1675
  * releases the lock, and the backoff sleeps with NO lock held, so a write parked in
1572
- * backoff never stalls the other local writers queued behind it. Same reason
1573
- * {@link withUseNumberRetry} re-takes the lock per attempt — see the note on
1574
- * `USE_NUMBER_ATTEMPTS`. Safe because every locked body is atomic and re-runnable
1575
- * (the contract on {@link withWriteLock}).
1676
+ * backoff never stalls the other local writers queued behind it. Safe because every
1677
+ * locked body is atomic and re-runnable (the contract on {@link withWriteLock}).
1576
1678
  *
1577
1679
  * Also carries {@link loadSchema}'s distributed DDL, the one caller that runs BEFORE
1578
1680
  * `initialized` is set — so it reaches this method directly rather than through
@@ -1581,9 +1683,16 @@ export class ControlDatabase {
1581
1683
  * rather than carving out a lockless path. It is also the ONLY caller that passes a
1582
1684
  * `policy` — everything else takes the default, whose classifier and attempt count are
1583
1685
  * unchanged by that call site existing.
1686
+ *
1687
+ * `label` names the operation in this loop's debug lines and nothing else
1688
+ * ({@link ControlWriteRetryOptions.label}). Every call site in this class supplies one:
1689
+ * several writes retry CONCURRENTLY in a real party, so an unlabelled line cannot be
1690
+ * attributed to a write — keep new call sites labelled.
1584
1691
  */
1585
- lockedWithRetry(fn, policy = {}) {
1586
- return retryControlWrite(() => this.withWriteLock(fn), { ...policy, ...this.controlWriteRetryPacing });
1692
+ lockedWithRetry(fn, policy = {}, label) {
1693
+ // The label lands LAST so a call site's own name survives both the policy and the
1694
+ // spec-injected pacing; it changes log attribution only, never behaviour.
1695
+ return retryControlWrite(() => this.withWriteLock(fn), { ...policy, ...this.controlWriteRetryPacing, ...(label !== undefined ? { label } : {}) });
1587
1696
  }
1588
1697
  /**
1589
1698
  * One-statement local write, serialized by {@link withWriteLock} and retried on a
@@ -1594,10 +1703,13 @@ export class ControlDatabase {
1594
1703
  * Public so `SeedBootstrapService`'s direct `CadrePeer`/`DeviceToken` SQL writes go
1595
1704
  * through the same seam. Must NOT be called from inside an already-locked body (the
1596
1705
  * lock is not re-entrant); such a caller uses a bare `getDatabase().exec` instead.
1706
+ *
1707
+ * `label` names the write in the retry loop's debug lines (see {@link lockedWithRetry});
1708
+ * optional only because it is log-only, but every caller in this repo passes one.
1597
1709
  */
1598
- execWrite(sql, params) {
1710
+ execWrite(sql, params, label) {
1599
1711
  this.ensureInitialized();
1600
- return this.lockedWithRetry(() => this.db.exec(sql, params));
1712
+ return this.lockedWithRetry(() => this.db.exec(sql, params), {}, label);
1601
1713
  }
1602
1714
  /**
1603
1715
  * Throw unless the database sits at a committed boundary — `getAutocommit()` is false
@@ -1666,7 +1778,8 @@ export class ControlDatabase {
1666
1778
  * `StrandId` binds the invite to a pre-existing host strand (provision-then-record):
1667
1779
  * when set, a responder redeeming this token records a `FormationUsage` against that
1668
1780
  * strand and returns it (see {@link ControlFormationUsageRecorder.resolveStrand}); a
1669
- * null `StrandId` (the default) leaves the legacy responder-provisions path in place.
1781
+ * null `StrandId` (the default) takes the unbound responder-provisions path: the
1782
+ * responder provisions a fresh open strand and atomically records its one consent row.
1670
1783
  * Like ValidationUrl it is a nullable bound field, signed as `''` when absent.
1671
1784
  *
1672
1785
  * The ExpiresAt and TotalUses message fields must byte-match what the (auto-deferred,
@@ -1723,7 +1836,7 @@ export class ControlDatabase {
1723
1836
  options.validationUrl ?? null,
1724
1837
  options.strandId ?? null,
1725
1838
  stampId,
1726
- ]);
1839
+ ], 'formation-invite-insert');
1727
1840
  log('Formation invite inserted: %s', token);
1728
1841
  }
1729
1842
  /**
@@ -1751,50 +1864,60 @@ export class ControlDatabase {
1751
1864
  * takes an owner re-seat ({@link insertStrand}) plus a bound invite, never another
1752
1865
  * redemption.
1753
1866
  *
1754
- * `UseNumber` is computed as `max(UseNumber)+1` for the token (the `Monotonic`
1755
- * constraint), inside the write lock and re-read per attempt by
1756
- * {@link withUseNumberRetry} — so concurrent redemptions of one token no longer need
1757
- * the caller to serialise them, and a use number lost to another writer is retried here
1758
- * with the caller's parameters passed through UNCHANGED. That is what makes the retry
1759
- * safe: the approval is bound to the nonce, not to the use number, so re-presenting it
1760
- * cannot drift, and no second trip through the approval hook is possible from in here.
1867
+ * The usage row is keyed by the joiner's own nonce (`UsageStampId` is the primary key), so
1868
+ * concurrent redemptions of one token never contend for a shared row key — there is no
1869
+ * lost race to retry and no second trip through the approval hook. The invite's seat
1870
+ * budget is checked by COUNT ({@link assertSeatRemains}) inside the write lock, ahead of
1871
+ * the write, so a spent invite is refused by name (`InvitationExhaustedError`) instead of
1872
+ * as the schema cap clause's generic `Authorized` CHECK failure.
1761
1873
  */
1762
1874
  async redeemInvitation(params) {
1763
1875
  this.ensureInitialized();
1764
1876
  const { token, strandId, disclosure = '', peerKey, peerSignature, usageStampId, nowMs, validationKey, validationSignature, signal, totalUses, } = params;
1765
1877
  log('Redeeming invitation %s -> strand %s', token, strandId);
1766
1878
  const localPeerId = this.config.libp2pNode.peerId.toString();
1767
- // Minted ONCE, outside the loop: a failed attempt rolls its whole transaction back, so
1768
- // nothing was ever persisted under this stamp, and it is deliberately not part of the
1769
- // approver's signed digest. Re-minting per attempt would only churn.
1879
+ // A failed attempt rolls its whole transaction back, so nothing was ever persisted under
1880
+ // this stamp, and it is deliberately not part of the approver's signed digest.
1770
1881
  const strandStampId = generateStampId(localPeerId);
1771
- const useNumber = await this.withUseNumberRetry(token, 'redemption', signal, totalUses, attemptUseNumber => this.inTransaction('redemption', async () => {
1772
- // 1. Strand row — authorised by the FormationUsage branch (no owner sig),
1773
- // still carrying a fresh unique StampId for the anti-replay column.
1774
- // Hard-coded open + keyless: the consent branch admits no other shape.
1775
- await this.db.exec(`
1882
+ // Ordering inside the locked body is contractual: the abort check, then the seat check,
1883
+ // then the write — FormationAbortedError may only ever be thrown before the write is
1884
+ // issued, and a transient-failure retry (lockedWithRetry re-runs the whole body) must
1885
+ // re-check both before re-presenting the write.
1886
+ await this.lockedWithRetry(async () => {
1887
+ // Inside the lock: a write still parked behind another writer when the caller's
1888
+ // budget expired is abandoned rather than executed.
1889
+ if (signal?.aborted) {
1890
+ throw new FormationAbortedError(token, 'redemption');
1891
+ }
1892
+ await this.assertSeatRemains(token, totalUses);
1893
+ await this.inTransaction('redemption', async () => {
1894
+ // 1. Strand row — authorised by the FormationUsage branch (no owner sig),
1895
+ // still carrying a fresh unique StampId for the anti-replay column.
1896
+ // Hard-coded open + keyless: the consent branch admits no other shape.
1897
+ await this.db.exec(`
1776
1898
  insert into CadreControl.Strand (Id, Type, MemberPrivateKey, StampId)
1777
1899
  with context OwnerKey = null, Signature = null
1778
1900
  values (?, 'o', null, ?)
1779
1901
  `, [strandId, strandStampId]);
1780
- // 2. FormationUsage row — authorised by the matching FormationInvite, and
1781
- // carrying the strand's stamp so it authorizes THIS row and no other.
1782
- await this.execFormationUsageInsert({
1783
- token, useNumber: attemptUseNumber, usageStampId, disclosure, strandId, strandStampId,
1784
- peerKey, peerSignature, nowMs: nowMs ?? Date.now(),
1785
- validationKey: validationKey ?? null, validationSignature: validationSignature ?? null,
1902
+ // 2. FormationUsage row — authorised by the matching FormationInvite, and
1903
+ // carrying the strand's stamp so it authorizes THIS row and no other.
1904
+ await this.execFormationUsageInsert({
1905
+ token, usageStampId, disclosure, strandId, strandStampId,
1906
+ peerKey, peerSignature, nowMs: nowMs ?? Date.now(),
1907
+ validationKey: validationKey ?? null, validationSignature: validationSignature ?? null,
1908
+ });
1786
1909
  });
1787
- }));
1788
- log('Redeemed invitation %s -> strand %s (use #%d)', token, strandId, useNumber);
1789
- return { useNumber, usageStampId };
1910
+ }, {}, 'formation-usage');
1911
+ log('Redeemed invitation %s -> strand %s (stamp %s)', token, strandId, usageStampId);
1912
+ return { usageStampId };
1790
1913
  }
1791
1914
  /**
1792
1915
  * Record a `FormationUsage` against an **already-existing** `Strand` (no strand
1793
1916
  * insert). This is the redemption path when the strand was provisioned
1794
1917
  * separately (e.g. owner-signed) and the consent record is added after the
1795
1918
  * fact: the single insert auto-commits, and the deferred `StrandExists` CHECK
1796
- * is satisfied by the pre-existing committed strand row. Returns the assigned
1797
- * `UseNumber`.
1919
+ * is satisfied by the pre-existing committed strand row. Echoes back the
1920
+ * redemption's `usageStampId` (the row's primary key).
1798
1921
  *
1799
1922
  * Use {@link redeemInvitation} instead when the strand must be created by
1800
1923
  * consent atomically with the usage.
@@ -1808,108 +1931,47 @@ export class ControlDatabase {
1808
1931
  * by {@link ControlFormationUsageRecorder.resolveStrand}, so an absent row at this
1809
1932
  * point is a genuine race and deserves a named error, not a silent rollback.
1810
1933
  *
1811
- * `UseNumber` is read inside the write lock and re-read per attempt by
1812
- * {@link withUseNumberRetry}: this is the path production actually races on (a bound
1813
- * invite published by `cadre-web` / `cadre-phone` is always record-only), and a use number
1814
- * lost to another writer is retried here with the caller's parameters — nonce, approver
1815
- * sign-off, strand, peer key, disclosure — passed through UNCHANGED, so a granted approval
1816
- * is never thrown away for a second trip through the approval hook.
1934
+ * The row is keyed by the joiner's own nonce (`UsageStampId` is the primary key), so this
1935
+ * path — the one production actually races on (a bound invite published by `cadre-web` /
1936
+ * `cadre-phone` is always record-only) — never contends with another redemption for a
1937
+ * shared row key: there is no lost race to retry and no second trip through the approval
1938
+ * hook. The invite's seat budget is checked by COUNT ({@link assertSeatRemains}) inside
1939
+ * the write lock, ahead of the write.
1817
1940
  */
1818
1941
  async recordFormationUsage(params) {
1819
1942
  this.ensureInitialized();
1820
1943
  const { token, strandId, disclosure = '', peerKey, peerSignature, usageStampId, nowMs, validationKey, validationSignature, signal, totalUses, } = params;
1821
- // Read ONCE, outside the loop: the host strand is pre-existing and owner-signed, and a
1822
- // retry changes which use number we take, never which strand row we name.
1944
+ // The host strand is pre-existing and owner-signed; its stamp names the strand ROW the
1945
+ // consent record authorizes.
1823
1946
  const strandStampId = await this.queryStrandStampId(strandId);
1824
1947
  if (strandStampId === null) {
1825
1948
  throw new MissingHostStrandError(strandId, token);
1826
1949
  }
1827
- const useNumber = await this.withUseNumberRetry(token, 'usage recording', signal, totalUses, attemptUseNumber => this.execFormationUsageInsert({
1828
- token, useNumber: attemptUseNumber, usageStampId, disclosure, strandId, strandStampId,
1829
- peerKey, peerSignature, nowMs: nowMs ?? Date.now(),
1830
- validationKey: validationKey ?? null, validationSignature: validationSignature ?? null,
1831
- }));
1832
- log('Recorded formation usage: token=%s strand=%s (use #%d)', token, strandId, useNumber);
1833
- return { useNumber, usageStampId };
1834
- }
1835
- /**
1836
- * Assign a `UseNumber` and run `write` under it, re-picking and retrying when another
1837
- * writer took that number first.
1838
- *
1839
- * Each attempt takes {@link withWriteLock} and, inside it, reads {@link nextUseNumber}
1840
- * afresh. Reading under the lock is the whole fix for the common case: two redemptions of
1841
- * one token on the SAME node are serialized by the local write queue, so each computes a
1842
- * number the other has already committed and neither collides. The retry covers the writers
1843
- * this lock does not reach — another node of the cadre, another `Database` handle over the
1844
- * same store — which can still take the number between our read and our commit.
1845
- *
1846
- * `write` receives ONLY the use number; every other parameter of the redemption (the nonce,
1847
- * the approver's signature, the strand, the joining peer key, the disclosure) is closed over
1848
- * by the caller and passed through untouched. So by construction a retry re-presents the
1849
- * IDENTICAL approval and the four non-nonce fields inside the approver's signed digest
1850
- * cannot drift. No second call to the approval hook is reachable from in here either:
1851
- * `obtainApproval` lives a layer up in `ControlFormationUsageRecorder` and is never
1852
- * re-entered.
1853
- *
1854
- * `write` must be a BARE private body ({@link inTransaction} /
1855
- * {@link execFormationUsageInsert}) — {@link withWriteLock} is not re-entrant and a `write`
1856
- * that called another locked public method would hang silently and strand the write queue.
1857
- *
1858
- * A failed attempt has written nothing (a bare insert never landed; a transactional one
1859
- * rolled back), so the abort check at the top of each attempt stays inside
1860
- * {@link FormationAbortedError}'s contract — it is only ever thrown before this attempt's
1861
- * write is issued.
1862
- *
1863
- * @param operation - `'redemption'` | `'usage recording'`, for the abort error's text.
1864
- * @param knownTotalUses - The invite's seat budget when the caller already has it in hand
1865
- * (`undefined` to fall back to a fresh {@link queryFormationInvite} read; `null` for "no
1866
- * invite row" / unlimited). Passed through to {@link assertSeatRemains} on every attempt.
1867
- * @returns The `UseNumber` the successful attempt wrote under.
1868
- */
1869
- async withUseNumberRetry(token, operation, signal, knownTotalUses, write) {
1870
- let lastError;
1871
- for (let attempt = 1; attempt <= USE_NUMBER_ATTEMPTS; attempt++) {
1872
- try {
1873
- // lockedWithRetry carries the transient-cluster retry INSIDE this use-number
1874
- // retry. The loops cannot multiply: a lost use number is a constraint failure,
1875
- // which `isRetriableControlWriteFailure` never matches, and a transient cluster
1876
- // failure is not a lost use number, so exactly one loop claims any given error.
1877
- return await this.lockedWithRetry(async () => {
1878
- // Inside the lock: a write still parked behind another writer when the caller's
1879
- // budget expired is abandoned rather than executed.
1880
- if (signal?.aborted) {
1881
- throw new FormationAbortedError(token, operation);
1882
- }
1883
- const useNumber = await this.nextUseNumber(token);
1884
- await this.assertSeatRemains(token, useNumber, knownTotalUses);
1885
- await write(useNumber);
1886
- return useNumber;
1887
- });
1950
+ // Ordering inside the locked body is contractual — abort check, seat check, write — see
1951
+ // the matching note in redeemInvitation.
1952
+ await this.lockedWithRetry(async () => {
1953
+ if (signal?.aborted) {
1954
+ throw new FormationAbortedError(token, 'usage recording');
1888
1955
  }
1889
- catch (error) {
1890
- if (!isLostUseNumberRace(error)) {
1891
- throw error;
1892
- }
1893
- lastError = error;
1894
- log('Use number for token %s was taken by another writer (attempt %d/%d): %s', token, attempt, USE_NUMBER_ATTEMPTS, error);
1895
- }
1896
- }
1897
- // Out of attempts under sustained contention. Rethrowing the engine's own error keeps
1898
- // today's behaviour: `StrandFormationManager.provisionAsResponder`'s catch-all maps it to
1899
- // `Formation conflict, retry` and the joiner starts over with a fresh nonce.
1900
- throw lastError;
1956
+ await this.assertSeatRemains(token, totalUses);
1957
+ await this.execFormationUsageInsert({
1958
+ token, usageStampId, disclosure, strandId, strandStampId,
1959
+ peerKey, peerSignature, nowMs: nowMs ?? Date.now(),
1960
+ validationKey: validationKey ?? null, validationSignature: validationSignature ?? null,
1961
+ });
1962
+ }, {}, 'formation-usage');
1963
+ log('Recorded formation usage: token=%s strand=%s (stamp %s)', token, strandId, usageStampId);
1964
+ return { usageStampId };
1901
1965
  }
1902
1966
  /**
1903
- * Refuse an attempt that would consume a seat the invite does not have.
1967
+ * Refuse a redemption that would consume a seat the invite does not have.
1904
1968
  *
1905
- * `FormationUsage.Authorized` already enforces `FI.TotalUses is null or FI.TotalUses >=
1906
- * new.UseNumber`, so an over-limit write fails at the database anyway — but it fails as a
1907
- * generic `CHECK constraint failed: Authorized`, which the manager reports as a retryable
1908
- * conflict. Catching it here as {@link InvitationExhaustedError} lets the joiner be told the
1909
- * invitation is spent instead of being sent around a loop that can never close. Run on EVERY
1910
- * attempt, including the first: two redemptions racing on the SAME node serialize behind the
1911
- * local write queue, so the loser's first attempt is already over budget, with no retry
1912
- * involved.
1969
+ * The schema's count-based cap clause (`FormationUsage.Authorized`) refuses an over-limit
1970
+ * write at the database anyway — but it fails as a generic `CHECK constraint failed:
1971
+ * Authorized`, which the manager reports as a retryable conflict. Catching it here as
1972
+ * {@link InvitationExhaustedError} lets the joiner be told the invitation is spent instead
1973
+ * of being sent around a loop that can never close. Runs inside the write lock, so on a
1974
+ * same-node race the loser reads the winner's committed row and is refused by name.
1913
1975
  *
1914
1976
  * `knownTotalUses` lets a caller that already read the `FormationInvite` row (both
1915
1977
  * production callers in `ControlFormationUsageRecorder` do) skip a second read on the common,
@@ -1919,9 +1981,9 @@ export class ControlDatabase {
1919
1981
  * invite, before an outbound approval call). Safe today because `FormationInvite` is
1920
1982
  * insert/delete only (its `Immutable` constraint), so the only way to stale it is an owner
1921
1983
  * revoking the token and re-issuing it with MORE seats mid-redemption; if invites ever gain
1922
- * an update path, drop the parameter and read here on every attempt instead.
1984
+ * an update path, drop the parameter and read here instead.
1923
1985
  */
1924
- async assertSeatRemains(token, useNumber, knownTotalUses) {
1986
+ async assertSeatRemains(token, knownTotalUses) {
1925
1987
  // A missing invite is left to `Authorized` — it is not an exhaustion, and the CHECK's
1926
1988
  // rejection is already the right, non-retryable answer for it. That fallback is also what
1927
1989
  // makes this read's failure mode safe: it is a full-primary-key point lookup
@@ -1929,11 +1991,19 @@ export class ControlDatabase {
1929
1991
  // is still open (tracked by `debt-composite-pk-point-lookup-unreliable-untracked`). A
1930
1992
  // spurious empty result only costs the attempt its NAMED exhaustion error, reverting it to
1931
1993
  // today's generic `Authorized` refusal — never a seat the invite does not have.
1994
+ // Both reads pass retry: false — assertSeatRemains only ever runs inside the locked
1995
+ // write bodies of redeemInvitation / recordFormationUsage, where a read retrying its
1996
+ // own backoff would sleep holding the write lock and the write funnel re-runs these
1997
+ // reads anyway (see the NOTE on readRows).
1932
1998
  const totalUses = knownTotalUses !== undefined
1933
1999
  ? knownTotalUses
1934
- : (await this.queryFormationInvite(token))?.totalUses ?? null;
1935
- if (totalUses != null && useNumber > totalUses) {
1936
- throw new InvitationExhaustedError(token, useNumber, totalUses);
2000
+ : (await this.queryFormationInvite(token, false))?.totalUses ?? null;
2001
+ if (totalUses == null) {
2002
+ return;
2003
+ }
2004
+ const used = await this.countFormationUsage(token, false);
2005
+ if (used >= totalUses) {
2006
+ throw new InvitationExhaustedError(token, used, totalUses);
1937
2007
  }
1938
2008
  }
1939
2009
  /** Parameterised `FormationUsage` insert shared by redeem + record paths. */
@@ -1948,12 +2018,12 @@ export class ControlDatabase {
1948
2018
  // not a fix for an observable mis-ordering.
1949
2019
  const nowCanonical = await canonicalDatetime(this.db, opts.nowMs);
1950
2020
  await this.db.exec(`
1951
- insert into CadreControl.FormationUsage (Token, UseNumber, UsageStampId, PeerKey, PeerSig, Disclosure, StrandId, StrandStampId)
2021
+ insert into CadreControl.FormationUsage (Token, UsageStampId, PeerKey, PeerSig, Disclosure, StrandId, StrandStampId)
1952
2022
  with context Now = ?, ValidationKey = ?, ValidationSignature = ?
1953
- values (?, ?, ?, ?, ?, ?, ?, ?)
2023
+ values (?, ?, ?, ?, ?, ?, ?)
1954
2024
  `, [
1955
2025
  nowCanonical, opts.validationKey, opts.validationSignature,
1956
- opts.token, opts.useNumber, opts.usageStampId, opts.peerKey,
2026
+ opts.token, opts.usageStampId, opts.peerKey,
1957
2027
  opts.peerSignature, opts.disclosure, opts.strandId, opts.strandStampId,
1958
2028
  ]);
1959
2029
  }
@@ -1961,10 +2031,15 @@ export class ControlDatabase {
1961
2031
  * Read a `FormationInvite` row by token, or null when absent. `expiresAtMs` is
1962
2032
  * the parsed epoch-ms of the stored `datetime` (null when the invite never
1963
2033
  * expires); the caller compares it against the wall clock for freshness.
2034
+ *
2035
+ * `retry: false` is passed only by {@link assertSeatRemains}, which runs INSIDE a
2036
+ * locked write body — same per-call opt-out, and for the same reason, as
2037
+ * {@link queryStampId}'s. Every unlocked caller keeps the default retried path.
1964
2038
  */
1965
- async queryFormationInvite(token) {
2039
+ async queryFormationInvite(token, retry = true) {
1966
2040
  this.ensureInitialized();
1967
- for await (const row of this.db.eval('select Token, sAppId, ExpiresAt, TotalUses, ValidationUrl, StrandId from CadreControl.FormationInvite where Token = ?', [token])) {
2041
+ const sql = 'select Token, sAppId, ExpiresAt, TotalUses, ValidationUrl, StrandId from CadreControl.FormationInvite where Token = ?';
2042
+ for (const row of await this.readRows(sql, [token], 'formation-invite', retry)) {
1968
2043
  return {
1969
2044
  token: row.Token,
1970
2045
  sAppId: row.sAppId,
@@ -1978,10 +2053,32 @@ export class ControlDatabase {
1978
2053
  }
1979
2054
  /**
1980
2055
  * Count `FormationUsage` rows recorded against a token (uses consumed so far).
2056
+ *
2057
+ * This count IS the seat cap (`enforceFormationUseCap`), so an under-report admits a seat
2058
+ * the invitation never paid for. It is served by a table SCAN, deliberately: a bare
2059
+ * equality on a non-key column is a shape the optimystic vtab declines to claim, so the
2060
+ * engine applies the filter itself over a full read of the table.
2061
+ *
2062
+ * NOTE: it was briefly an index seek, and that is what the scan is protecting against.
2063
+ * `formation-unique-token-redesign` took `Token` out of the primary key and declared
2064
+ * `FormationUsageByToken` purely to spare this scan, recording as a tripwire that an
2065
+ * under-reporting descent would admit an unpaid-for seat — "a failure mode a scan could
2066
+ * not produce". The tripwire fired. Across machines a secondary-index descent returns only
2067
+ * the rows the READING machine wrote, so this count under-reported without bound on a
2068
+ * multi-machine party and the cap over-admitted. The index was removed 2026-08-25; the
2069
+ * measurement is recorded in the block comment at the `FormationUsage` declaration in
2070
+ * `schemas/control.qsql`, and the engine defect in
2071
+ * `blocked/secondary-index-seek-blind-to-sibling-rows`. Do NOT re-declare the index to
2072
+ * speed this up while that ticket is open.
2073
+ *
2074
+ * `retry: false` is passed only by {@link assertSeatRemains}, which runs INSIDE a
2075
+ * locked write body — same per-call opt-out, and for the same reason, as
2076
+ * {@link queryStampId}'s. Every unlocked caller keeps the default retried path.
1981
2077
  */
1982
- async countFormationUsage(token) {
2078
+ async countFormationUsage(token, retry = true) {
1983
2079
  this.ensureInitialized();
1984
- for await (const row of this.db.eval('select count(1) as Count from CadreControl.FormationUsage where Token = ?', [token])) {
2080
+ const sql = 'select count(1) as Count from CadreControl.FormationUsage where Token = ?';
2081
+ for (const row of await this.readRows(sql, [token], 'formation-usage-count', retry)) {
1985
2082
  return row.Count ?? 0;
1986
2083
  }
1987
2084
  return 0;
@@ -2012,7 +2109,7 @@ export class ControlDatabase {
2012
2109
  // prune redeemed/expired rows.
2013
2110
  const metered = [];
2014
2111
  let unlimitedOutstanding = false;
2015
- for await (const row of this.db.eval('select Token, ExpiresAt, TotalUses from CadreControl.FormationInvite')) {
2112
+ for (const row of await this.readRows('select Token, ExpiresAt, TotalUses from CadreControl.FormationInvite', undefined, 'outstanding-invites')) {
2016
2113
  const expiresAtMs = parseNullableStoredDatetimeMs(row.ExpiresAt);
2017
2114
  if (expiresAtMs !== null && expiresAtMs <= nowMs) {
2018
2115
  continue;
@@ -2028,6 +2125,12 @@ export class ControlDatabase {
2028
2125
  if (unlimitedOutstanding) {
2029
2126
  return true;
2030
2127
  }
2128
+ // NOTE: this loop issues one retried read per metered invite, each carrying its own
2129
+ // CONTROL_READ_RETRY_BUDGET_MS, on a path the inbound admission gate awaits under a
2130
+ // 2 s FAIL-OPEN deadline — so with N unexpired metered invites the worst case is
2131
+ // (1 + N) budgets, not one. Fine while a cadre holds a couple of live invites and
2132
+ // transient read failures are rare; if a party ever keeps many metered invites open,
2133
+ // give this whole method one shared deadline rather than raising the per-read budget.
2031
2134
  for (const invite of metered) {
2032
2135
  if (await this.countFormationUsage(invite.token) < invite.totalUses) {
2033
2136
  return true;
@@ -2035,19 +2138,6 @@ export class ControlDatabase {
2035
2138
  }
2036
2139
  return false;
2037
2140
  }
2038
- /**
2039
- * Next `UseNumber` for a token = max(existing)+1, per the `Monotonic` constraint.
2040
- *
2041
- * Called only from inside {@link withUseNumberRetry}'s locked body — a read takes no
2042
- * transaction of its own, so it is safe there, and reading it under the lock is what stops
2043
- * two local redemptions of one token from picking the same number.
2044
- */
2045
- async nextUseNumber(token) {
2046
- for await (const row of this.db.eval('select coalesce(max(UseNumber), 0) as MaxUse from CadreControl.FormationUsage where Token = ?', [token])) {
2047
- return row.MaxUse + 1;
2048
- }
2049
- return 1;
2050
- }
2051
2141
  /**
2052
2142
  * Close the database and cleanup resources.
2053
2143
  *