@serfab/cadre-core 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/README.md +255 -255
  2. package/dist/announce-addrs.d.ts +49 -0
  3. package/dist/announce-addrs.js +78 -0
  4. package/dist/announce-addrs.js.map +1 -0
  5. package/dist/cadre-node.d.ts +484 -58
  6. package/dist/cadre-node.js +953 -182
  7. package/dist/cadre-node.js.map +1 -1
  8. package/dist/control-cohort.d.ts +26 -0
  9. package/dist/control-cohort.js +26 -0
  10. package/dist/control-cohort.js.map +1 -1
  11. package/dist/control-database.d.ts +222 -125
  12. package/dist/control-database.js +356 -266
  13. package/dist/control-database.js.map +1 -1
  14. package/dist/control-formation-recorder.d.ts +8 -7
  15. package/dist/control-formation-recorder.js +8 -7
  16. package/dist/control-formation-recorder.js.map +1 -1
  17. package/dist/control-read-retry.d.ts +103 -0
  18. package/dist/control-read-retry.js +145 -0
  19. package/dist/control-read-retry.js.map +1 -0
  20. package/dist/control-retry.d.ts +101 -0
  21. package/dist/control-retry.js +108 -0
  22. package/dist/control-retry.js.map +1 -0
  23. package/dist/control-schema.d.ts +1 -1
  24. package/dist/control-schema.js +805 -774
  25. package/dist/control-schema.js.map +1 -1
  26. package/dist/control-write-retry.d.ts +65 -31
  27. package/dist/control-write-retry.js +33 -103
  28. package/dist/control-write-retry.js.map +1 -1
  29. package/dist/delegate-admission.d.ts +17 -15
  30. package/dist/delegate-admission.js +17 -15
  31. package/dist/delegate-admission.js.map +1 -1
  32. package/dist/index.d.ts +7 -5
  33. package/dist/index.js +18 -9
  34. package/dist/index.js.map +1 -1
  35. package/dist/membership-connection-gater.d.ts +211 -25
  36. package/dist/membership-connection-gater.js +309 -29
  37. package/dist/membership-connection-gater.js.map +1 -1
  38. package/dist/peer-addr-book.d.ts +92 -0
  39. package/dist/peer-addr-book.js +220 -0
  40. package/dist/peer-addr-book.js.map +1 -0
  41. package/dist/peer-record.d.ts +26 -0
  42. package/dist/peer-record.js +32 -0
  43. package/dist/peer-record.js.map +1 -1
  44. package/dist/relay-addrs.d.ts +76 -26
  45. package/dist/relay-addrs.js +115 -27
  46. package/dist/relay-addrs.js.map +1 -1
  47. package/dist/relay-reservation.d.ts +20 -15
  48. package/dist/relay-reservation.js +18 -14
  49. package/dist/relay-reservation.js.map +1 -1
  50. package/dist/seed-bootstrap.d.ts +7 -0
  51. package/dist/seed-bootstrap.js +15 -13
  52. package/dist/seed-bootstrap.js.map +1 -1
  53. package/dist/strand-addr-protocol.js +16 -0
  54. package/dist/strand-addr-protocol.js.map +1 -1
  55. package/dist/strand-backfill.js +13 -4
  56. package/dist/strand-backfill.js.map +1 -1
  57. package/dist/strand-cohort.d.ts +4 -37
  58. package/dist/strand-cohort.js +4 -15
  59. package/dist/strand-cohort.js.map +1 -1
  60. package/dist/strand-database.d.ts +11 -16
  61. package/dist/strand-database.js +19 -7
  62. package/dist/strand-database.js.map +1 -1
  63. package/dist/strand-formation-manager.d.ts +18 -17
  64. package/dist/strand-formation-manager.js +25 -27
  65. package/dist/strand-formation-manager.js.map +1 -1
  66. package/dist/strand-formation-protocol.d.ts +7 -7
  67. package/dist/strand-formation-protocol.js +8 -8
  68. package/dist/strand-formation-protocol.js.map +1 -1
  69. package/dist/strand-instance-manager.d.ts +53 -21
  70. package/dist/strand-instance-manager.js +128 -37
  71. package/dist/strand-instance-manager.js.map +1 -1
  72. package/dist/strand-membership-writer.d.ts +113 -19
  73. package/dist/strand-membership-writer.js +219 -51
  74. package/dist/strand-membership-writer.js.map +1 -1
  75. package/dist/strand-solicitation.d.ts +3 -3
  76. package/dist/strand-solicitation.js +4 -5
  77. package/dist/strand-solicitation.js.map +1 -1
  78. package/dist/strand-wake-protocol.d.ts +47 -2
  79. package/dist/strand-wake-protocol.js +76 -6
  80. package/dist/strand-wake-protocol.js.map +1 -1
  81. package/dist/types.d.ts +107 -54
  82. package/dist/types.js.map +1 -1
  83. package/package.json +8 -7
  84. package/src/announce-addrs.ts +93 -0
  85. package/src/arachnode-stub.ts +159 -159
  86. package/src/bootstrap-peer-store-file.ts +44 -44
  87. package/src/bootstrap-peer-store.ts +172 -172
  88. package/src/cadre-node.ts +1026 -194
  89. package/src/canonical-json.ts +30 -30
  90. package/src/control-authorization.ts +114 -114
  91. package/src/control-cohort.ts +130 -103
  92. package/src/control-database.ts +376 -298
  93. package/src/control-formation-recorder.ts +305 -304
  94. package/src/control-read-retry.ts +167 -0
  95. package/src/control-retry.ts +190 -0
  96. package/src/control-schema.ts +817 -786
  97. package/src/control-stream.ts +205 -205
  98. package/src/control-write-retry.ts +383 -473
  99. package/src/delegate-admission.ts +304 -302
  100. package/src/device-token.ts +94 -94
  101. package/src/diagnostics/connection-path.ts +277 -277
  102. package/src/diagnostics/webrtc-turn-tracker.ts +222 -222
  103. package/src/ed25519-key.ts +134 -134
  104. package/src/enrollment.ts +199 -199
  105. package/src/file-durable-slot.ts +61 -61
  106. package/src/formation-approval.ts +604 -604
  107. package/src/fs-atomic.ts +107 -107
  108. package/src/hibernation-manager.ts +437 -437
  109. package/src/identity-key.ts +109 -109
  110. package/src/index.ts +435 -406
  111. package/src/key-store-file.ts +132 -132
  112. package/src/key-store.ts +99 -99
  113. package/src/membership-connection-gater.ts +368 -39
  114. package/src/node-local-snapshot.ts +260 -260
  115. package/src/peer-addr-book.ts +250 -0
  116. package/src/peer-authorization.ts +259 -259
  117. package/src/peer-record.ts +197 -164
  118. package/src/push-credentials.ts +84 -84
  119. package/src/push-node.ts +64 -64
  120. package/src/push-notifier-apns.ts +247 -247
  121. package/src/push-notifier-fcm.ts +220 -220
  122. package/src/push-notifier-shared.ts +28 -28
  123. package/src/push-notifier.ts +52 -52
  124. package/src/relay-addrs.ts +134 -27
  125. package/src/relay-reservation.ts +22 -17
  126. package/src/seed-bootstrap.ts +15 -15
  127. package/src/seed-trust-policy.ts +140 -140
  128. package/src/strand-addr-protocol.ts +458 -441
  129. package/src/strand-backfill.ts +28 -5
  130. package/src/strand-cohort.ts +38 -77
  131. package/src/strand-database.ts +202 -204
  132. package/src/strand-formation-manager.ts +26 -28
  133. package/src/strand-formation-protocol.ts +692 -692
  134. package/src/strand-instance-manager.ts +636 -553
  135. package/src/strand-member-key.ts +39 -39
  136. package/src/strand-member-registry.ts +182 -182
  137. package/src/strand-membership-writer.ts +1627 -1430
  138. package/src/strand-solicitation.ts +471 -472
  139. package/src/strand-transport-key.ts +58 -58
  140. package/src/strand-wake-payload.ts +32 -32
  141. package/src/strand-wake-protocol.ts +400 -314
  142. package/src/strand-watcher.ts +314 -314
  143. package/src/trusted-owner-store-file.ts +41 -41
  144. package/src/trusted-owner-store.ts +171 -171
  145. package/src/types.ts +1354 -1303
@@ -1,7 +1,7 @@
1
1
  import debug from 'debug';
2
2
  import { toString as uint8ArrayToString } from 'uint8arrays';
3
- import { Database, registerPlugin, unwrapError } from '@quereus/quereus';
4
- import type { VTablePluginInfo, FunctionPluginInfo, SqlParameters } from '@quereus/quereus';
3
+ import { Database, registerPlugin } from '@quereus/quereus';
4
+ import type { VTablePluginInfo, FunctionPluginInfo, SqlParameters, SqlValue } from '@quereus/quereus';
5
5
  import cryptoPlugin from '@optimystic/quereus-plugin-crypto/plugin';
6
6
  import optimysticPlugin from '@optimystic/quereus-plugin-optimystic/plugin';
7
7
  import { digest, randomBytes } from '@optimystic/quereus-plugin-crypto';
@@ -15,6 +15,8 @@ import type { ControlTable, RevocableTable, ControlDomain, ControlAction } from
15
15
  import { requireEd25519PublicKeyB64 } from './ed25519-key.js';
16
16
  import { retryControlWrite, SCHEMA_INIT_RETRY_POLICY } from './control-write-retry.js';
17
17
  import type { ControlWriteRetryOptions } from './control-write-retry.js';
18
+ import { retryControlRead } from './control-read-retry.js';
19
+ import type { ControlReadRetryOptions } from './control-read-retry.js';
18
20
 
19
21
  export type { ControlTable, RevocableTable, ControlDomain, ControlAction } from './control-authorization.js';
20
22
 
@@ -90,102 +92,34 @@ export class FormationAbortedError extends Error {
90
92
  }
91
93
 
92
94
  /**
93
- * A redemption could not be given a `UseNumber` within the invite's seat budget: the use
94
- * number the writer would have to take is past `FormationInvite.TotalUses`, so the
95
- * invitation is spent.
95
+ * The invitation's seat budget is already spent: the count of recorded `FormationUsage` rows
96
+ * for the token has reached `FormationInvite.TotalUses`, so this redemption cannot be given a
97
+ * seat.
96
98
  *
97
- * Raised inside {@link ControlDatabase.redeemInvitation} / {@link ControlDatabase.recordFormationUsage}
98
- * on EVERY attempt, including the first: two redemptions of one token racing on the SAME node
99
- * serialize behind the local write queue, so the loser's first attempt already takes a use
100
- * number past the seat budget — `StrandFormationManager.validateToken`'s `isTokenUsed` check
101
- * cannot catch this, since it ran before either redemption wrote anything. Without this the
102
- * write would fail as a generic `CHECK constraint failed: Authorized`, which the manager
103
- * reports as a retryable `Formation conflict, retry` — telling the joiner to retry something
104
- * that can never succeed.
99
+ * Raised off the committed-count check ({@link ControlDatabase.assertSeatRemains}) ahead of
100
+ * the write, inside {@link ControlDatabase.redeemInvitation} /
101
+ * {@link ControlDatabase.recordFormationUsage} — and off nothing else. It exists because
102
+ * without it the refusal surfaces as a generic `CHECK constraint failed: Authorized` from the
103
+ * schema's own count-based cap clause, which the manager reports as a retryable
104
+ * `Formation conflict, retry` — telling the joiner to retry something that can never succeed.
105
+ * Two redemptions of one token racing on the SAME node serialize behind the local write
106
+ * queue, so the loser reads the winner's committed row and is refused here by name —
107
+ * `StrandFormationManager.validateToken`'s `isTokenUsed` check cannot catch that, since it
108
+ * ran before either redemption wrote anything.
105
109
  */
106
110
  export class InvitationExhaustedError extends Error {
107
111
  constructor(
108
112
  readonly token: string,
109
- readonly useNumber: number,
113
+ readonly usesRecorded: number,
110
114
  readonly totalUses: number,
111
115
  ) {
112
116
  super(
113
- `Invitation ${token} is exhausted: use #${useNumber} is past its limit of ${totalUses} use(s)`
117
+ `Invitation ${token} is exhausted: ${usesRecorded} of ${totalUses} use(s) already recorded`
114
118
  );
115
119
  this.name = 'InvitationExhaustedError';
116
120
  }
117
121
  }
118
122
 
119
- /**
120
- * Attempts allowed per redemption: the first, plus two retries of a use number another
121
- * writer took. Bounded so sustained contention terminates in a clean rejection rather than
122
- * spinning; two retries is enough for the realistic case (a handful of nodes redeeming one
123
- * multi-use token at once) without holding the write lock hostage.
124
- *
125
- * NOTE: the attempts fire back-to-back, with no backoff and no jitter — each one only takes
126
- * and releases the write lock. Fine while cross-node contention on a single token is a
127
- * handful of writers; if one hot multi-use token ever draws sustained concurrent redemption
128
- * (a broadcast invite, a bulk onboarding run), all three attempts can burn against the same
129
- * competitor and the caller gets the engine's error. Add a short randomised delay between
130
- * attempts then — NOT a longer attempt budget, which would hold the write queue.
131
- */
132
- const USE_NUMBER_ATTEMPTS = 3;
133
-
134
- /**
135
- * The two error messages that mean "another writer took the `UseNumber` we picked", and
136
- * nothing else.
137
- *
138
- * - the `(Token, UseNumber)` primary-key collision, which the optimystic vtab reports on the
139
- * insert itself (`uniqueConstraintMessage` renders the columns table-qualified, no schema
140
- * prefix);
141
- * - `CHECK constraint failed: Monotonic` — `FormationUsage.Monotonic` contains a subquery, so
142
- * Quereus defers it to commit where it reads `committed.FormationUsage`. A concurrent row
143
- * that our insert's key probe did not see still trips it there. Same meaning, different
144
- * surface.
145
- *
146
- * Deliberately NOT matched: `FormationUsage.UsageStampId`'s `unique` (a repeated nonce is
147
- * replay, never a race — retrying it would re-present a spent approval), and every named
148
- * CHECK other than `Monotonic`. `Authorized` in particular renders with the same
149
- * `CHECK constraint failed:` prefix, so the constraint NAME is anchored with `\b` exactly as
150
- * `expectConstraintFailure` does rather than matched by prefix.
151
- *
152
- * NOTE: a CHECK failure names only the CONSTRAINT, never its table, and `Monotonic` is a
153
- * generic enough name that a second table could plausibly claim it. Today it is unique across
154
- * `schemas/control.qsql`, so the pattern cannot mis-fire; if another table ever declares a
155
- * `Monotonic` constraint, a failure of THAT constraint would be read here as a lost use-number
156
- * race and pointlessly retried up to {@link USE_NUMBER_ATTEMPTS} times. Rename the other
157
- * constraint (cheapest) rather than trying to recover the table from the message.
158
- */
159
- const LOST_USE_NUMBER_PATTERNS = [
160
- /UNIQUE constraint failed: FormationUsage\.Token, FormationUsage\.UseNumber/i,
161
- /CHECK constraint failed: Monotonic\b/,
162
- ];
163
-
164
- /**
165
- * Did this write fail purely because another writer took the `UseNumber` it picked — i.e. is
166
- * re-presenting the SAME approval under the next free use number the right response?
167
- *
168
- * Classifies by MESSAGE, walking the `cause` chain with Quereus' own {@link unwrapError}:
169
- * neither surface can be recognised by type. The deferred `Monotonic` failure is thrown as a
170
- * bare `QuereusError`, not a `ConstraintError` (see
171
- * `quereus/src/runtime/deferred-constraint-queue.ts`), so an `instanceof ConstraintError` gate
172
- * would silently drop the branch this retry exists for. Messages also carry an
173
- * ` (at line N, column M)` suffix and arrive wrapped when the write ran inside a transaction,
174
- * so the match is a substring test rather than an equality one.
175
- *
176
- * NOTE: this is the one piece of the retry that depends on engine error TEXT. Its own test
177
- * (`control-formation-use-number-retry.spec.ts`) produces all three messages from the real
178
- * engine rather than from string literals, so a reworded storage-layer error fails that spec
179
- * instead of silently disabling the retry.
180
- */
181
- export function isLostUseNumberRace(error: unknown): boolean {
182
- if (!(error instanceof Error)) {
183
- return false;
184
- }
185
- return unwrapError(error).some(({ message }) =>
186
- LOST_USE_NUMBER_PATTERNS.some(pattern => pattern.test(message)));
187
- }
188
-
189
123
  /**
190
124
  * What a recorded `FormationUsage` row came out as.
191
125
  *
@@ -194,9 +128,7 @@ export function isLostUseNumberRace(error: unknown): boolean {
194
128
  * `FormationUsage.Authorized` rejects the row (see {@link formationVouchMessage}).
195
129
  */
196
130
  export interface FormationUsageResult {
197
- /** The `UseNumber` assigned to this redemption (1-based per token). */
198
- useNumber: number;
199
- /** The single-use nonce written to `UsageStampId` — the caller's, or a freshly minted one. */
131
+ /** The single-use nonce written to `UsageStampId` (the row's primary key) — the caller's, or a freshly minted one. */
200
132
  usageStampId: string;
201
133
  }
202
134
 
@@ -449,6 +381,13 @@ export class ControlDatabase {
449
381
  * having to restate the policy's attempts or classifier.
450
382
  */
451
383
  private controlWriteRetryPacing: ControlWriteRetryOptions = {};
384
+ /**
385
+ * Pacing seams for {@link readRows}'s retry — the read twin of
386
+ * {@link controlWriteRetryPacing}, with the same contract: production leaves this
387
+ * empty, specs inject a recorded `sleep` / fake `now` so no test waits out a real
388
+ * backoff. Applied OVER the read policy's defaults but UNDER the call site's `label`.
389
+ */
390
+ private controlReadRetryPacing: ControlReadRetryOptions = {};
452
391
 
453
392
  constructor(config: ControlDatabaseConfig) {
454
393
  this.config = config;
@@ -532,6 +471,20 @@ export class ControlDatabase {
532
471
  log('Hydrated control catalog (tables=%d, indexes=%d)', hydrated.tables, hydrated.indexes);
533
472
 
534
473
  // Load and execute the schema
474
+ // NOTE: this is where a slow launch is felt. Duration is (raw-storage operations
475
+ // issued) × per-operation storage latency: ~1ms/op on an idle machine, but
476
+ // 50-90ms/op on a loaded disk or a phone's flash under launch contention. A cold
477
+ // start now reaches the backend 172 times (8 tables + 1 index, 21 distinct blocks
478
+ // — dominated by its 130 genuine writes), a warm restart 52, because cadre-core
479
+ // wraps every embedder storage in `@optimystic/db-p2p`'s write-through cache
480
+ // (@serfab/quereus-plugin-sereus's cached-storage.ts). Uncached the same start issued ~2000 operations — the
481
+ // upstream re-read amplification measured in
482
+ // tickets/complete/optimystic-block-read-amplification-on-control-start.md (and
483
+ // its follow-up ...-schema-catalog-reread-per-write-blows-storage-budgets.md),
484
+ // which also rules out the retry policy and cluster deadlines. The counts are
485
+ // pinned by packages/cadre-core/test/control-start-storage-op-budget.spec.ts; if
486
+ // they grow — e.g. the cache leaves this path — that spec fails before anyone
487
+ // has to debug a launch.
535
488
  t0 = performance.now();
536
489
  await this.loadSchema();
537
490
  timing('[controlDb] loadSchema: %dms', Math.round(performance.now() - t0));
@@ -605,17 +558,139 @@ export class ControlDatabase {
605
558
  // also what lets schema init absorb optimystic's self-coordination grace refusal, which
606
559
  // is unsafe for control writes in general. Why, on
607
560
  // `RETRIABLE_SCHEMA_INIT_MATCHERS` in `control-write-retry.ts`.
608
- await this.lockedWithRetry(() => this.db!.exec(schemaContent), SCHEMA_INIT_RETRY_POLICY);
561
+ await this.lockedWithRetry(() => this.db!.exec(schemaContent), SCHEMA_INIT_RETRY_POLICY, 'schema-init');
609
562
  log('Schema loaded and executed');
610
563
  }
611
564
 
565
+ /**
566
+ * One UNRETRIED drain of a control read to rows — every read in this class bottoms out
567
+ * here (almost all via {@link readRows}, which adds the transient-failure retry), and
568
+ * this is the ONE place the committed-read opt-in is spelled.
569
+ *
570
+ * Reached only through {@link readRows}, which drops straight here for the call sites
571
+ * that pass `retry: false` — reads issued from INSIDE a locked write body, where a
572
+ * backoff sleep would hold the write lock and the write funnel re-runs the read anyway
573
+ * (see the NOTE on {@link readRows}).
574
+ *
575
+ * A control read normally queues on the database's exec mutex, so it does not answer
576
+ * until whatever statement holds that mutex finishes. When a control WRITE is parked
577
+ * against an unresponsive cohort member that is minutes, and a read that only wants
578
+ * the last committed state waits the whole time for no reason
579
+ * (`complete/control-reads-blocked-by-stalled-write`).
580
+ *
581
+ * Quereus's remedy is `readConcurrency: 'committed'` — a PER-CALL opt-in
582
+ * (`Statement.tryRouteConcurrent` refuses anything that did not ask for it by name)
583
+ * that runs an eligible read off the mutex against each table's last committed state.
584
+ * Routing is best-effort and never an error: an ineligible statement falls back to
585
+ * the serialized path silently — any side-effecting node, any table-valued function
586
+ * call (pure ones included), any table whose module does not declare
587
+ * `readCommittedSnapshot`, or an EXPLICIT transaction being open on this database.
588
+ *
589
+ * **It is asked for only while a write is in flight, and that condition is the whole
590
+ * point.** The opt-in does not merely move the read off the mutex: it connects the
591
+ * table with `_readCommitted`, and the optimystic vtab serves such a connection from
592
+ * a pinned pre-transaction snapshot that NEVER refreshes from the network, where an
593
+ * ordinary read calls `update()` on the collection first. Taking that path
594
+ * unconditionally therefore trades "reads block behind a stalled write" for "reads
595
+ * stop seeing what other machines wrote" — measured, not theorised: with every read
596
+ * opted in, `control-write-degraded-cohort-member` could no longer observe a sibling
597
+ * publishing its own `CadrePeer` row and failed at suite setup on two runs out of two.
598
+ *
599
+ * `getAutocommit()` reports the whole `Database` rather than this call (the same
600
+ * property {@link assertCommitBoundary} leans on), which is exactly what is wanted:
601
+ * false means some writer's transaction is open, which is the only situation where a
602
+ * read would have blocked. The check races that writer harmlessly in both directions
603
+ * — losing it either reads normally (today's behaviour) or reads committed state.
604
+ *
605
+ * What a committed read gives up is observing a write that has not finished
606
+ * committing, and that is safe for every caller here: an AWAITED write is committed by
607
+ * the time it returns, and a read inside {@link inTransaction} is disqualified by the
608
+ * open explicit transaction and stays serialized, so it still sees its own
609
+ * transaction's rows. A read after an UNAWAITED write on this database would be wrong
610
+ * — no such caller exists, and {@link withWriteLock} is what keeps it that way.
611
+ */
612
+ private async readRowsOnce(sql: string, params?: SqlParameters): Promise<Record<string, SqlValue>[]> {
613
+ let iterator: AsyncIterableIterator<Record<string, SqlValue>>;
614
+ if (this.db!.getAutocommit()) {
615
+ iterator = this.db!.eval(sql, params);
616
+ } else {
617
+ log('read-eval: a write is in flight, asking for a committed read: %s', sql);
618
+ iterator = this.db!.eval(sql, params, { readConcurrency: 'committed' });
619
+ }
620
+ const rows: Record<string, SqlValue>[] = [];
621
+ for await (const row of iterator) {
622
+ rows.push(row);
623
+ }
624
+ return rows;
625
+ }
626
+
627
+ /**
628
+ * {@link readRowsOnce} plus the bounded transient-failure retry
629
+ * ({@link retryControlRead}) — the funnel every UNLOCKED control read goes through, so a
630
+ * read that failed because the cluster could not be asked properly (a stream reset
631
+ * mid-scan, a partially unreachable cohort) is re-presented a moment later instead of
632
+ * surfacing to every reader.
633
+ *
634
+ * MATERIALIZES the read rather than returning a lazy iterator, and that shape is what
635
+ * makes the retry possible at all: the failure happens during ITERATION, not at call
636
+ * time, and a half-consumed iterator cannot be retried without re-yielding rows the
637
+ * caller already saw. Materializing is safe here — every caller either drains to an
638
+ * array anyway or takes the first row of a statement that yields at most one (a
639
+ * primary-key lookup or a `count(1)`); nothing streams.
640
+ *
641
+ * Each ATTEMPT is a fresh {@link readRowsOnce} call, so the committed-read opt-in
642
+ * (`getAutocommit()`) is re-evaluated per attempt — a write can finish between attempts
643
+ * and change the right answer.
644
+ *
645
+ * The retry's budget ({@link CONTROL_READ_RETRY_BUDGET_MS}, 1.5 s) is deliberately far
646
+ * under the write funnel's: the tightest caller deadline over a control read is the
647
+ * inbound admission gate's 2 s FAIL-OPEN timeout, and a read that outlives it spends its
648
+ * retries after the gate has already admitted (see `control-read-retry.ts`).
649
+ *
650
+ * `label` names the read in the retry loop's `Control read [<label>] …` debug lines and
651
+ * nothing else; several reads are in flight concurrently in a real party, so an
652
+ * unlabelled line cannot be attributed — keep new call sites labelled.
653
+ *
654
+ * `retry: false` drops straight to {@link readRowsOnce}: reads issued from INSIDE a
655
+ * locked write body must not retry on their own. `retryControlWrite`'s contract is that
656
+ * backoff sleeps happen with NO lock held; a read retrying its own backoff inside a
657
+ * locked body would sleep holding the write lock (stalling every other local writer),
658
+ * and the write funnel already re-runs the body's reads when it re-runs the body. See
659
+ * {@link queryStampId} / {@link assertSeatRemains} / {@link queryCadrePeers} for the
660
+ * per-call opt-outs.
661
+ *
662
+ * NOTE: nothing structural enforces that opt-out — it is per-call because an unlocked
663
+ * read runs CONCURRENTLY with a locked body, so `this`-state cannot tell the two apart,
664
+ * and there is no async-context primitive available on every target platform (browser,
665
+ * React Native) to carry the answer. Reads reached through a CALLBACK invoked under the
666
+ * lock are the easy miss: `notifyMembershipChanged` runs its listener with the lock
667
+ * held, and its membership read is opted out at {@link queryCadrePeers}. If another
668
+ * under-lock callback seam is ever added, audit its read graph the same way.
669
+ */
670
+ private readRows(
671
+ sql: string,
672
+ params: SqlParameters | undefined,
673
+ label: string,
674
+ retry = true
675
+ ): Promise<Record<string, SqlValue>[]> {
676
+ if (!retry) {
677
+ return this.readRowsOnce(sql, params);
678
+ }
679
+ // The label lands LAST so the call site's name survives the spec-injected pacing —
680
+ // it changes log attribution only, never behaviour (same layering as lockedWithRetry).
681
+ return retryControlRead(
682
+ () => this.readRowsOnce(sql, params),
683
+ { ...this.controlReadRetryPacing, label }
684
+ );
685
+ }
686
+
612
687
  /**
613
688
  * Query all strands from the control database
614
689
  */
615
690
  async queryStrands(): Promise<StrandRow[]> {
616
691
  this.ensureInitialized();
617
692
  const results: StrandRow[] = [];
618
- for await (const row of this.db!.eval('select Id, MemberPrivateKey, Type from CadreControl.Strand')) {
693
+ for (const row of await this.readRows('select Id, MemberPrivateKey, Type from CadreControl.Strand', undefined, 'strands')) {
619
694
  results.push({
620
695
  Id: row.Id as string,
621
696
  MemberPrivateKey: row.MemberPrivateKey as string | null,
@@ -633,9 +708,10 @@ export class ControlDatabase {
633
708
  */
634
709
  async queryStrand(strandId: string): Promise<StrandRow | null> {
635
710
  this.ensureInitialized();
636
- for await (const row of this.db!.eval(
711
+ for (const row of await this.readRows(
637
712
  'select Id, MemberPrivateKey, Type from CadreControl.Strand where Id = ?',
638
- [strandId]
713
+ [strandId],
714
+ 'strand'
639
715
  )) {
640
716
  return {
641
717
  Id: row.Id as string,
@@ -662,7 +738,7 @@ export class ControlDatabase {
662
738
  if (!CONTROL_TABLE_SET.has(table)) {
663
739
  throw new Error(`Unknown CadreControl table: ${table}`);
664
740
  }
665
- for await (const row of this.db!.eval(`select count(1) as Count from CadreControl.${table}`)) {
741
+ for (const row of await this.readRows(`select count(1) as Count from CadreControl.${table}`, undefined, 'count-rows')) {
666
742
  return (row.Count as number) ?? 0;
667
743
  }
668
744
  return 0;
@@ -682,7 +758,7 @@ export class ControlDatabase {
682
758
  */
683
759
  async hasOwnerKey(): Promise<boolean> {
684
760
  this.ensureInitialized();
685
- for await (const row of this.db!.eval('select count(1) as Count from CadreControl.OwnerKey')) {
761
+ for (const row of await this.readRows('select count(1) as Count from CadreControl.OwnerKey', undefined, 'owner-key-exists')) {
686
762
  return (row.Count as number) > 0;
687
763
  }
688
764
  return false;
@@ -715,7 +791,7 @@ export class ControlDatabase {
715
791
  async getOwnerKeys(): Promise<Set<string>> {
716
792
  this.ensureInitialized();
717
793
  const keys = new Set<string>();
718
- for await (const row of this.db!.eval('select Key from CadreControl.OwnerKey')) {
794
+ for (const row of await this.readRows('select Key from CadreControl.OwnerKey', undefined, 'owner-keys')) {
719
795
  keys.add(row.Key as string);
720
796
  }
721
797
  return keys;
@@ -737,12 +813,29 @@ export class ControlDatabase {
737
813
  * membership-gate refresh path; control tables are a handful of rows today, so this
738
814
  * is cheap — if they ever grow, fold the exclusion into one statement or cache the
739
815
  * retired-stamp set.
816
+ *
817
+ * NOTE: this filter is HAND-MIRRORED by the gate suites' fake control databases
818
+ * (`test/membership-gate-helpers.ts` and `test/cadre-node-authorized-surface.spec.ts`),
819
+ * which feed the node rows directly. Nothing enforces that the copies stay true, so a
820
+ * change to the filter's SHAPE here (what it keys on, which table it consults) must be
821
+ * carried into both, or those suites keep passing against a contract the database no
822
+ * longer honours. Fine while the filter is one `Set.has` on `StampId`; if it grows a
823
+ * second dimension, give the fakes a shared fixture or a contract test instead.
824
+ *
825
+ * `retry: false` is passed by exactly one caller —
826
+ * `CadreNode.refreshAuthorizedControlPeers`, the membership-gate snapshot refresh —
827
+ * because {@link notifyMembershipChanged} invokes that listener with the write lock
828
+ * HELD, and a retrying read there would sleep its backoff holding the lock (see the
829
+ * NOTE on {@link readRows}). It costs nothing: that refresh is best-effort, keeps the
830
+ * previous snapshot on failure, and is re-driven by the next write and by the timed
831
+ * reconcile. BOTH of this method's reads take the flag — retrying either one under the
832
+ * lock is the same defect.
740
833
  */
741
- async queryCadrePeers(): Promise<CadrePeerRow[]> {
834
+ async queryCadrePeers(retry = true): Promise<CadrePeerRow[]> {
742
835
  this.ensureInitialized();
743
- const revoked = await this.queryRevokedStamps('CadrePeer');
836
+ const revoked = await this.queryRevokedStamps('CadrePeer', retry);
744
837
  const rows: CadrePeerRow[] = [];
745
- for await (const row of this.db!.eval('select PeerId, Multiaddr, StampId, VouchOwner, VouchSig from CadreControl.CadrePeer')) {
838
+ for (const row of await this.readRows('select PeerId, Multiaddr, StampId, VouchOwner, VouchSig from CadreControl.CadrePeer', undefined, 'cadre-peers', retry)) {
746
839
  const stampId = (row.StampId as string | null) ?? null;
747
840
  if (stampId !== null && revoked.has(stampId)) {
748
841
  continue;
@@ -770,7 +863,7 @@ export class ControlDatabase {
770
863
  async queryValidationKeys(): Promise<string[]> {
771
864
  this.ensureInitialized();
772
865
  const keys: string[] = [];
773
- for await (const row of this.db!.eval('select Key from CadreControl.ValidationKey')) {
866
+ for (const row of await this.readRows('select Key from CadreControl.ValidationKey', undefined, 'validation-keys')) {
774
867
  keys.push(row.Key as string);
775
868
  }
776
869
  return keys.sort();
@@ -790,14 +883,25 @@ export class ControlDatabase {
790
883
  * `table` and its {@link GUARDED_KEY_COLUMN} column are interpolated into the SQL; both
791
884
  * come from closed literal unions — no caller-supplied string can reach the statement
792
885
  * (same injection-surface discipline as {@link countRows}'s `CONTROL_TABLE_SET` guard).
886
+ *
887
+ * NOTE: `retry: false` is passed by exactly the call sites that run INSIDE a locked
888
+ * write body — {@link deleteGuardedRow} and {@link insertCadrePeer}'s insert-if-absent
889
+ * guard. A read retrying there would sleep its backoff HOLDING the write lock
890
+ * (`retryControlWrite`'s contract is that backoff sleeps happen with no lock held), and
891
+ * the write funnel already re-runs the body's reads when it re-runs the body. The
892
+ * unlocked callers — the public stamp readers, {@link reauthorizeCadrePeer} and
893
+ * {@link reapRevokedRow}, whose stamp reads run BEFORE their locks are taken — keep the
894
+ * default retried path. Explicit per call rather than an ambient flag: unlocked reads
895
+ * run concurrently with a locked body, so `this`-state cannot tell the two apart.
793
896
  */
794
897
  private async queryStampId(
795
898
  table: RevocableTable,
796
- keyValue: string
899
+ keyValue: string,
900
+ retry = true
797
901
  ): Promise<string | null> {
798
902
  this.ensureInitialized();
799
903
  const sql = `select StampId from CadreControl.${table} where ${GUARDED_KEY_COLUMN[table]} = ?`;
800
- for await (const row of this.db!.eval(sql, [keyValue])) {
904
+ for (const row of await this.readRows(sql, [keyValue], `stamp-${table}`, retry)) {
801
905
  return (row.StampId as string | null) ?? null;
802
906
  }
803
907
  return null;
@@ -847,11 +951,14 @@ export class ControlDatabase {
847
951
  * NOTE: re-reads the whole retired set on every call, and the caller runs per inbound
848
952
  * gate request while the table only ever grows. Cheap today (a cadre removes peers
849
953
  * rarely); if removals ever become routine, cache the set and invalidate it on write.
954
+ *
955
+ * `retry: false` is forwarded by {@link queryCadrePeers} for the one caller that reads
956
+ * under the write lock — see that method's note.
850
957
  */
851
- async queryRevokedStamps(tableName: RevocableTable): Promise<Set<string>> {
958
+ async queryRevokedStamps(tableName: RevocableTable, retry = true): Promise<Set<string>> {
852
959
  this.ensureInitialized();
853
960
  const stamps = new Set<string>();
854
- for await (const row of this.db!.eval('select StampId from CadreControl.Revocation where TableName = ?', [tableName])) {
961
+ for (const row of await this.readRows('select StampId from CadreControl.Revocation where TableName = ?', [tableName], 'revoked-stamps', retry)) {
855
962
  stamps.add(row.StampId as string);
856
963
  }
857
964
  return stamps;
@@ -868,7 +975,7 @@ export class ControlDatabase {
868
975
  async queryRevocations(): Promise<RevocationRow[]> {
869
976
  this.ensureInitialized();
870
977
  const rows: RevocationRow[] = [];
871
- for await (const row of this.db!.eval('select TableName, RowKey, StampId, ReissuedAt from CadreControl.Revocation')) {
978
+ for (const row of await this.readRows('select TableName, RowKey, StampId, ReissuedAt from CadreControl.Revocation', undefined, 'revocations')) {
872
979
  rows.push({
873
980
  tableName: row.TableName as RevocableTable,
874
981
  rowKey: row.RowKey as string,
@@ -890,7 +997,7 @@ export class ControlDatabase {
890
997
  * {@link queryRevokedStamps} rather than inlining the SQL, so tests can interpose on
891
998
  * that seam (see {@link queryCadrePeers}).
892
999
  *
893
- * Missing/legacy column values are coalesced
1000
+ * Missing/null column values are coalesced
894
1001
  * to their empty form (`''` key/sig, `[]` addrs, `0` stamp) so the caller's
895
1002
  * verify/freshness gates uniformly reject an unpublished or malformed row.
896
1003
  * The split `addrs` re-join to the exact stored `Multiaddr` (split-on-`,` is
@@ -900,9 +1007,10 @@ export class ControlDatabase {
900
1007
  async queryPeerRecord(peerId: string): Promise<PeerAddressRecord | null> {
901
1008
  this.ensureInitialized();
902
1009
  const revoked = await this.queryRevokedStamps('CadrePeer');
903
- for await (const row of this.db!.eval(
1010
+ for (const row of await this.readRows(
904
1011
  'select PeerId, PublicKey, Multiaddr, UpdatedAt, Sig, StampId from CadreControl.CadrePeer where PeerId = ?',
905
- [peerId]
1012
+ [peerId],
1013
+ 'peer-record'
906
1014
  )) {
907
1015
  const stampId = (row.StampId as string | null) ?? null;
908
1016
  if (stampId !== null && revoked.has(stampId)) {
@@ -951,14 +1059,14 @@ export class ControlDatabase {
951
1059
  with context OwnerKey = null, Signature = ?
952
1060
  set Multiaddr = ?, UpdatedAt = ?, Sig = ?
953
1061
  where PeerId = ?
954
- `, [record.sig, multiaddr, record.updatedAt, record.sig, record.peerId]);
1062
+ `, [record.sig, multiaddr, record.updatedAt, record.sig, record.peerId], 'self-record-update');
955
1063
  log('Self peer record updated: %s (updatedAt=%d)', record.peerId, record.updatedAt);
956
1064
  }
957
1065
 
958
1066
  /**
959
1067
  * Read a single peer's device push token (the full `DeviceToken` row) by PeerId.
960
1068
  *
961
- * Returns null when no row exists. Missing/legacy column values are coalesced to
1069
+ * Returns null when no row exists. Missing/null column values are coalesced to
962
1070
  * their empty form (`''` token/sig, `0` stamp) so the caller's verify/freshness
963
1071
  * gates uniformly reject an unpublished or malformed row. `platform` is returned
964
1072
  * verbatim (the resolver validates it against {@link PushPlatform} and re-verifies
@@ -970,9 +1078,10 @@ export class ControlDatabase {
970
1078
  */
971
1079
  async queryDeviceToken(peerId: string): Promise<DeviceTokenRow | null> {
972
1080
  this.ensureInitialized();
973
- for await (const row of this.db!.eval(
1081
+ for (const row of await this.readRows(
974
1082
  'select PeerId, Platform, Token, UpdatedAt, Sig, StampId from CadreControl.DeviceToken where PeerId = ?',
975
- [peerId]
1083
+ [peerId],
1084
+ 'device-token'
976
1085
  )) {
977
1086
  return {
978
1087
  peerId: row.PeerId as string,
@@ -1004,7 +1113,7 @@ export class ControlDatabase {
1004
1113
  with context OwnerKey = null, Signature = ?
1005
1114
  set Platform = ?, Token = ?, UpdatedAt = ?, Sig = ?
1006
1115
  where PeerId = ?
1007
- `, [record.sig, record.platform, record.token, record.updatedAt, record.sig, record.peerId]);
1116
+ `, [record.sig, record.platform, record.token, record.updatedAt, record.sig, record.peerId], 'device-token-update');
1008
1117
  log('Self device token updated: %s (platform=%s, updatedAt=%d)', record.peerId, record.platform, record.updatedAt);
1009
1118
  }
1010
1119
 
@@ -1028,7 +1137,7 @@ export class ControlDatabase {
1028
1137
  insert into CadreControl.OwnerKey (Key, StampId)
1029
1138
  with context OwnerKey = null, Signature = null
1030
1139
  values (?, ?)
1031
- `, [key, stampId]);
1140
+ `, [key, stampId], 'owner-key-insert');
1032
1141
  log('Owner key inserted');
1033
1142
  }
1034
1143
 
@@ -1071,7 +1180,7 @@ export class ControlDatabase {
1071
1180
  insert into CadreControl.Strand (Id, Type, MemberPrivateKey, StampId)
1072
1181
  with context OwnerKey = ?, Signature = ?
1073
1182
  values (?, ?, ?, ?)
1074
- `, [ownerKey, signature, strandId, type, memberPrivateKey ?? null, stampId]);
1183
+ `, [ownerKey, signature, strandId, type, memberPrivateKey ?? null, stampId], 'strand-insert');
1075
1184
 
1076
1185
  log('Strand inserted: %s', strandId);
1077
1186
  }
@@ -1099,7 +1208,7 @@ export class ControlDatabase {
1099
1208
  ownerKey: string,
1100
1209
  signMessage: (message: Uint8Array) => string
1101
1210
  ): Promise<boolean> {
1102
- return this.lockedWithRetry(() => this.deleteGuardedRow('Strand', strandId, ownerKey, signMessage));
1211
+ return this.lockedWithRetry(() => this.deleteGuardedRow('Strand', strandId, ownerKey, signMessage), {}, 'strand-delete');
1103
1212
  }
1104
1213
 
1105
1214
  /**
@@ -1134,7 +1243,7 @@ export class ControlDatabase {
1134
1243
  insert into CadreControl.ValidationKey (Key, StampId)
1135
1244
  with context OwnerKey = ?, Signature = ?
1136
1245
  values (?, ?)
1137
- `, [ownerKey, signature, key, stampId]);
1246
+ `, [ownerKey, signature, key, stampId], 'validation-key-insert');
1138
1247
 
1139
1248
  log('Validation key inserted: %s', key);
1140
1249
  }
@@ -1160,7 +1269,7 @@ export class ControlDatabase {
1160
1269
  ownerKey: string,
1161
1270
  signMessage: (message: Uint8Array) => string
1162
1271
  ): Promise<boolean> {
1163
- return this.lockedWithRetry(() => this.deleteGuardedRow('ValidationKey', key, ownerKey, signMessage));
1272
+ return this.lockedWithRetry(() => this.deleteGuardedRow('ValidationKey', key, ownerKey, signMessage), {}, 'validation-key-delete');
1164
1273
  }
1165
1274
 
1166
1275
  /**
@@ -1219,7 +1328,8 @@ export class ControlDatabase {
1219
1328
  buildAuthorizationMessage('CadreControl.CadrePeer', 'vouch', [row.peerId, stampId])
1220
1329
  );
1221
1330
  return await this.mutateCadrePeer('peer-insert', async () => {
1222
- if (await this.queryStampId('CadrePeer', row.peerId) !== null) {
1331
+ // retry: false — this guard runs inside the locked write body (see queryStampId's NOTE).
1332
+ if (await this.queryStampId('CadrePeer', row.peerId, false) !== null) {
1223
1333
  log('CadrePeer row already present for %s; insert skipped (already a member)', row.peerId);
1224
1334
  return false;
1225
1335
  }
@@ -1360,7 +1470,7 @@ export class ControlDatabase {
1360
1470
  ownerKey: string,
1361
1471
  signMessage: (message: Uint8Array) => string
1362
1472
  ): Promise<boolean> {
1363
- return this.lockedWithRetry(() => this.deleteGuardedRow('DeviceToken', peerId, ownerKey, signMessage));
1473
+ return this.lockedWithRetry(() => this.deleteGuardedRow('DeviceToken', peerId, ownerKey, signMessage), {}, 'device-token-delete');
1364
1474
  }
1365
1475
 
1366
1476
  /**
@@ -1403,7 +1513,8 @@ export class ControlDatabase {
1403
1513
  signMessage: (message: Uint8Array) => string
1404
1514
  ): Promise<boolean> {
1405
1515
  this.ensureInitialized();
1406
- const stampId = await this.queryStampId(table, keyValue);
1516
+ // retry: false — this runs inside the locked write body (see queryStampId's NOTE).
1517
+ const stampId = await this.queryStampId(table, keyValue, false);
1407
1518
  if (stampId === null) {
1408
1519
  log('delete %s: no row for %s (already absent)', table, keyValue);
1409
1520
  return false;
@@ -1527,7 +1638,7 @@ export class ControlDatabase {
1527
1638
  // in one pass, hoist the notify to one refresh after the sweep instead.
1528
1639
  await this.mutateCadrePeer('peer-reap', () => this.db!.exec(sql, [rowKey, stampId]));
1529
1640
  } else {
1530
- await this.execWrite(sql, [rowKey, stampId]);
1641
+ await this.execWrite(sql, [rowKey, stampId], `reap-${table}`);
1531
1642
  }
1532
1643
  // A row leaving the control plane without an owner signature at the write site
1533
1644
  // should be visible in a log.
@@ -1689,7 +1800,7 @@ export class ControlDatabase {
1689
1800
  });
1690
1801
  log('Reissued %d revocation tombstone(s) at %d', executed, reissuedAt);
1691
1802
  return executed;
1692
- });
1803
+ }, {}, 'revocation-reissue');
1693
1804
  }
1694
1805
 
1695
1806
  /**
@@ -1794,7 +1905,7 @@ export class ControlDatabase {
1794
1905
  this.assertCommitBoundary(reason, 'on return from');
1795
1906
  await this.notifyMembershipChanged(reason);
1796
1907
  return result;
1797
- });
1908
+ }, {}, reason);
1798
1909
  }
1799
1910
 
1800
1911
  /**
@@ -1807,7 +1918,8 @@ export class ControlDatabase {
1807
1918
  * {@link assertCommitBoundary} (any `CadrePeer` path) or silently join each other's
1808
1919
  * open transaction — a torn commit if either side rolls back. This lock closes that
1809
1920
  * class: every public write method of this class runs its statement(s) under it, and
1810
- * `SeedBootstrapService` wraps its direct `db.exec` writes in it too. The race is
1921
+ * `SeedBootstrapService` reaches it through {@link execWrite} rather than touching the
1922
+ * `Database` directly, so it is covered too. The race is
1811
1923
  * real, not theoretical: the background self-record publish on a control connection
1812
1924
  * opening (`CadreNode.drainPendingControlReplication` → `registerSelf`) collided with
1813
1925
  * a foreground `authorizePeer` — both `CadrePeer` inserts — and tripped the boundary
@@ -1851,10 +1963,8 @@ export class ControlDatabase {
1851
1963
  *
1852
1964
  * The retry wraps the LOCK, it does not live inside it: each attempt takes and
1853
1965
  * releases the lock, and the backoff sleeps with NO lock held, so a write parked in
1854
- * backoff never stalls the other local writers queued behind it. Same reason
1855
- * {@link withUseNumberRetry} re-takes the lock per attempt — see the note on
1856
- * `USE_NUMBER_ATTEMPTS`. Safe because every locked body is atomic and re-runnable
1857
- * (the contract on {@link withWriteLock}).
1966
+ * backoff never stalls the other local writers queued behind it. Safe because every
1967
+ * locked body is atomic and re-runnable (the contract on {@link withWriteLock}).
1858
1968
  *
1859
1969
  * Also carries {@link loadSchema}'s distributed DDL, the one caller that runs BEFORE
1860
1970
  * `initialized` is set — so it reaches this method directly rather than through
@@ -1863,11 +1973,18 @@ export class ControlDatabase {
1863
1973
  * rather than carving out a lockless path. It is also the ONLY caller that passes a
1864
1974
  * `policy` — everything else takes the default, whose classifier and attempt count are
1865
1975
  * unchanged by that call site existing.
1976
+ *
1977
+ * `label` names the operation in this loop's debug lines and nothing else
1978
+ * ({@link ControlWriteRetryOptions.label}). Every call site in this class supplies one:
1979
+ * several writes retry CONCURRENTLY in a real party, so an unlabelled line cannot be
1980
+ * attributed to a write — keep new call sites labelled.
1866
1981
  */
1867
- private lockedWithRetry<T>(fn: () => Promise<T>, policy: ControlWriteRetryOptions = {}): Promise<T> {
1982
+ private lockedWithRetry<T>(fn: () => Promise<T>, policy: ControlWriteRetryOptions = {}, label?: string): Promise<T> {
1983
+ // The label lands LAST so a call site's own name survives both the policy and the
1984
+ // spec-injected pacing; it changes log attribution only, never behaviour.
1868
1985
  return retryControlWrite(
1869
1986
  () => this.withWriteLock(fn),
1870
- { ...policy, ...this.controlWriteRetryPacing }
1987
+ { ...policy, ...this.controlWriteRetryPacing, ...(label !== undefined ? { label } : {}) }
1871
1988
  );
1872
1989
  }
1873
1990
 
@@ -1880,10 +1997,13 @@ export class ControlDatabase {
1880
1997
  * Public so `SeedBootstrapService`'s direct `CadrePeer`/`DeviceToken` SQL writes go
1881
1998
  * through the same seam. Must NOT be called from inside an already-locked body (the
1882
1999
  * lock is not re-entrant); such a caller uses a bare `getDatabase().exec` instead.
2000
+ *
2001
+ * `label` names the write in the retry loop's debug lines (see {@link lockedWithRetry});
2002
+ * optional only because it is log-only, but every caller in this repo passes one.
1883
2003
  */
1884
- execWrite(sql: string, params?: SqlParameters): Promise<void> {
2004
+ execWrite(sql: string, params?: SqlParameters, label?: string): Promise<void> {
1885
2005
  this.ensureInitialized();
1886
- return this.lockedWithRetry(() => this.db!.exec(sql, params));
2006
+ return this.lockedWithRetry(() => this.db!.exec(sql, params), {}, label);
1887
2007
  }
1888
2008
 
1889
2009
  /**
@@ -1956,7 +2076,8 @@ export class ControlDatabase {
1956
2076
  * `StrandId` binds the invite to a pre-existing host strand (provision-then-record):
1957
2077
  * when set, a responder redeeming this token records a `FormationUsage` against that
1958
2078
  * strand and returns it (see {@link ControlFormationUsageRecorder.resolveStrand}); a
1959
- * null `StrandId` (the default) leaves the legacy responder-provisions path in place.
2079
+ * null `StrandId` (the default) takes the unbound responder-provisions path: the
2080
+ * responder provisions a fresh open strand and atomically records its one consent row.
1960
2081
  * Like ValidationUrl it is a nullable bound field, signed as `''` when absent.
1961
2082
  *
1962
2083
  * The ExpiresAt and TotalUses message fields must byte-match what the (auto-deferred,
@@ -2023,7 +2144,7 @@ export class ControlDatabase {
2023
2144
  options.validationUrl ?? null,
2024
2145
  options.strandId ?? null,
2025
2146
  stampId,
2026
- ]);
2147
+ ], 'formation-invite-insert');
2027
2148
 
2028
2149
  log('Formation invite inserted: %s', token);
2029
2150
  }
@@ -2053,13 +2174,12 @@ export class ControlDatabase {
2053
2174
  * takes an owner re-seat ({@link insertStrand}) plus a bound invite, never another
2054
2175
  * redemption.
2055
2176
  *
2056
- * `UseNumber` is computed as `max(UseNumber)+1` for the token (the `Monotonic`
2057
- * constraint), inside the write lock and re-read per attempt by
2058
- * {@link withUseNumberRetry} — so concurrent redemptions of one token no longer need
2059
- * the caller to serialise them, and a use number lost to another writer is retried here
2060
- * with the caller's parameters passed through UNCHANGED. That is what makes the retry
2061
- * safe: the approval is bound to the nonce, not to the use number, so re-presenting it
2062
- * cannot drift, and no second trip through the approval hook is possible from in here.
2177
+ * The usage row is keyed by the joiner's own nonce (`UsageStampId` is the primary key), so
2178
+ * concurrent redemptions of one token never contend for a shared row key — there is no
2179
+ * lost race to retry and no second trip through the approval hook. The invite's seat
2180
+ * budget is checked by COUNT ({@link assertSeatRemains}) inside the write lock, ahead of
2181
+ * the write, so a spent invite is refused by name (`InvitationExhaustedError`) instead of
2182
+ * as the schema cap clause's generic `Authorized` CHECK failure.
2063
2183
  */
2064
2184
  async redeemInvitation(params: {
2065
2185
  token: string;
@@ -2082,9 +2202,9 @@ export class ControlDatabase {
2082
2202
  signal?: AbortSignal;
2083
2203
  /**
2084
2204
  * The invite's seat budget, when the caller already has it in hand (e.g. it already read
2085
- * the `FormationInvite` row to get here). Passed through to {@link withUseNumberRetry} so
2086
- * every attempt can check the seat budget without an extra read. `undefined` falls back to
2087
- * a fresh {@link queryFormationInvite} read; `null` means "no invite row" / unlimited.
2205
+ * the `FormationInvite` row to get here). Passed through to {@link assertSeatRemains} so
2206
+ * the seat check needs no extra read. `undefined` falls back to a fresh
2207
+ * {@link queryFormationInvite} read; `null` means "no invite row" / unlimited.
2088
2208
  */
2089
2209
  totalUses?: number | null;
2090
2210
  }): Promise<FormationUsageResult> {
@@ -2097,13 +2217,22 @@ export class ControlDatabase {
2097
2217
  log('Redeeming invitation %s -> strand %s', token, strandId);
2098
2218
 
2099
2219
  const localPeerId = this.config.libp2pNode.peerId.toString();
2100
- // Minted ONCE, outside the loop: a failed attempt rolls its whole transaction back, so
2101
- // nothing was ever persisted under this stamp, and it is deliberately not part of the
2102
- // approver's signed digest. Re-minting per attempt would only churn.
2220
+ // A failed attempt rolls its whole transaction back, so nothing was ever persisted under
2221
+ // this stamp, and it is deliberately not part of the approver's signed digest.
2103
2222
  const strandStampId = generateStampId(localPeerId);
2104
2223
 
2105
- const useNumber = await this.withUseNumberRetry(token, 'redemption', signal, totalUses, attemptUseNumber =>
2106
- this.inTransaction('redemption', async () => {
2224
+ // Ordering inside the locked body is contractual: the abort check, then the seat check,
2225
+ // then the write — FormationAbortedError may only ever be thrown before the write is
2226
+ // issued, and a transient-failure retry (lockedWithRetry re-runs the whole body) must
2227
+ // re-check both before re-presenting the write.
2228
+ await this.lockedWithRetry(async () => {
2229
+ // Inside the lock: a write still parked behind another writer when the caller's
2230
+ // budget expired is abandoned rather than executed.
2231
+ if (signal?.aborted) {
2232
+ throw new FormationAbortedError(token, 'redemption');
2233
+ }
2234
+ await this.assertSeatRemains(token, totalUses);
2235
+ await this.inTransaction('redemption', async () => {
2107
2236
  // 1. Strand row — authorised by the FormationUsage branch (no owner sig),
2108
2237
  // still carrying a fresh unique StampId for the anti-replay column.
2109
2238
  // Hard-coded open + keyless: the consent branch admits no other shape.
@@ -2116,14 +2245,15 @@ export class ControlDatabase {
2116
2245
  // 2. FormationUsage row — authorised by the matching FormationInvite, and
2117
2246
  // carrying the strand's stamp so it authorizes THIS row and no other.
2118
2247
  await this.execFormationUsageInsert({
2119
- token, useNumber: attemptUseNumber, usageStampId, disclosure, strandId, strandStampId,
2248
+ token, usageStampId, disclosure, strandId, strandStampId,
2120
2249
  peerKey, peerSignature, nowMs: nowMs ?? Date.now(),
2121
2250
  validationKey: validationKey ?? null, validationSignature: validationSignature ?? null,
2122
2251
  });
2123
- }));
2252
+ });
2253
+ }, {}, 'formation-usage');
2124
2254
 
2125
- log('Redeemed invitation %s -> strand %s (use #%d)', token, strandId, useNumber);
2126
- return { useNumber, usageStampId };
2255
+ log('Redeemed invitation %s -> strand %s (stamp %s)', token, strandId, usageStampId);
2256
+ return { usageStampId };
2127
2257
  }
2128
2258
 
2129
2259
  /**
@@ -2131,8 +2261,8 @@ export class ControlDatabase {
2131
2261
  * insert). This is the redemption path when the strand was provisioned
2132
2262
  * separately (e.g. owner-signed) and the consent record is added after the
2133
2263
  * fact: the single insert auto-commits, and the deferred `StrandExists` CHECK
2134
- * is satisfied by the pre-existing committed strand row. Returns the assigned
2135
- * `UseNumber`.
2264
+ * is satisfied by the pre-existing committed strand row. Echoes back the
2265
+ * redemption's `usageStampId` (the row's primary key).
2136
2266
  *
2137
2267
  * Use {@link redeemInvitation} instead when the strand must be created by
2138
2268
  * consent atomically with the usage.
@@ -2146,12 +2276,12 @@ export class ControlDatabase {
2146
2276
  * by {@link ControlFormationUsageRecorder.resolveStrand}, so an absent row at this
2147
2277
  * point is a genuine race and deserves a named error, not a silent rollback.
2148
2278
  *
2149
- * `UseNumber` is read inside the write lock and re-read per attempt by
2150
- * {@link withUseNumberRetry}: this is the path production actually races on (a bound
2151
- * invite published by `cadre-web` / `cadre-phone` is always record-only), and a use number
2152
- * lost to another writer is retried here with the caller's parameters — nonce, approver
2153
- * sign-off, strand, peer key, disclosure — passed through UNCHANGED, so a granted approval
2154
- * is never thrown away for a second trip through the approval hook.
2279
+ * The row is keyed by the joiner's own nonce (`UsageStampId` is the primary key), so this
2280
+ * path — the one production actually races on (a bound invite published by `cadre-web` /
2281
+ * `cadre-phone` is always record-only) — never contends with another redemption for a
2282
+ * shared row key: there is no lost race to retry and no second trip through the approval
2283
+ * hook. The invite's seat budget is checked by COUNT ({@link assertSeatRemains}) inside
2284
+ * the write lock, ahead of the write.
2155
2285
  */
2156
2286
  async recordFormationUsage(params: {
2157
2287
  token: string;
@@ -2167,17 +2297,16 @@ export class ControlDatabase {
2167
2297
  validationKey?: string;
2168
2298
  validationSignature?: string;
2169
2299
  /**
2170
- * Aborted when the caller has given up. Checked once PER ATTEMPT, inside the write lock,
2171
- * at the top of the attempt — before it reads a use number and before its insert is
2172
- * issued — throwing {@link FormationAbortedError} with the invite unspent. Never checked
2173
- * once an insert has been issued.
2300
+ * Aborted when the caller has given up. Checked inside the write lock, before the insert
2301
+ * is issued, throwing {@link FormationAbortedError} with the invite unspent. Never
2302
+ * checked once the insert has been issued.
2174
2303
  */
2175
2304
  signal?: AbortSignal;
2176
2305
  /**
2177
2306
  * The invite's seat budget, when the caller already has it in hand (e.g. it already read
2178
- * the `FormationInvite` row to get here). Passed through to {@link withUseNumberRetry} so
2179
- * every attempt can check the seat budget without an extra read. `undefined` falls back to
2180
- * a fresh {@link queryFormationInvite} read; `null` means "no invite row" / unlimited.
2307
+ * the `FormationInvite` row to get here). Passed through to {@link assertSeatRemains} so
2308
+ * the seat check needs no extra read. `undefined` falls back to a fresh
2309
+ * {@link queryFormationInvite} read; `null` means "no invite row" / unlimited.
2181
2310
  */
2182
2311
  totalUses?: number | null;
2183
2312
  }): Promise<FormationUsageResult> {
@@ -2188,109 +2317,40 @@ export class ControlDatabase {
2188
2317
  totalUses,
2189
2318
  } = params;
2190
2319
 
2191
- // Read ONCE, outside the loop: the host strand is pre-existing and owner-signed, and a
2192
- // retry changes which use number we take, never which strand row we name.
2320
+ // The host strand is pre-existing and owner-signed; its stamp names the strand ROW the
2321
+ // consent record authorizes.
2193
2322
  const strandStampId = await this.queryStrandStampId(strandId);
2194
2323
  if (strandStampId === null) {
2195
2324
  throw new MissingHostStrandError(strandId, token);
2196
2325
  }
2197
2326
 
2198
- const useNumber = await this.withUseNumberRetry(token, 'usage recording', signal, totalUses, attemptUseNumber =>
2199
- this.execFormationUsageInsert({
2200
- token, useNumber: attemptUseNumber, usageStampId, disclosure, strandId, strandStampId,
2327
+ // Ordering inside the locked body is contractual — abort check, seat check, write — see
2328
+ // the matching note in redeemInvitation.
2329
+ await this.lockedWithRetry(async () => {
2330
+ if (signal?.aborted) {
2331
+ throw new FormationAbortedError(token, 'usage recording');
2332
+ }
2333
+ await this.assertSeatRemains(token, totalUses);
2334
+ await this.execFormationUsageInsert({
2335
+ token, usageStampId, disclosure, strandId, strandStampId,
2201
2336
  peerKey, peerSignature, nowMs: nowMs ?? Date.now(),
2202
2337
  validationKey: validationKey ?? null, validationSignature: validationSignature ?? null,
2203
- }));
2338
+ });
2339
+ }, {}, 'formation-usage');
2204
2340
 
2205
- log('Recorded formation usage: token=%s strand=%s (use #%d)', token, strandId, useNumber);
2206
- return { useNumber, usageStampId };
2341
+ log('Recorded formation usage: token=%s strand=%s (stamp %s)', token, strandId, usageStampId);
2342
+ return { usageStampId };
2207
2343
  }
2208
2344
 
2209
2345
  /**
2210
- * Assign a `UseNumber` and run `write` under it, re-picking and retrying when another
2211
- * writer took that number first.
2212
- *
2213
- * Each attempt takes {@link withWriteLock} and, inside it, reads {@link nextUseNumber}
2214
- * afresh. Reading under the lock is the whole fix for the common case: two redemptions of
2215
- * one token on the SAME node are serialized by the local write queue, so each computes a
2216
- * number the other has already committed and neither collides. The retry covers the writers
2217
- * this lock does not reach — another node of the cadre, another `Database` handle over the
2218
- * same store — which can still take the number between our read and our commit.
2219
- *
2220
- * `write` receives ONLY the use number; every other parameter of the redemption (the nonce,
2221
- * the approver's signature, the strand, the joining peer key, the disclosure) is closed over
2222
- * by the caller and passed through untouched. So by construction a retry re-presents the
2223
- * IDENTICAL approval and the four non-nonce fields inside the approver's signed digest
2224
- * cannot drift. No second call to the approval hook is reachable from in here either:
2225
- * `obtainApproval` lives a layer up in `ControlFormationUsageRecorder` and is never
2226
- * re-entered.
2227
- *
2228
- * `write` must be a BARE private body ({@link inTransaction} /
2229
- * {@link execFormationUsageInsert}) — {@link withWriteLock} is not re-entrant and a `write`
2230
- * that called another locked public method would hang silently and strand the write queue.
2231
- *
2232
- * A failed attempt has written nothing (a bare insert never landed; a transactional one
2233
- * rolled back), so the abort check at the top of each attempt stays inside
2234
- * {@link FormationAbortedError}'s contract — it is only ever thrown before this attempt's
2235
- * write is issued.
2236
- *
2237
- * @param operation - `'redemption'` | `'usage recording'`, for the abort error's text.
2238
- * @param knownTotalUses - The invite's seat budget when the caller already has it in hand
2239
- * (`undefined` to fall back to a fresh {@link queryFormationInvite} read; `null` for "no
2240
- * invite row" / unlimited). Passed through to {@link assertSeatRemains} on every attempt.
2241
- * @returns The `UseNumber` the successful attempt wrote under.
2242
- */
2243
- private async withUseNumberRetry(
2244
- token: string,
2245
- operation: string,
2246
- signal: AbortSignal | undefined,
2247
- knownTotalUses: number | null | undefined,
2248
- write: (useNumber: number) => Promise<void>,
2249
- ): Promise<number> {
2250
- let lastError: unknown;
2251
- for (let attempt = 1; attempt <= USE_NUMBER_ATTEMPTS; attempt++) {
2252
- try {
2253
- // lockedWithRetry carries the transient-cluster retry INSIDE this use-number
2254
- // retry. The loops cannot multiply: a lost use number is a constraint failure,
2255
- // which `isRetriableControlWriteFailure` never matches, and a transient cluster
2256
- // failure is not a lost use number, so exactly one loop claims any given error.
2257
- return await this.lockedWithRetry(async () => {
2258
- // Inside the lock: a write still parked behind another writer when the caller's
2259
- // budget expired is abandoned rather than executed.
2260
- if (signal?.aborted) {
2261
- throw new FormationAbortedError(token, operation);
2262
- }
2263
- const useNumber = await this.nextUseNumber(token);
2264
- await this.assertSeatRemains(token, useNumber, knownTotalUses);
2265
- await write(useNumber);
2266
- return useNumber;
2267
- });
2268
- } catch (error) {
2269
- if (!isLostUseNumberRace(error)) {
2270
- throw error;
2271
- }
2272
- lastError = error;
2273
- log('Use number for token %s was taken by another writer (attempt %d/%d): %s',
2274
- token, attempt, USE_NUMBER_ATTEMPTS, error);
2275
- }
2276
- }
2277
- // Out of attempts under sustained contention. Rethrowing the engine's own error keeps
2278
- // today's behaviour: `StrandFormationManager.provisionAsResponder`'s catch-all maps it to
2279
- // `Formation conflict, retry` and the joiner starts over with a fresh nonce.
2280
- throw lastError;
2281
- }
2282
-
2283
- /**
2284
- * Refuse an attempt that would consume a seat the invite does not have.
2346
+ * Refuse a redemption that would consume a seat the invite does not have.
2285
2347
  *
2286
- * `FormationUsage.Authorized` already enforces `FI.TotalUses is null or FI.TotalUses >=
2287
- * new.UseNumber`, so an over-limit write fails at the database anyway — but it fails as a
2288
- * generic `CHECK constraint failed: Authorized`, which the manager reports as a retryable
2289
- * conflict. Catching it here as {@link InvitationExhaustedError} lets the joiner be told the
2290
- * invitation is spent instead of being sent around a loop that can never close. Run on EVERY
2291
- * attempt, including the first: two redemptions racing on the SAME node serialize behind the
2292
- * local write queue, so the loser's first attempt is already over budget, with no retry
2293
- * involved.
2348
+ * The schema's count-based cap clause (`FormationUsage.Authorized`) refuses an over-limit
2349
+ * write at the database anyway — but it fails as a generic `CHECK constraint failed:
2350
+ * Authorized`, which the manager reports as a retryable conflict. Catching it here as
2351
+ * {@link InvitationExhaustedError} lets the joiner be told the invitation is spent instead
2352
+ * of being sent around a loop that can never close. Runs inside the write lock, so on a
2353
+ * same-node race the loser reads the winner's committed row and is refused by name.
2294
2354
  *
2295
2355
  * `knownTotalUses` lets a caller that already read the `FormationInvite` row (both
2296
2356
  * production callers in `ControlFormationUsageRecorder` do) skip a second read on the common,
@@ -2300,11 +2360,10 @@ export class ControlDatabase {
2300
2360
  * invite, before an outbound approval call). Safe today because `FormationInvite` is
2301
2361
  * insert/delete only (its `Immutable` constraint), so the only way to stale it is an owner
2302
2362
  * revoking the token and re-issuing it with MORE seats mid-redemption; if invites ever gain
2303
- * an update path, drop the parameter and read here on every attempt instead.
2363
+ * an update path, drop the parameter and read here instead.
2304
2364
  */
2305
2365
  private async assertSeatRemains(
2306
2366
  token: string,
2307
- useNumber: number,
2308
2367
  knownTotalUses: number | null | undefined,
2309
2368
  ): Promise<void> {
2310
2369
  // A missing invite is left to `Authorized` — it is not an exhaustion, and the CHECK's
@@ -2314,19 +2373,26 @@ export class ControlDatabase {
2314
2373
  // is still open (tracked by `debt-composite-pk-point-lookup-unreliable-untracked`). A
2315
2374
  // spurious empty result only costs the attempt its NAMED exhaustion error, reverting it to
2316
2375
  // today's generic `Authorized` refusal — never a seat the invite does not have.
2376
+ // Both reads pass retry: false — assertSeatRemains only ever runs inside the locked
2377
+ // write bodies of redeemInvitation / recordFormationUsage, where a read retrying its
2378
+ // own backoff would sleep holding the write lock and the write funnel re-runs these
2379
+ // reads anyway (see the NOTE on readRows).
2317
2380
  const totalUses = knownTotalUses !== undefined
2318
2381
  ? knownTotalUses
2319
- : (await this.queryFormationInvite(token))?.totalUses ?? null;
2320
- if (totalUses != null && useNumber > totalUses) {
2321
- throw new InvitationExhaustedError(token, useNumber, totalUses);
2382
+ : (await this.queryFormationInvite(token, false))?.totalUses ?? null;
2383
+ if (totalUses == null) {
2384
+ return;
2385
+ }
2386
+ const used = await this.countFormationUsage(token, false);
2387
+ if (used >= totalUses) {
2388
+ throw new InvitationExhaustedError(token, used, totalUses);
2322
2389
  }
2323
2390
  }
2324
2391
 
2325
2392
  /** Parameterised `FormationUsage` insert shared by redeem + record paths. */
2326
2393
  private async execFormationUsageInsert(opts: {
2327
2394
  token: string;
2328
- useNumber: number;
2329
- /** Single-use nonce bound into the approver's signed digest; `unique`, so one approval spends once. */
2395
+ /** Single-use nonce bound into the approver's signed digest; the row's primary key, so one approval spends once. */
2330
2396
  usageStampId: string;
2331
2397
  disclosure: string;
2332
2398
  strandId: string;
@@ -2348,12 +2414,12 @@ export class ControlDatabase {
2348
2414
  // not a fix for an observable mis-ordering.
2349
2415
  const nowCanonical = await canonicalDatetime(this.db!, opts.nowMs);
2350
2416
  await this.db!.exec(`
2351
- insert into CadreControl.FormationUsage (Token, UseNumber, UsageStampId, PeerKey, PeerSig, Disclosure, StrandId, StrandStampId)
2417
+ insert into CadreControl.FormationUsage (Token, UsageStampId, PeerKey, PeerSig, Disclosure, StrandId, StrandStampId)
2352
2418
  with context Now = ?, ValidationKey = ?, ValidationSignature = ?
2353
- values (?, ?, ?, ?, ?, ?, ?, ?)
2419
+ values (?, ?, ?, ?, ?, ?, ?)
2354
2420
  `, [
2355
2421
  nowCanonical, opts.validationKey, opts.validationSignature,
2356
- opts.token, opts.useNumber, opts.usageStampId, opts.peerKey,
2422
+ opts.token, opts.usageStampId, opts.peerKey,
2357
2423
  opts.peerSignature, opts.disclosure, opts.strandId, opts.strandStampId,
2358
2424
  ]);
2359
2425
  }
@@ -2362,8 +2428,12 @@ export class ControlDatabase {
2362
2428
  * Read a `FormationInvite` row by token, or null when absent. `expiresAtMs` is
2363
2429
  * the parsed epoch-ms of the stored `datetime` (null when the invite never
2364
2430
  * expires); the caller compares it against the wall clock for freshness.
2431
+ *
2432
+ * `retry: false` is passed only by {@link assertSeatRemains}, which runs INSIDE a
2433
+ * locked write body — same per-call opt-out, and for the same reason, as
2434
+ * {@link queryStampId}'s. Every unlocked caller keeps the default retried path.
2365
2435
  */
2366
- async queryFormationInvite(token: string): Promise<{
2436
+ async queryFormationInvite(token: string, retry = true): Promise<{
2367
2437
  token: string;
2368
2438
  sAppId: string;
2369
2439
  expiresAtMs: number | null;
@@ -2372,10 +2442,8 @@ export class ControlDatabase {
2372
2442
  strandId: string | null;
2373
2443
  } | null> {
2374
2444
  this.ensureInitialized();
2375
- for await (const row of this.db!.eval(
2376
- 'select Token, sAppId, ExpiresAt, TotalUses, ValidationUrl, StrandId from CadreControl.FormationInvite where Token = ?',
2377
- [token]
2378
- )) {
2445
+ const sql = 'select Token, sAppId, ExpiresAt, TotalUses, ValidationUrl, StrandId from CadreControl.FormationInvite where Token = ?';
2446
+ for (const row of await this.readRows(sql, [token], 'formation-invite', retry)) {
2379
2447
  return {
2380
2448
  token: row.Token as string,
2381
2449
  sAppId: row.sAppId as string,
@@ -2390,13 +2458,32 @@ export class ControlDatabase {
2390
2458
 
2391
2459
  /**
2392
2460
  * Count `FormationUsage` rows recorded against a token (uses consumed so far).
2461
+ *
2462
+ * This count IS the seat cap (`enforceFormationUseCap`), so an under-report admits a seat
2463
+ * the invitation never paid for. It is served by a table SCAN, deliberately: a bare
2464
+ * equality on a non-key column is a shape the optimystic vtab declines to claim, so the
2465
+ * engine applies the filter itself over a full read of the table.
2466
+ *
2467
+ * NOTE: it was briefly an index seek, and that is what the scan is protecting against.
2468
+ * `formation-unique-token-redesign` took `Token` out of the primary key and declared
2469
+ * `FormationUsageByToken` purely to spare this scan, recording as a tripwire that an
2470
+ * under-reporting descent would admit an unpaid-for seat — "a failure mode a scan could
2471
+ * not produce". The tripwire fired. Across machines a secondary-index descent returns only
2472
+ * the rows the READING machine wrote, so this count under-reported without bound on a
2473
+ * multi-machine party and the cap over-admitted. The index was removed 2026-08-25; the
2474
+ * measurement is recorded in the block comment at the `FormationUsage` declaration in
2475
+ * `schemas/control.qsql`, and the engine defect in
2476
+ * `blocked/secondary-index-seek-blind-to-sibling-rows`. Do NOT re-declare the index to
2477
+ * speed this up while that ticket is open.
2478
+ *
2479
+ * `retry: false` is passed only by {@link assertSeatRemains}, which runs INSIDE a
2480
+ * locked write body — same per-call opt-out, and for the same reason, as
2481
+ * {@link queryStampId}'s. Every unlocked caller keeps the default retried path.
2393
2482
  */
2394
- async countFormationUsage(token: string): Promise<number> {
2483
+ async countFormationUsage(token: string, retry = true): Promise<number> {
2395
2484
  this.ensureInitialized();
2396
- for await (const row of this.db!.eval(
2397
- 'select count(1) as Count from CadreControl.FormationUsage where Token = ?',
2398
- [token]
2399
- )) {
2485
+ const sql = 'select count(1) as Count from CadreControl.FormationUsage where Token = ?';
2486
+ for (const row of await this.readRows(sql, [token], 'formation-usage-count', retry)) {
2400
2487
  return (row.Count as number) ?? 0;
2401
2488
  }
2402
2489
  return 0;
@@ -2428,8 +2515,10 @@ export class ControlDatabase {
2428
2515
  // prune redeemed/expired rows.
2429
2516
  const metered: Array<{ token: string; totalUses: number }> = [];
2430
2517
  let unlimitedOutstanding = false;
2431
- for await (const row of this.db!.eval(
2432
- 'select Token, ExpiresAt, TotalUses from CadreControl.FormationInvite'
2518
+ for (const row of await this.readRows(
2519
+ 'select Token, ExpiresAt, TotalUses from CadreControl.FormationInvite',
2520
+ undefined,
2521
+ 'outstanding-invites'
2433
2522
  )) {
2434
2523
  const expiresAtMs = parseNullableStoredDatetimeMs(row.ExpiresAt as string | number | null);
2435
2524
  if (expiresAtMs !== null && expiresAtMs <= nowMs) {
@@ -2445,6 +2534,12 @@ export class ControlDatabase {
2445
2534
  if (unlimitedOutstanding) {
2446
2535
  return true;
2447
2536
  }
2537
+ // NOTE: this loop issues one retried read per metered invite, each carrying its own
2538
+ // CONTROL_READ_RETRY_BUDGET_MS, on a path the inbound admission gate awaits under a
2539
+ // 2 s FAIL-OPEN deadline — so with N unexpired metered invites the worst case is
2540
+ // (1 + N) budgets, not one. Fine while a cadre holds a couple of live invites and
2541
+ // transient read failures are rare; if a party ever keeps many metered invites open,
2542
+ // give this whole method one shared deadline rather than raising the per-read budget.
2448
2543
  for (const invite of metered) {
2449
2544
  if (await this.countFormationUsage(invite.token) < invite.totalUses) {
2450
2545
  return true;
@@ -2453,23 +2548,6 @@ export class ControlDatabase {
2453
2548
  return false;
2454
2549
  }
2455
2550
 
2456
- /**
2457
- * Next `UseNumber` for a token = max(existing)+1, per the `Monotonic` constraint.
2458
- *
2459
- * Called only from inside {@link withUseNumberRetry}'s locked body — a read takes no
2460
- * transaction of its own, so it is safe there, and reading it under the lock is what stops
2461
- * two local redemptions of one token from picking the same number.
2462
- */
2463
- private async nextUseNumber(token: string): Promise<number> {
2464
- for await (const row of this.db!.eval(
2465
- 'select coalesce(max(UseNumber), 0) as MaxUse from CadreControl.FormationUsage where Token = ?',
2466
- [token]
2467
- )) {
2468
- return (row.MaxUse as number) + 1;
2469
- }
2470
- return 1;
2471
- }
2472
-
2473
2551
  /**
2474
2552
  * Close the database and cleanup resources.
2475
2553
  *