@serfab/cadre-core 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +255 -255
- package/dist/announce-addrs.d.ts +49 -0
- package/dist/announce-addrs.js +78 -0
- package/dist/announce-addrs.js.map +1 -0
- package/dist/cadre-node.d.ts +484 -58
- package/dist/cadre-node.js +953 -182
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-cohort.d.ts +26 -0
- package/dist/control-cohort.js +26 -0
- package/dist/control-cohort.js.map +1 -1
- package/dist/control-database.d.ts +222 -125
- package/dist/control-database.js +356 -266
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +8 -7
- package/dist/control-formation-recorder.js +8 -7
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-read-retry.d.ts +103 -0
- package/dist/control-read-retry.js +145 -0
- package/dist/control-read-retry.js.map +1 -0
- package/dist/control-retry.d.ts +101 -0
- package/dist/control-retry.js +108 -0
- package/dist/control-retry.js.map +1 -0
- package/dist/control-schema.d.ts +1 -1
- package/dist/control-schema.js +805 -774
- package/dist/control-schema.js.map +1 -1
- package/dist/control-write-retry.d.ts +65 -31
- package/dist/control-write-retry.js +33 -103
- package/dist/control-write-retry.js.map +1 -1
- package/dist/delegate-admission.d.ts +17 -15
- package/dist/delegate-admission.js +17 -15
- package/dist/delegate-admission.js.map +1 -1
- package/dist/index.d.ts +7 -5
- package/dist/index.js +18 -9
- package/dist/index.js.map +1 -1
- package/dist/membership-connection-gater.d.ts +211 -25
- package/dist/membership-connection-gater.js +309 -29
- package/dist/membership-connection-gater.js.map +1 -1
- package/dist/peer-addr-book.d.ts +92 -0
- package/dist/peer-addr-book.js +220 -0
- package/dist/peer-addr-book.js.map +1 -0
- package/dist/peer-record.d.ts +26 -0
- package/dist/peer-record.js +32 -0
- package/dist/peer-record.js.map +1 -1
- package/dist/relay-addrs.d.ts +76 -26
- package/dist/relay-addrs.js +115 -27
- package/dist/relay-addrs.js.map +1 -1
- package/dist/relay-reservation.d.ts +20 -15
- package/dist/relay-reservation.js +18 -14
- package/dist/relay-reservation.js.map +1 -1
- package/dist/seed-bootstrap.d.ts +7 -0
- package/dist/seed-bootstrap.js +15 -13
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/strand-addr-protocol.js +16 -0
- package/dist/strand-addr-protocol.js.map +1 -1
- package/dist/strand-backfill.js +13 -4
- package/dist/strand-backfill.js.map +1 -1
- package/dist/strand-cohort.d.ts +4 -37
- package/dist/strand-cohort.js +4 -15
- package/dist/strand-cohort.js.map +1 -1
- package/dist/strand-database.d.ts +11 -16
- package/dist/strand-database.js +19 -7
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-formation-manager.d.ts +18 -17
- package/dist/strand-formation-manager.js +25 -27
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +7 -7
- package/dist/strand-formation-protocol.js +8 -8
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +53 -21
- package/dist/strand-instance-manager.js +128 -37
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +113 -19
- package/dist/strand-membership-writer.js +219 -51
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-solicitation.d.ts +3 -3
- package/dist/strand-solicitation.js +4 -5
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-wake-protocol.d.ts +47 -2
- package/dist/strand-wake-protocol.js +76 -6
- package/dist/strand-wake-protocol.js.map +1 -1
- package/dist/types.d.ts +107 -54
- package/dist/types.js.map +1 -1
- package/package.json +8 -7
- package/src/announce-addrs.ts +93 -0
- package/src/arachnode-stub.ts +159 -159
- package/src/bootstrap-peer-store-file.ts +44 -44
- package/src/bootstrap-peer-store.ts +172 -172
- package/src/cadre-node.ts +1026 -194
- package/src/canonical-json.ts +30 -30
- package/src/control-authorization.ts +114 -114
- package/src/control-cohort.ts +130 -103
- package/src/control-database.ts +376 -298
- package/src/control-formation-recorder.ts +305 -304
- package/src/control-read-retry.ts +167 -0
- package/src/control-retry.ts +190 -0
- package/src/control-schema.ts +817 -786
- package/src/control-stream.ts +205 -205
- package/src/control-write-retry.ts +383 -473
- package/src/delegate-admission.ts +304 -302
- package/src/device-token.ts +94 -94
- package/src/diagnostics/connection-path.ts +277 -277
- package/src/diagnostics/webrtc-turn-tracker.ts +222 -222
- package/src/ed25519-key.ts +134 -134
- package/src/enrollment.ts +199 -199
- package/src/file-durable-slot.ts +61 -61
- package/src/formation-approval.ts +604 -604
- package/src/fs-atomic.ts +107 -107
- package/src/hibernation-manager.ts +437 -437
- package/src/identity-key.ts +109 -109
- package/src/index.ts +435 -406
- package/src/key-store-file.ts +132 -132
- package/src/key-store.ts +99 -99
- package/src/membership-connection-gater.ts +368 -39
- package/src/node-local-snapshot.ts +260 -260
- package/src/peer-addr-book.ts +250 -0
- package/src/peer-authorization.ts +259 -259
- package/src/peer-record.ts +197 -164
- package/src/push-credentials.ts +84 -84
- package/src/push-node.ts +64 -64
- package/src/push-notifier-apns.ts +247 -247
- package/src/push-notifier-fcm.ts +220 -220
- package/src/push-notifier-shared.ts +28 -28
- package/src/push-notifier.ts +52 -52
- package/src/relay-addrs.ts +134 -27
- package/src/relay-reservation.ts +22 -17
- package/src/seed-bootstrap.ts +15 -15
- package/src/seed-trust-policy.ts +140 -140
- package/src/strand-addr-protocol.ts +458 -441
- package/src/strand-backfill.ts +28 -5
- package/src/strand-cohort.ts +38 -77
- package/src/strand-database.ts +202 -204
- package/src/strand-formation-manager.ts +26 -28
- package/src/strand-formation-protocol.ts +692 -692
- package/src/strand-instance-manager.ts +636 -553
- package/src/strand-member-key.ts +39 -39
- package/src/strand-member-registry.ts +182 -182
- package/src/strand-membership-writer.ts +1627 -1430
- package/src/strand-solicitation.ts +471 -472
- package/src/strand-transport-key.ts +58 -58
- package/src/strand-wake-payload.ts +32 -32
- package/src/strand-wake-protocol.ts +400 -314
- package/src/strand-watcher.ts +314 -314
- package/src/trusted-owner-store-file.ts +41 -41
- package/src/trusted-owner-store.ts +171 -171
- package/src/types.ts +1354 -1303
package/src/control-database.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import debug from 'debug';
|
|
2
2
|
import { toString as uint8ArrayToString } from 'uint8arrays';
|
|
3
|
-
import { Database, registerPlugin
|
|
4
|
-
import type { VTablePluginInfo, FunctionPluginInfo, SqlParameters } from '@quereus/quereus';
|
|
3
|
+
import { Database, registerPlugin } from '@quereus/quereus';
|
|
4
|
+
import type { VTablePluginInfo, FunctionPluginInfo, SqlParameters, SqlValue } from '@quereus/quereus';
|
|
5
5
|
import cryptoPlugin from '@optimystic/quereus-plugin-crypto/plugin';
|
|
6
6
|
import optimysticPlugin from '@optimystic/quereus-plugin-optimystic/plugin';
|
|
7
7
|
import { digest, randomBytes } from '@optimystic/quereus-plugin-crypto';
|
|
@@ -15,6 +15,8 @@ import type { ControlTable, RevocableTable, ControlDomain, ControlAction } from
|
|
|
15
15
|
import { requireEd25519PublicKeyB64 } from './ed25519-key.js';
|
|
16
16
|
import { retryControlWrite, SCHEMA_INIT_RETRY_POLICY } from './control-write-retry.js';
|
|
17
17
|
import type { ControlWriteRetryOptions } from './control-write-retry.js';
|
|
18
|
+
import { retryControlRead } from './control-read-retry.js';
|
|
19
|
+
import type { ControlReadRetryOptions } from './control-read-retry.js';
|
|
18
20
|
|
|
19
21
|
export type { ControlTable, RevocableTable, ControlDomain, ControlAction } from './control-authorization.js';
|
|
20
22
|
|
|
@@ -90,102 +92,34 @@ export class FormationAbortedError extends Error {
|
|
|
90
92
|
}
|
|
91
93
|
|
|
92
94
|
/**
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
95
|
+
* The invitation's seat budget is already spent: the count of recorded `FormationUsage` rows
|
|
96
|
+
* for the token has reached `FormationInvite.TotalUses`, so this redemption cannot be given a
|
|
97
|
+
* seat.
|
|
96
98
|
*
|
|
97
|
-
* Raised
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
*
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
99
|
+
* Raised off the committed-count check ({@link ControlDatabase.assertSeatRemains}) ahead of
|
|
100
|
+
* the write, inside {@link ControlDatabase.redeemInvitation} /
|
|
101
|
+
* {@link ControlDatabase.recordFormationUsage} — and off nothing else. It exists because
|
|
102
|
+
* without it the refusal surfaces as a generic `CHECK constraint failed: Authorized` from the
|
|
103
|
+
* schema's own count-based cap clause, which the manager reports as a retryable
|
|
104
|
+
* `Formation conflict, retry` — telling the joiner to retry something that can never succeed.
|
|
105
|
+
* Two redemptions of one token racing on the SAME node serialize behind the local write
|
|
106
|
+
* queue, so the loser reads the winner's committed row and is refused here by name —
|
|
107
|
+
* `StrandFormationManager.validateToken`'s `isTokenUsed` check cannot catch that, since it
|
|
108
|
+
* ran before either redemption wrote anything.
|
|
105
109
|
*/
|
|
106
110
|
export class InvitationExhaustedError extends Error {
|
|
107
111
|
constructor(
|
|
108
112
|
readonly token: string,
|
|
109
|
-
readonly
|
|
113
|
+
readonly usesRecorded: number,
|
|
110
114
|
readonly totalUses: number,
|
|
111
115
|
) {
|
|
112
116
|
super(
|
|
113
|
-
`Invitation ${token} is exhausted:
|
|
117
|
+
`Invitation ${token} is exhausted: ${usesRecorded} of ${totalUses} use(s) already recorded`
|
|
114
118
|
);
|
|
115
119
|
this.name = 'InvitationExhaustedError';
|
|
116
120
|
}
|
|
117
121
|
}
|
|
118
122
|
|
|
119
|
-
/**
|
|
120
|
-
* Attempts allowed per redemption: the first, plus two retries of a use number another
|
|
121
|
-
* writer took. Bounded so sustained contention terminates in a clean rejection rather than
|
|
122
|
-
* spinning; two retries is enough for the realistic case (a handful of nodes redeeming one
|
|
123
|
-
* multi-use token at once) without holding the write lock hostage.
|
|
124
|
-
*
|
|
125
|
-
* NOTE: the attempts fire back-to-back, with no backoff and no jitter — each one only takes
|
|
126
|
-
* and releases the write lock. Fine while cross-node contention on a single token is a
|
|
127
|
-
* handful of writers; if one hot multi-use token ever draws sustained concurrent redemption
|
|
128
|
-
* (a broadcast invite, a bulk onboarding run), all three attempts can burn against the same
|
|
129
|
-
* competitor and the caller gets the engine's error. Add a short randomised delay between
|
|
130
|
-
* attempts then — NOT a longer attempt budget, which would hold the write queue.
|
|
131
|
-
*/
|
|
132
|
-
const USE_NUMBER_ATTEMPTS = 3;
|
|
133
|
-
|
|
134
|
-
/**
|
|
135
|
-
* The two error messages that mean "another writer took the `UseNumber` we picked", and
|
|
136
|
-
* nothing else.
|
|
137
|
-
*
|
|
138
|
-
* - the `(Token, UseNumber)` primary-key collision, which the optimystic vtab reports on the
|
|
139
|
-
* insert itself (`uniqueConstraintMessage` renders the columns table-qualified, no schema
|
|
140
|
-
* prefix);
|
|
141
|
-
* - `CHECK constraint failed: Monotonic` — `FormationUsage.Monotonic` contains a subquery, so
|
|
142
|
-
* Quereus defers it to commit where it reads `committed.FormationUsage`. A concurrent row
|
|
143
|
-
* that our insert's key probe did not see still trips it there. Same meaning, different
|
|
144
|
-
* surface.
|
|
145
|
-
*
|
|
146
|
-
* Deliberately NOT matched: `FormationUsage.UsageStampId`'s `unique` (a repeated nonce is
|
|
147
|
-
* replay, never a race — retrying it would re-present a spent approval), and every named
|
|
148
|
-
* CHECK other than `Monotonic`. `Authorized` in particular renders with the same
|
|
149
|
-
* `CHECK constraint failed:` prefix, so the constraint NAME is anchored with `\b` exactly as
|
|
150
|
-
* `expectConstraintFailure` does rather than matched by prefix.
|
|
151
|
-
*
|
|
152
|
-
* NOTE: a CHECK failure names only the CONSTRAINT, never its table, and `Monotonic` is a
|
|
153
|
-
* generic enough name that a second table could plausibly claim it. Today it is unique across
|
|
154
|
-
* `schemas/control.qsql`, so the pattern cannot mis-fire; if another table ever declares a
|
|
155
|
-
* `Monotonic` constraint, a failure of THAT constraint would be read here as a lost use-number
|
|
156
|
-
* race and pointlessly retried up to {@link USE_NUMBER_ATTEMPTS} times. Rename the other
|
|
157
|
-
* constraint (cheapest) rather than trying to recover the table from the message.
|
|
158
|
-
*/
|
|
159
|
-
const LOST_USE_NUMBER_PATTERNS = [
|
|
160
|
-
/UNIQUE constraint failed: FormationUsage\.Token, FormationUsage\.UseNumber/i,
|
|
161
|
-
/CHECK constraint failed: Monotonic\b/,
|
|
162
|
-
];
|
|
163
|
-
|
|
164
|
-
/**
|
|
165
|
-
* Did this write fail purely because another writer took the `UseNumber` it picked — i.e. is
|
|
166
|
-
* re-presenting the SAME approval under the next free use number the right response?
|
|
167
|
-
*
|
|
168
|
-
* Classifies by MESSAGE, walking the `cause` chain with Quereus' own {@link unwrapError}:
|
|
169
|
-
* neither surface can be recognised by type. The deferred `Monotonic` failure is thrown as a
|
|
170
|
-
* bare `QuereusError`, not a `ConstraintError` (see
|
|
171
|
-
* `quereus/src/runtime/deferred-constraint-queue.ts`), so an `instanceof ConstraintError` gate
|
|
172
|
-
* would silently drop the branch this retry exists for. Messages also carry an
|
|
173
|
-
* ` (at line N, column M)` suffix and arrive wrapped when the write ran inside a transaction,
|
|
174
|
-
* so the match is a substring test rather than an equality one.
|
|
175
|
-
*
|
|
176
|
-
* NOTE: this is the one piece of the retry that depends on engine error TEXT. Its own test
|
|
177
|
-
* (`control-formation-use-number-retry.spec.ts`) produces all three messages from the real
|
|
178
|
-
* engine rather than from string literals, so a reworded storage-layer error fails that spec
|
|
179
|
-
* instead of silently disabling the retry.
|
|
180
|
-
*/
|
|
181
|
-
export function isLostUseNumberRace(error: unknown): boolean {
|
|
182
|
-
if (!(error instanceof Error)) {
|
|
183
|
-
return false;
|
|
184
|
-
}
|
|
185
|
-
return unwrapError(error).some(({ message }) =>
|
|
186
|
-
LOST_USE_NUMBER_PATTERNS.some(pattern => pattern.test(message)));
|
|
187
|
-
}
|
|
188
|
-
|
|
189
123
|
/**
|
|
190
124
|
* What a recorded `FormationUsage` row came out as.
|
|
191
125
|
*
|
|
@@ -194,9 +128,7 @@ export function isLostUseNumberRace(error: unknown): boolean {
|
|
|
194
128
|
* `FormationUsage.Authorized` rejects the row (see {@link formationVouchMessage}).
|
|
195
129
|
*/
|
|
196
130
|
export interface FormationUsageResult {
|
|
197
|
-
/** The `
|
|
198
|
-
useNumber: number;
|
|
199
|
-
/** The single-use nonce written to `UsageStampId` — the caller's, or a freshly minted one. */
|
|
131
|
+
/** The single-use nonce written to `UsageStampId` (the row's primary key) — the caller's, or a freshly minted one. */
|
|
200
132
|
usageStampId: string;
|
|
201
133
|
}
|
|
202
134
|
|
|
@@ -449,6 +381,13 @@ export class ControlDatabase {
|
|
|
449
381
|
* having to restate the policy's attempts or classifier.
|
|
450
382
|
*/
|
|
451
383
|
private controlWriteRetryPacing: ControlWriteRetryOptions = {};
|
|
384
|
+
/**
|
|
385
|
+
* Pacing seams for {@link readRows}'s retry — the read twin of
|
|
386
|
+
* {@link controlWriteRetryPacing}, with the same contract: production leaves this
|
|
387
|
+
* empty, specs inject a recorded `sleep` / fake `now` so no test waits out a real
|
|
388
|
+
* backoff. Applied OVER the read policy's defaults but UNDER the call site's `label`.
|
|
389
|
+
*/
|
|
390
|
+
private controlReadRetryPacing: ControlReadRetryOptions = {};
|
|
452
391
|
|
|
453
392
|
constructor(config: ControlDatabaseConfig) {
|
|
454
393
|
this.config = config;
|
|
@@ -532,6 +471,20 @@ export class ControlDatabase {
|
|
|
532
471
|
log('Hydrated control catalog (tables=%d, indexes=%d)', hydrated.tables, hydrated.indexes);
|
|
533
472
|
|
|
534
473
|
// Load and execute the schema
|
|
474
|
+
// NOTE: this is where a slow launch is felt. Duration is (raw-storage operations
|
|
475
|
+
// issued) × per-operation storage latency: ~1ms/op on an idle machine, but
|
|
476
|
+
// 50-90ms/op on a loaded disk or a phone's flash under launch contention. A cold
|
|
477
|
+
// start now reaches the backend 172 times (8 tables + 1 index, 21 distinct blocks
|
|
478
|
+
// — dominated by its 130 genuine writes), a warm restart 52, because cadre-core
|
|
479
|
+
// wraps every embedder storage in `@optimystic/db-p2p`'s write-through cache
|
|
480
|
+
// (@serfab/quereus-plugin-sereus's cached-storage.ts). Uncached the same start issued ~2000 operations — the
|
|
481
|
+
// upstream re-read amplification measured in
|
|
482
|
+
// tickets/complete/optimystic-block-read-amplification-on-control-start.md (and
|
|
483
|
+
// its follow-up ...-schema-catalog-reread-per-write-blows-storage-budgets.md),
|
|
484
|
+
// which also rules out the retry policy and cluster deadlines. The counts are
|
|
485
|
+
// pinned by packages/cadre-core/test/control-start-storage-op-budget.spec.ts; if
|
|
486
|
+
// they grow — e.g. the cache leaves this path — that spec fails before anyone
|
|
487
|
+
// has to debug a launch.
|
|
535
488
|
t0 = performance.now();
|
|
536
489
|
await this.loadSchema();
|
|
537
490
|
timing('[controlDb] loadSchema: %dms', Math.round(performance.now() - t0));
|
|
@@ -605,17 +558,139 @@ export class ControlDatabase {
|
|
|
605
558
|
// also what lets schema init absorb optimystic's self-coordination grace refusal, which
|
|
606
559
|
// is unsafe for control writes in general. Why, on
|
|
607
560
|
// `RETRIABLE_SCHEMA_INIT_MATCHERS` in `control-write-retry.ts`.
|
|
608
|
-
await this.lockedWithRetry(() => this.db!.exec(schemaContent), SCHEMA_INIT_RETRY_POLICY);
|
|
561
|
+
await this.lockedWithRetry(() => this.db!.exec(schemaContent), SCHEMA_INIT_RETRY_POLICY, 'schema-init');
|
|
609
562
|
log('Schema loaded and executed');
|
|
610
563
|
}
|
|
611
564
|
|
|
565
|
+
/**
|
|
566
|
+
* One UNRETRIED drain of a control read to rows — every read in this class bottoms out
|
|
567
|
+
* here (almost all via {@link readRows}, which adds the transient-failure retry), and
|
|
568
|
+
* this is the ONE place the committed-read opt-in is spelled.
|
|
569
|
+
*
|
|
570
|
+
* Reached only through {@link readRows}, which drops straight here for the call sites
|
|
571
|
+
* that pass `retry: false` — reads issued from INSIDE a locked write body, where a
|
|
572
|
+
* backoff sleep would hold the write lock and the write funnel re-runs the read anyway
|
|
573
|
+
* (see the NOTE on {@link readRows}).
|
|
574
|
+
*
|
|
575
|
+
* A control read normally queues on the database's exec mutex, so it does not answer
|
|
576
|
+
* until whatever statement holds that mutex finishes. When a control WRITE is parked
|
|
577
|
+
* against an unresponsive cohort member that is minutes, and a read that only wants
|
|
578
|
+
* the last committed state waits the whole time for no reason
|
|
579
|
+
* (`complete/control-reads-blocked-by-stalled-write`).
|
|
580
|
+
*
|
|
581
|
+
* Quereus's remedy is `readConcurrency: 'committed'` — a PER-CALL opt-in
|
|
582
|
+
* (`Statement.tryRouteConcurrent` refuses anything that did not ask for it by name)
|
|
583
|
+
* that runs an eligible read off the mutex against each table's last committed state.
|
|
584
|
+
* Routing is best-effort and never an error: an ineligible statement falls back to
|
|
585
|
+
* the serialized path silently — any side-effecting node, any table-valued function
|
|
586
|
+
* call (pure ones included), any table whose module does not declare
|
|
587
|
+
* `readCommittedSnapshot`, or an EXPLICIT transaction being open on this database.
|
|
588
|
+
*
|
|
589
|
+
* **It is asked for only while a write is in flight, and that condition is the whole
|
|
590
|
+
* point.** The opt-in does not merely move the read off the mutex: it connects the
|
|
591
|
+
* table with `_readCommitted`, and the optimystic vtab serves such a connection from
|
|
592
|
+
* a pinned pre-transaction snapshot that NEVER refreshes from the network, where an
|
|
593
|
+
* ordinary read calls `update()` on the collection first. Taking that path
|
|
594
|
+
* unconditionally therefore trades "reads block behind a stalled write" for "reads
|
|
595
|
+
* stop seeing what other machines wrote" — measured, not theorised: with every read
|
|
596
|
+
* opted in, `control-write-degraded-cohort-member` could no longer observe a sibling
|
|
597
|
+
* publishing its own `CadrePeer` row and failed at suite setup on two runs out of two.
|
|
598
|
+
*
|
|
599
|
+
* `getAutocommit()` reports the whole `Database` rather than this call (the same
|
|
600
|
+
* property {@link assertCommitBoundary} leans on), which is exactly what is wanted:
|
|
601
|
+
* false means some writer's transaction is open, which is the only situation where a
|
|
602
|
+
* read would have blocked. The check races that writer harmlessly in both directions
|
|
603
|
+
* — losing it either reads normally (today's behaviour) or reads committed state.
|
|
604
|
+
*
|
|
605
|
+
* What a committed read gives up is observing a write that has not finished
|
|
606
|
+
* committing, and that is safe for every caller here: an AWAITED write is committed by
|
|
607
|
+
* the time it returns, and a read inside {@link inTransaction} is disqualified by the
|
|
608
|
+
* open explicit transaction and stays serialized, so it still sees its own
|
|
609
|
+
* transaction's rows. A read after an UNAWAITED write on this database would be wrong
|
|
610
|
+
* — no such caller exists, and {@link withWriteLock} is what keeps it that way.
|
|
611
|
+
*/
|
|
612
|
+
private async readRowsOnce(sql: string, params?: SqlParameters): Promise<Record<string, SqlValue>[]> {
|
|
613
|
+
let iterator: AsyncIterableIterator<Record<string, SqlValue>>;
|
|
614
|
+
if (this.db!.getAutocommit()) {
|
|
615
|
+
iterator = this.db!.eval(sql, params);
|
|
616
|
+
} else {
|
|
617
|
+
log('read-eval: a write is in flight, asking for a committed read: %s', sql);
|
|
618
|
+
iterator = this.db!.eval(sql, params, { readConcurrency: 'committed' });
|
|
619
|
+
}
|
|
620
|
+
const rows: Record<string, SqlValue>[] = [];
|
|
621
|
+
for await (const row of iterator) {
|
|
622
|
+
rows.push(row);
|
|
623
|
+
}
|
|
624
|
+
return rows;
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
/**
|
|
628
|
+
* {@link readRowsOnce} plus the bounded transient-failure retry
|
|
629
|
+
* ({@link retryControlRead}) — the funnel every UNLOCKED control read goes through, so a
|
|
630
|
+
* read that failed because the cluster could not be asked properly (a stream reset
|
|
631
|
+
* mid-scan, a partially unreachable cohort) is re-presented a moment later instead of
|
|
632
|
+
* surfacing to every reader.
|
|
633
|
+
*
|
|
634
|
+
* MATERIALIZES the read rather than returning a lazy iterator, and that shape is what
|
|
635
|
+
* makes the retry possible at all: the failure happens during ITERATION, not at call
|
|
636
|
+
* time, and a half-consumed iterator cannot be retried without re-yielding rows the
|
|
637
|
+
* caller already saw. Materializing is safe here — every caller either drains to an
|
|
638
|
+
* array anyway or takes the first row of a statement that yields at most one (a
|
|
639
|
+
* primary-key lookup or a `count(1)`); nothing streams.
|
|
640
|
+
*
|
|
641
|
+
* Each ATTEMPT is a fresh {@link readRowsOnce} call, so the committed-read opt-in
|
|
642
|
+
* (`getAutocommit()`) is re-evaluated per attempt — a write can finish between attempts
|
|
643
|
+
* and change the right answer.
|
|
644
|
+
*
|
|
645
|
+
* The retry's budget ({@link CONTROL_READ_RETRY_BUDGET_MS}, 1.5 s) is deliberately far
|
|
646
|
+
* under the write funnel's: the tightest caller deadline over a control read is the
|
|
647
|
+
* inbound admission gate's 2 s FAIL-OPEN timeout, and a read that outlives it spends its
|
|
648
|
+
* retries after the gate has already admitted (see `control-read-retry.ts`).
|
|
649
|
+
*
|
|
650
|
+
* `label` names the read in the retry loop's `Control read [<label>] …` debug lines and
|
|
651
|
+
* nothing else; several reads are in flight concurrently in a real party, so an
|
|
652
|
+
* unlabelled line cannot be attributed — keep new call sites labelled.
|
|
653
|
+
*
|
|
654
|
+
* `retry: false` drops straight to {@link readRowsOnce}: reads issued from INSIDE a
|
|
655
|
+
* locked write body must not retry on their own. `retryControlWrite`'s contract is that
|
|
656
|
+
* backoff sleeps happen with NO lock held; a read retrying its own backoff inside a
|
|
657
|
+
* locked body would sleep holding the write lock (stalling every other local writer),
|
|
658
|
+
* and the write funnel already re-runs the body's reads when it re-runs the body. See
|
|
659
|
+
* {@link queryStampId} / {@link assertSeatRemains} / {@link queryCadrePeers} for the
|
|
660
|
+
* per-call opt-outs.
|
|
661
|
+
*
|
|
662
|
+
* NOTE: nothing structural enforces that opt-out — it is per-call because an unlocked
|
|
663
|
+
* read runs CONCURRENTLY with a locked body, so `this`-state cannot tell the two apart,
|
|
664
|
+
* and there is no async-context primitive available on every target platform (browser,
|
|
665
|
+
* React Native) to carry the answer. Reads reached through a CALLBACK invoked under the
|
|
666
|
+
* lock are the easy miss: `notifyMembershipChanged` runs its listener with the lock
|
|
667
|
+
* held, and its membership read is opted out at {@link queryCadrePeers}. If another
|
|
668
|
+
* under-lock callback seam is ever added, audit its read graph the same way.
|
|
669
|
+
*/
|
|
670
|
+
private readRows(
|
|
671
|
+
sql: string,
|
|
672
|
+
params: SqlParameters | undefined,
|
|
673
|
+
label: string,
|
|
674
|
+
retry = true
|
|
675
|
+
): Promise<Record<string, SqlValue>[]> {
|
|
676
|
+
if (!retry) {
|
|
677
|
+
return this.readRowsOnce(sql, params);
|
|
678
|
+
}
|
|
679
|
+
// The label lands LAST so the call site's name survives the spec-injected pacing —
|
|
680
|
+
// it changes log attribution only, never behaviour (same layering as lockedWithRetry).
|
|
681
|
+
return retryControlRead(
|
|
682
|
+
() => this.readRowsOnce(sql, params),
|
|
683
|
+
{ ...this.controlReadRetryPacing, label }
|
|
684
|
+
);
|
|
685
|
+
}
|
|
686
|
+
|
|
612
687
|
/**
|
|
613
688
|
* Query all strands from the control database
|
|
614
689
|
*/
|
|
615
690
|
async queryStrands(): Promise<StrandRow[]> {
|
|
616
691
|
this.ensureInitialized();
|
|
617
692
|
const results: StrandRow[] = [];
|
|
618
|
-
for
|
|
693
|
+
for (const row of await this.readRows('select Id, MemberPrivateKey, Type from CadreControl.Strand', undefined, 'strands')) {
|
|
619
694
|
results.push({
|
|
620
695
|
Id: row.Id as string,
|
|
621
696
|
MemberPrivateKey: row.MemberPrivateKey as string | null,
|
|
@@ -633,9 +708,10 @@ export class ControlDatabase {
|
|
|
633
708
|
*/
|
|
634
709
|
async queryStrand(strandId: string): Promise<StrandRow | null> {
|
|
635
710
|
this.ensureInitialized();
|
|
636
|
-
for
|
|
711
|
+
for (const row of await this.readRows(
|
|
637
712
|
'select Id, MemberPrivateKey, Type from CadreControl.Strand where Id = ?',
|
|
638
|
-
[strandId]
|
|
713
|
+
[strandId],
|
|
714
|
+
'strand'
|
|
639
715
|
)) {
|
|
640
716
|
return {
|
|
641
717
|
Id: row.Id as string,
|
|
@@ -662,7 +738,7 @@ export class ControlDatabase {
|
|
|
662
738
|
if (!CONTROL_TABLE_SET.has(table)) {
|
|
663
739
|
throw new Error(`Unknown CadreControl table: ${table}`);
|
|
664
740
|
}
|
|
665
|
-
for
|
|
741
|
+
for (const row of await this.readRows(`select count(1) as Count from CadreControl.${table}`, undefined, 'count-rows')) {
|
|
666
742
|
return (row.Count as number) ?? 0;
|
|
667
743
|
}
|
|
668
744
|
return 0;
|
|
@@ -682,7 +758,7 @@ export class ControlDatabase {
|
|
|
682
758
|
*/
|
|
683
759
|
async hasOwnerKey(): Promise<boolean> {
|
|
684
760
|
this.ensureInitialized();
|
|
685
|
-
for
|
|
761
|
+
for (const row of await this.readRows('select count(1) as Count from CadreControl.OwnerKey', undefined, 'owner-key-exists')) {
|
|
686
762
|
return (row.Count as number) > 0;
|
|
687
763
|
}
|
|
688
764
|
return false;
|
|
@@ -715,7 +791,7 @@ export class ControlDatabase {
|
|
|
715
791
|
async getOwnerKeys(): Promise<Set<string>> {
|
|
716
792
|
this.ensureInitialized();
|
|
717
793
|
const keys = new Set<string>();
|
|
718
|
-
for
|
|
794
|
+
for (const row of await this.readRows('select Key from CadreControl.OwnerKey', undefined, 'owner-keys')) {
|
|
719
795
|
keys.add(row.Key as string);
|
|
720
796
|
}
|
|
721
797
|
return keys;
|
|
@@ -737,12 +813,29 @@ export class ControlDatabase {
|
|
|
737
813
|
* membership-gate refresh path; control tables are a handful of rows today, so this
|
|
738
814
|
* is cheap — if they ever grow, fold the exclusion into one statement or cache the
|
|
739
815
|
* retired-stamp set.
|
|
816
|
+
*
|
|
817
|
+
* NOTE: this filter is HAND-MIRRORED by the gate suites' fake control databases
|
|
818
|
+
* (`test/membership-gate-helpers.ts` and `test/cadre-node-authorized-surface.spec.ts`),
|
|
819
|
+
* which feed the node rows directly. Nothing enforces that the copies stay true, so a
|
|
820
|
+
* change to the filter's SHAPE here (what it keys on, which table it consults) must be
|
|
821
|
+
* carried into both, or those suites keep passing against a contract the database no
|
|
822
|
+
* longer honours. Fine while the filter is one `Set.has` on `StampId`; if it grows a
|
|
823
|
+
* second dimension, give the fakes a shared fixture or a contract test instead.
|
|
824
|
+
*
|
|
825
|
+
* `retry: false` is passed by exactly one caller —
|
|
826
|
+
* `CadreNode.refreshAuthorizedControlPeers`, the membership-gate snapshot refresh —
|
|
827
|
+
* because {@link notifyMembershipChanged} invokes that listener with the write lock
|
|
828
|
+
* HELD, and a retrying read there would sleep its backoff holding the lock (see the
|
|
829
|
+
* NOTE on {@link readRows}). It costs nothing: that refresh is best-effort, keeps the
|
|
830
|
+
* previous snapshot on failure, and is re-driven by the next write and by the timed
|
|
831
|
+
* reconcile. BOTH of this method's reads take the flag — retrying either one under the
|
|
832
|
+
* lock is the same defect.
|
|
740
833
|
*/
|
|
741
|
-
async queryCadrePeers(): Promise<CadrePeerRow[]> {
|
|
834
|
+
async queryCadrePeers(retry = true): Promise<CadrePeerRow[]> {
|
|
742
835
|
this.ensureInitialized();
|
|
743
|
-
const revoked = await this.queryRevokedStamps('CadrePeer');
|
|
836
|
+
const revoked = await this.queryRevokedStamps('CadrePeer', retry);
|
|
744
837
|
const rows: CadrePeerRow[] = [];
|
|
745
|
-
for
|
|
838
|
+
for (const row of await this.readRows('select PeerId, Multiaddr, StampId, VouchOwner, VouchSig from CadreControl.CadrePeer', undefined, 'cadre-peers', retry)) {
|
|
746
839
|
const stampId = (row.StampId as string | null) ?? null;
|
|
747
840
|
if (stampId !== null && revoked.has(stampId)) {
|
|
748
841
|
continue;
|
|
@@ -770,7 +863,7 @@ export class ControlDatabase {
|
|
|
770
863
|
async queryValidationKeys(): Promise<string[]> {
|
|
771
864
|
this.ensureInitialized();
|
|
772
865
|
const keys: string[] = [];
|
|
773
|
-
for
|
|
866
|
+
for (const row of await this.readRows('select Key from CadreControl.ValidationKey', undefined, 'validation-keys')) {
|
|
774
867
|
keys.push(row.Key as string);
|
|
775
868
|
}
|
|
776
869
|
return keys.sort();
|
|
@@ -790,14 +883,25 @@ export class ControlDatabase {
|
|
|
790
883
|
* `table` and its {@link GUARDED_KEY_COLUMN} column are interpolated into the SQL; both
|
|
791
884
|
* come from closed literal unions — no caller-supplied string can reach the statement
|
|
792
885
|
* (same injection-surface discipline as {@link countRows}'s `CONTROL_TABLE_SET` guard).
|
|
886
|
+
*
|
|
887
|
+
* NOTE: `retry: false` is passed by exactly the call sites that run INSIDE a locked
|
|
888
|
+
* write body — {@link deleteGuardedRow} and {@link insertCadrePeer}'s insert-if-absent
|
|
889
|
+
* guard. A read retrying there would sleep its backoff HOLDING the write lock
|
|
890
|
+
* (`retryControlWrite`'s contract is that backoff sleeps happen with no lock held), and
|
|
891
|
+
* the write funnel already re-runs the body's reads when it re-runs the body. The
|
|
892
|
+
* unlocked callers — the public stamp readers, {@link reauthorizeCadrePeer} and
|
|
893
|
+
* {@link reapRevokedRow}, whose stamp reads run BEFORE their locks are taken — keep the
|
|
894
|
+
* default retried path. Explicit per call rather than an ambient flag: unlocked reads
|
|
895
|
+
* run concurrently with a locked body, so `this`-state cannot tell the two apart.
|
|
793
896
|
*/
|
|
794
897
|
private async queryStampId(
|
|
795
898
|
table: RevocableTable,
|
|
796
|
-
keyValue: string
|
|
899
|
+
keyValue: string,
|
|
900
|
+
retry = true
|
|
797
901
|
): Promise<string | null> {
|
|
798
902
|
this.ensureInitialized();
|
|
799
903
|
const sql = `select StampId from CadreControl.${table} where ${GUARDED_KEY_COLUMN[table]} = ?`;
|
|
800
|
-
for
|
|
904
|
+
for (const row of await this.readRows(sql, [keyValue], `stamp-${table}`, retry)) {
|
|
801
905
|
return (row.StampId as string | null) ?? null;
|
|
802
906
|
}
|
|
803
907
|
return null;
|
|
@@ -847,11 +951,14 @@ export class ControlDatabase {
|
|
|
847
951
|
* NOTE: re-reads the whole retired set on every call, and the caller runs per inbound
|
|
848
952
|
* gate request while the table only ever grows. Cheap today (a cadre removes peers
|
|
849
953
|
* rarely); if removals ever become routine, cache the set and invalidate it on write.
|
|
954
|
+
*
|
|
955
|
+
* `retry: false` is forwarded by {@link queryCadrePeers} for the one caller that reads
|
|
956
|
+
* under the write lock — see that method's note.
|
|
850
957
|
*/
|
|
851
|
-
async queryRevokedStamps(tableName: RevocableTable): Promise<Set<string>> {
|
|
958
|
+
async queryRevokedStamps(tableName: RevocableTable, retry = true): Promise<Set<string>> {
|
|
852
959
|
this.ensureInitialized();
|
|
853
960
|
const stamps = new Set<string>();
|
|
854
|
-
for
|
|
961
|
+
for (const row of await this.readRows('select StampId from CadreControl.Revocation where TableName = ?', [tableName], 'revoked-stamps', retry)) {
|
|
855
962
|
stamps.add(row.StampId as string);
|
|
856
963
|
}
|
|
857
964
|
return stamps;
|
|
@@ -868,7 +975,7 @@ export class ControlDatabase {
|
|
|
868
975
|
async queryRevocations(): Promise<RevocationRow[]> {
|
|
869
976
|
this.ensureInitialized();
|
|
870
977
|
const rows: RevocationRow[] = [];
|
|
871
|
-
for
|
|
978
|
+
for (const row of await this.readRows('select TableName, RowKey, StampId, ReissuedAt from CadreControl.Revocation', undefined, 'revocations')) {
|
|
872
979
|
rows.push({
|
|
873
980
|
tableName: row.TableName as RevocableTable,
|
|
874
981
|
rowKey: row.RowKey as string,
|
|
@@ -890,7 +997,7 @@ export class ControlDatabase {
|
|
|
890
997
|
* {@link queryRevokedStamps} rather than inlining the SQL, so tests can interpose on
|
|
891
998
|
* that seam (see {@link queryCadrePeers}).
|
|
892
999
|
*
|
|
893
|
-
* Missing/
|
|
1000
|
+
* Missing/null column values are coalesced
|
|
894
1001
|
* to their empty form (`''` key/sig, `[]` addrs, `0` stamp) so the caller's
|
|
895
1002
|
* verify/freshness gates uniformly reject an unpublished or malformed row.
|
|
896
1003
|
* The split `addrs` re-join to the exact stored `Multiaddr` (split-on-`,` is
|
|
@@ -900,9 +1007,10 @@ export class ControlDatabase {
|
|
|
900
1007
|
async queryPeerRecord(peerId: string): Promise<PeerAddressRecord | null> {
|
|
901
1008
|
this.ensureInitialized();
|
|
902
1009
|
const revoked = await this.queryRevokedStamps('CadrePeer');
|
|
903
|
-
for
|
|
1010
|
+
for (const row of await this.readRows(
|
|
904
1011
|
'select PeerId, PublicKey, Multiaddr, UpdatedAt, Sig, StampId from CadreControl.CadrePeer where PeerId = ?',
|
|
905
|
-
[peerId]
|
|
1012
|
+
[peerId],
|
|
1013
|
+
'peer-record'
|
|
906
1014
|
)) {
|
|
907
1015
|
const stampId = (row.StampId as string | null) ?? null;
|
|
908
1016
|
if (stampId !== null && revoked.has(stampId)) {
|
|
@@ -951,14 +1059,14 @@ export class ControlDatabase {
|
|
|
951
1059
|
with context OwnerKey = null, Signature = ?
|
|
952
1060
|
set Multiaddr = ?, UpdatedAt = ?, Sig = ?
|
|
953
1061
|
where PeerId = ?
|
|
954
|
-
`, [record.sig, multiaddr, record.updatedAt, record.sig, record.peerId]);
|
|
1062
|
+
`, [record.sig, multiaddr, record.updatedAt, record.sig, record.peerId], 'self-record-update');
|
|
955
1063
|
log('Self peer record updated: %s (updatedAt=%d)', record.peerId, record.updatedAt);
|
|
956
1064
|
}
|
|
957
1065
|
|
|
958
1066
|
/**
|
|
959
1067
|
* Read a single peer's device push token (the full `DeviceToken` row) by PeerId.
|
|
960
1068
|
*
|
|
961
|
-
* Returns null when no row exists. Missing/
|
|
1069
|
+
* Returns null when no row exists. Missing/null column values are coalesced to
|
|
962
1070
|
* their empty form (`''` token/sig, `0` stamp) so the caller's verify/freshness
|
|
963
1071
|
* gates uniformly reject an unpublished or malformed row. `platform` is returned
|
|
964
1072
|
* verbatim (the resolver validates it against {@link PushPlatform} and re-verifies
|
|
@@ -970,9 +1078,10 @@ export class ControlDatabase {
|
|
|
970
1078
|
*/
|
|
971
1079
|
async queryDeviceToken(peerId: string): Promise<DeviceTokenRow | null> {
|
|
972
1080
|
this.ensureInitialized();
|
|
973
|
-
for
|
|
1081
|
+
for (const row of await this.readRows(
|
|
974
1082
|
'select PeerId, Platform, Token, UpdatedAt, Sig, StampId from CadreControl.DeviceToken where PeerId = ?',
|
|
975
|
-
[peerId]
|
|
1083
|
+
[peerId],
|
|
1084
|
+
'device-token'
|
|
976
1085
|
)) {
|
|
977
1086
|
return {
|
|
978
1087
|
peerId: row.PeerId as string,
|
|
@@ -1004,7 +1113,7 @@ export class ControlDatabase {
|
|
|
1004
1113
|
with context OwnerKey = null, Signature = ?
|
|
1005
1114
|
set Platform = ?, Token = ?, UpdatedAt = ?, Sig = ?
|
|
1006
1115
|
where PeerId = ?
|
|
1007
|
-
`, [record.sig, record.platform, record.token, record.updatedAt, record.sig, record.peerId]);
|
|
1116
|
+
`, [record.sig, record.platform, record.token, record.updatedAt, record.sig, record.peerId], 'device-token-update');
|
|
1008
1117
|
log('Self device token updated: %s (platform=%s, updatedAt=%d)', record.peerId, record.platform, record.updatedAt);
|
|
1009
1118
|
}
|
|
1010
1119
|
|
|
@@ -1028,7 +1137,7 @@ export class ControlDatabase {
|
|
|
1028
1137
|
insert into CadreControl.OwnerKey (Key, StampId)
|
|
1029
1138
|
with context OwnerKey = null, Signature = null
|
|
1030
1139
|
values (?, ?)
|
|
1031
|
-
`, [key, stampId]);
|
|
1140
|
+
`, [key, stampId], 'owner-key-insert');
|
|
1032
1141
|
log('Owner key inserted');
|
|
1033
1142
|
}
|
|
1034
1143
|
|
|
@@ -1071,7 +1180,7 @@ export class ControlDatabase {
|
|
|
1071
1180
|
insert into CadreControl.Strand (Id, Type, MemberPrivateKey, StampId)
|
|
1072
1181
|
with context OwnerKey = ?, Signature = ?
|
|
1073
1182
|
values (?, ?, ?, ?)
|
|
1074
|
-
`, [ownerKey, signature, strandId, type, memberPrivateKey ?? null, stampId]);
|
|
1183
|
+
`, [ownerKey, signature, strandId, type, memberPrivateKey ?? null, stampId], 'strand-insert');
|
|
1075
1184
|
|
|
1076
1185
|
log('Strand inserted: %s', strandId);
|
|
1077
1186
|
}
|
|
@@ -1099,7 +1208,7 @@ export class ControlDatabase {
|
|
|
1099
1208
|
ownerKey: string,
|
|
1100
1209
|
signMessage: (message: Uint8Array) => string
|
|
1101
1210
|
): Promise<boolean> {
|
|
1102
|
-
return this.lockedWithRetry(() => this.deleteGuardedRow('Strand', strandId, ownerKey, signMessage));
|
|
1211
|
+
return this.lockedWithRetry(() => this.deleteGuardedRow('Strand', strandId, ownerKey, signMessage), {}, 'strand-delete');
|
|
1103
1212
|
}
|
|
1104
1213
|
|
|
1105
1214
|
/**
|
|
@@ -1134,7 +1243,7 @@ export class ControlDatabase {
|
|
|
1134
1243
|
insert into CadreControl.ValidationKey (Key, StampId)
|
|
1135
1244
|
with context OwnerKey = ?, Signature = ?
|
|
1136
1245
|
values (?, ?)
|
|
1137
|
-
`, [ownerKey, signature, key, stampId]);
|
|
1246
|
+
`, [ownerKey, signature, key, stampId], 'validation-key-insert');
|
|
1138
1247
|
|
|
1139
1248
|
log('Validation key inserted: %s', key);
|
|
1140
1249
|
}
|
|
@@ -1160,7 +1269,7 @@ export class ControlDatabase {
|
|
|
1160
1269
|
ownerKey: string,
|
|
1161
1270
|
signMessage: (message: Uint8Array) => string
|
|
1162
1271
|
): Promise<boolean> {
|
|
1163
|
-
return this.lockedWithRetry(() => this.deleteGuardedRow('ValidationKey', key, ownerKey, signMessage));
|
|
1272
|
+
return this.lockedWithRetry(() => this.deleteGuardedRow('ValidationKey', key, ownerKey, signMessage), {}, 'validation-key-delete');
|
|
1164
1273
|
}
|
|
1165
1274
|
|
|
1166
1275
|
/**
|
|
@@ -1219,7 +1328,8 @@ export class ControlDatabase {
|
|
|
1219
1328
|
buildAuthorizationMessage('CadreControl.CadrePeer', 'vouch', [row.peerId, stampId])
|
|
1220
1329
|
);
|
|
1221
1330
|
return await this.mutateCadrePeer('peer-insert', async () => {
|
|
1222
|
-
|
|
1331
|
+
// retry: false — this guard runs inside the locked write body (see queryStampId's NOTE).
|
|
1332
|
+
if (await this.queryStampId('CadrePeer', row.peerId, false) !== null) {
|
|
1223
1333
|
log('CadrePeer row already present for %s; insert skipped (already a member)', row.peerId);
|
|
1224
1334
|
return false;
|
|
1225
1335
|
}
|
|
@@ -1360,7 +1470,7 @@ export class ControlDatabase {
|
|
|
1360
1470
|
ownerKey: string,
|
|
1361
1471
|
signMessage: (message: Uint8Array) => string
|
|
1362
1472
|
): Promise<boolean> {
|
|
1363
|
-
return this.lockedWithRetry(() => this.deleteGuardedRow('DeviceToken', peerId, ownerKey, signMessage));
|
|
1473
|
+
return this.lockedWithRetry(() => this.deleteGuardedRow('DeviceToken', peerId, ownerKey, signMessage), {}, 'device-token-delete');
|
|
1364
1474
|
}
|
|
1365
1475
|
|
|
1366
1476
|
/**
|
|
@@ -1403,7 +1513,8 @@ export class ControlDatabase {
|
|
|
1403
1513
|
signMessage: (message: Uint8Array) => string
|
|
1404
1514
|
): Promise<boolean> {
|
|
1405
1515
|
this.ensureInitialized();
|
|
1406
|
-
|
|
1516
|
+
// retry: false — this runs inside the locked write body (see queryStampId's NOTE).
|
|
1517
|
+
const stampId = await this.queryStampId(table, keyValue, false);
|
|
1407
1518
|
if (stampId === null) {
|
|
1408
1519
|
log('delete %s: no row for %s (already absent)', table, keyValue);
|
|
1409
1520
|
return false;
|
|
@@ -1527,7 +1638,7 @@ export class ControlDatabase {
|
|
|
1527
1638
|
// in one pass, hoist the notify to one refresh after the sweep instead.
|
|
1528
1639
|
await this.mutateCadrePeer('peer-reap', () => this.db!.exec(sql, [rowKey, stampId]));
|
|
1529
1640
|
} else {
|
|
1530
|
-
await this.execWrite(sql, [rowKey, stampId]);
|
|
1641
|
+
await this.execWrite(sql, [rowKey, stampId], `reap-${table}`);
|
|
1531
1642
|
}
|
|
1532
1643
|
// A row leaving the control plane without an owner signature at the write site
|
|
1533
1644
|
// should be visible in a log.
|
|
@@ -1689,7 +1800,7 @@ export class ControlDatabase {
|
|
|
1689
1800
|
});
|
|
1690
1801
|
log('Reissued %d revocation tombstone(s) at %d', executed, reissuedAt);
|
|
1691
1802
|
return executed;
|
|
1692
|
-
});
|
|
1803
|
+
}, {}, 'revocation-reissue');
|
|
1693
1804
|
}
|
|
1694
1805
|
|
|
1695
1806
|
/**
|
|
@@ -1794,7 +1905,7 @@ export class ControlDatabase {
|
|
|
1794
1905
|
this.assertCommitBoundary(reason, 'on return from');
|
|
1795
1906
|
await this.notifyMembershipChanged(reason);
|
|
1796
1907
|
return result;
|
|
1797
|
-
});
|
|
1908
|
+
}, {}, reason);
|
|
1798
1909
|
}
|
|
1799
1910
|
|
|
1800
1911
|
/**
|
|
@@ -1807,7 +1918,8 @@ export class ControlDatabase {
|
|
|
1807
1918
|
* {@link assertCommitBoundary} (any `CadrePeer` path) or silently join each other's
|
|
1808
1919
|
* open transaction — a torn commit if either side rolls back. This lock closes that
|
|
1809
1920
|
* class: every public write method of this class runs its statement(s) under it, and
|
|
1810
|
-
* `SeedBootstrapService`
|
|
1921
|
+
* `SeedBootstrapService` reaches it through {@link execWrite} rather than touching the
|
|
1922
|
+
* `Database` directly, so it is covered too. The race is
|
|
1811
1923
|
* real, not theoretical: the background self-record publish on a control connection
|
|
1812
1924
|
* opening (`CadreNode.drainPendingControlReplication` → `registerSelf`) collided with
|
|
1813
1925
|
* a foreground `authorizePeer` — both `CadrePeer` inserts — and tripped the boundary
|
|
@@ -1851,10 +1963,8 @@ export class ControlDatabase {
|
|
|
1851
1963
|
*
|
|
1852
1964
|
* The retry wraps the LOCK, it does not live inside it: each attempt takes and
|
|
1853
1965
|
* releases the lock, and the backoff sleeps with NO lock held, so a write parked in
|
|
1854
|
-
* backoff never stalls the other local writers queued behind it.
|
|
1855
|
-
*
|
|
1856
|
-
* `USE_NUMBER_ATTEMPTS`. Safe because every locked body is atomic and re-runnable
|
|
1857
|
-
* (the contract on {@link withWriteLock}).
|
|
1966
|
+
* backoff never stalls the other local writers queued behind it. Safe because every
|
|
1967
|
+
* locked body is atomic and re-runnable (the contract on {@link withWriteLock}).
|
|
1858
1968
|
*
|
|
1859
1969
|
* Also carries {@link loadSchema}'s distributed DDL, the one caller that runs BEFORE
|
|
1860
1970
|
* `initialized` is set — so it reaches this method directly rather than through
|
|
@@ -1863,11 +1973,18 @@ export class ControlDatabase {
|
|
|
1863
1973
|
* rather than carving out a lockless path. It is also the ONLY caller that passes a
|
|
1864
1974
|
* `policy` — everything else takes the default, whose classifier and attempt count are
|
|
1865
1975
|
* unchanged by that call site existing.
|
|
1976
|
+
*
|
|
1977
|
+
* `label` names the operation in this loop's debug lines and nothing else
|
|
1978
|
+
* ({@link ControlWriteRetryOptions.label}). Every call site in this class supplies one:
|
|
1979
|
+
* several writes retry CONCURRENTLY in a real party, so an unlabelled line cannot be
|
|
1980
|
+
* attributed to a write — keep new call sites labelled.
|
|
1866
1981
|
*/
|
|
1867
|
-
private lockedWithRetry<T>(fn: () => Promise<T>, policy: ControlWriteRetryOptions = {}): Promise<T> {
|
|
1982
|
+
private lockedWithRetry<T>(fn: () => Promise<T>, policy: ControlWriteRetryOptions = {}, label?: string): Promise<T> {
|
|
1983
|
+
// The label lands LAST so a call site's own name survives both the policy and the
|
|
1984
|
+
// spec-injected pacing; it changes log attribution only, never behaviour.
|
|
1868
1985
|
return retryControlWrite(
|
|
1869
1986
|
() => this.withWriteLock(fn),
|
|
1870
|
-
{ ...policy, ...this.controlWriteRetryPacing }
|
|
1987
|
+
{ ...policy, ...this.controlWriteRetryPacing, ...(label !== undefined ? { label } : {}) }
|
|
1871
1988
|
);
|
|
1872
1989
|
}
|
|
1873
1990
|
|
|
@@ -1880,10 +1997,13 @@ export class ControlDatabase {
|
|
|
1880
1997
|
* Public so `SeedBootstrapService`'s direct `CadrePeer`/`DeviceToken` SQL writes go
|
|
1881
1998
|
* through the same seam. Must NOT be called from inside an already-locked body (the
|
|
1882
1999
|
* lock is not re-entrant); such a caller uses a bare `getDatabase().exec` instead.
|
|
2000
|
+
*
|
|
2001
|
+
* `label` names the write in the retry loop's debug lines (see {@link lockedWithRetry});
|
|
2002
|
+
* optional only because it is log-only, but every caller in this repo passes one.
|
|
1883
2003
|
*/
|
|
1884
|
-
execWrite(sql: string, params?: SqlParameters): Promise<void> {
|
|
2004
|
+
execWrite(sql: string, params?: SqlParameters, label?: string): Promise<void> {
|
|
1885
2005
|
this.ensureInitialized();
|
|
1886
|
-
return this.lockedWithRetry(() => this.db!.exec(sql, params));
|
|
2006
|
+
return this.lockedWithRetry(() => this.db!.exec(sql, params), {}, label);
|
|
1887
2007
|
}
|
|
1888
2008
|
|
|
1889
2009
|
/**
|
|
@@ -1956,7 +2076,8 @@ export class ControlDatabase {
|
|
|
1956
2076
|
* `StrandId` binds the invite to a pre-existing host strand (provision-then-record):
|
|
1957
2077
|
* when set, a responder redeeming this token records a `FormationUsage` against that
|
|
1958
2078
|
* strand and returns it (see {@link ControlFormationUsageRecorder.resolveStrand}); a
|
|
1959
|
-
* null `StrandId` (the default)
|
|
2079
|
+
* null `StrandId` (the default) takes the unbound responder-provisions path: the
|
|
2080
|
+
* responder provisions a fresh open strand and atomically records its one consent row.
|
|
1960
2081
|
* Like ValidationUrl it is a nullable bound field, signed as `''` when absent.
|
|
1961
2082
|
*
|
|
1962
2083
|
* The ExpiresAt and TotalUses message fields must byte-match what the (auto-deferred,
|
|
@@ -2023,7 +2144,7 @@ export class ControlDatabase {
|
|
|
2023
2144
|
options.validationUrl ?? null,
|
|
2024
2145
|
options.strandId ?? null,
|
|
2025
2146
|
stampId,
|
|
2026
|
-
]);
|
|
2147
|
+
], 'formation-invite-insert');
|
|
2027
2148
|
|
|
2028
2149
|
log('Formation invite inserted: %s', token);
|
|
2029
2150
|
}
|
|
@@ -2053,13 +2174,12 @@ export class ControlDatabase {
|
|
|
2053
2174
|
* takes an owner re-seat ({@link insertStrand}) plus a bound invite, never another
|
|
2054
2175
|
* redemption.
|
|
2055
2176
|
*
|
|
2056
|
-
*
|
|
2057
|
-
*
|
|
2058
|
-
*
|
|
2059
|
-
*
|
|
2060
|
-
*
|
|
2061
|
-
*
|
|
2062
|
-
* cannot drift, and no second trip through the approval hook is possible from in here.
|
|
2177
|
+
* The usage row is keyed by the joiner's own nonce (`UsageStampId` is the primary key), so
|
|
2178
|
+
* concurrent redemptions of one token never contend for a shared row key — there is no
|
|
2179
|
+
* lost race to retry and no second trip through the approval hook. The invite's seat
|
|
2180
|
+
* budget is checked by COUNT ({@link assertSeatRemains}) inside the write lock, ahead of
|
|
2181
|
+
* the write, so a spent invite is refused by name (`InvitationExhaustedError`) instead of
|
|
2182
|
+
* as the schema cap clause's generic `Authorized` CHECK failure.
|
|
2063
2183
|
*/
|
|
2064
2184
|
async redeemInvitation(params: {
|
|
2065
2185
|
token: string;
|
|
@@ -2082,9 +2202,9 @@ export class ControlDatabase {
|
|
|
2082
2202
|
signal?: AbortSignal;
|
|
2083
2203
|
/**
|
|
2084
2204
|
* The invite's seat budget, when the caller already has it in hand (e.g. it already read
|
|
2085
|
-
* the `FormationInvite` row to get here). Passed through to {@link
|
|
2086
|
-
*
|
|
2087
|
-
*
|
|
2205
|
+
* the `FormationInvite` row to get here). Passed through to {@link assertSeatRemains} so
|
|
2206
|
+
* the seat check needs no extra read. `undefined` falls back to a fresh
|
|
2207
|
+
* {@link queryFormationInvite} read; `null` means "no invite row" / unlimited.
|
|
2088
2208
|
*/
|
|
2089
2209
|
totalUses?: number | null;
|
|
2090
2210
|
}): Promise<FormationUsageResult> {
|
|
@@ -2097,13 +2217,22 @@ export class ControlDatabase {
|
|
|
2097
2217
|
log('Redeeming invitation %s -> strand %s', token, strandId);
|
|
2098
2218
|
|
|
2099
2219
|
const localPeerId = this.config.libp2pNode.peerId.toString();
|
|
2100
|
-
//
|
|
2101
|
-
//
|
|
2102
|
-
// approver's signed digest. Re-minting per attempt would only churn.
|
|
2220
|
+
// A failed attempt rolls its whole transaction back, so nothing was ever persisted under
|
|
2221
|
+
// this stamp, and it is deliberately not part of the approver's signed digest.
|
|
2103
2222
|
const strandStampId = generateStampId(localPeerId);
|
|
2104
2223
|
|
|
2105
|
-
|
|
2106
|
-
|
|
2224
|
+
// Ordering inside the locked body is contractual: the abort check, then the seat check,
|
|
2225
|
+
// then the write — FormationAbortedError may only ever be thrown before the write is
|
|
2226
|
+
// issued, and a transient-failure retry (lockedWithRetry re-runs the whole body) must
|
|
2227
|
+
// re-check both before re-presenting the write.
|
|
2228
|
+
await this.lockedWithRetry(async () => {
|
|
2229
|
+
// Inside the lock: a write still parked behind another writer when the caller's
|
|
2230
|
+
// budget expired is abandoned rather than executed.
|
|
2231
|
+
if (signal?.aborted) {
|
|
2232
|
+
throw new FormationAbortedError(token, 'redemption');
|
|
2233
|
+
}
|
|
2234
|
+
await this.assertSeatRemains(token, totalUses);
|
|
2235
|
+
await this.inTransaction('redemption', async () => {
|
|
2107
2236
|
// 1. Strand row — authorised by the FormationUsage branch (no owner sig),
|
|
2108
2237
|
// still carrying a fresh unique StampId for the anti-replay column.
|
|
2109
2238
|
// Hard-coded open + keyless: the consent branch admits no other shape.
|
|
@@ -2116,14 +2245,15 @@ export class ControlDatabase {
|
|
|
2116
2245
|
// 2. FormationUsage row — authorised by the matching FormationInvite, and
|
|
2117
2246
|
// carrying the strand's stamp so it authorizes THIS row and no other.
|
|
2118
2247
|
await this.execFormationUsageInsert({
|
|
2119
|
-
token,
|
|
2248
|
+
token, usageStampId, disclosure, strandId, strandStampId,
|
|
2120
2249
|
peerKey, peerSignature, nowMs: nowMs ?? Date.now(),
|
|
2121
2250
|
validationKey: validationKey ?? null, validationSignature: validationSignature ?? null,
|
|
2122
2251
|
});
|
|
2123
|
-
})
|
|
2252
|
+
});
|
|
2253
|
+
}, {}, 'formation-usage');
|
|
2124
2254
|
|
|
2125
|
-
log('Redeemed invitation %s -> strand %s (
|
|
2126
|
-
return {
|
|
2255
|
+
log('Redeemed invitation %s -> strand %s (stamp %s)', token, strandId, usageStampId);
|
|
2256
|
+
return { usageStampId };
|
|
2127
2257
|
}
|
|
2128
2258
|
|
|
2129
2259
|
/**
|
|
@@ -2131,8 +2261,8 @@ export class ControlDatabase {
|
|
|
2131
2261
|
* insert). This is the redemption path when the strand was provisioned
|
|
2132
2262
|
* separately (e.g. owner-signed) and the consent record is added after the
|
|
2133
2263
|
* fact: the single insert auto-commits, and the deferred `StrandExists` CHECK
|
|
2134
|
-
* is satisfied by the pre-existing committed strand row.
|
|
2135
|
-
* `
|
|
2264
|
+
* is satisfied by the pre-existing committed strand row. Echoes back the
|
|
2265
|
+
* redemption's `usageStampId` (the row's primary key).
|
|
2136
2266
|
*
|
|
2137
2267
|
* Use {@link redeemInvitation} instead when the strand must be created by
|
|
2138
2268
|
* consent atomically with the usage.
|
|
@@ -2146,12 +2276,12 @@ export class ControlDatabase {
|
|
|
2146
2276
|
* by {@link ControlFormationUsageRecorder.resolveStrand}, so an absent row at this
|
|
2147
2277
|
* point is a genuine race and deserves a named error, not a silent rollback.
|
|
2148
2278
|
*
|
|
2149
|
-
*
|
|
2150
|
-
*
|
|
2151
|
-
*
|
|
2152
|
-
*
|
|
2153
|
-
*
|
|
2154
|
-
*
|
|
2279
|
+
* The row is keyed by the joiner's own nonce (`UsageStampId` is the primary key), so this
|
|
2280
|
+
* path — the one production actually races on (a bound invite published by `cadre-web` /
|
|
2281
|
+
* `cadre-phone` is always record-only) — never contends with another redemption for a
|
|
2282
|
+
* shared row key: there is no lost race to retry and no second trip through the approval
|
|
2283
|
+
* hook. The invite's seat budget is checked by COUNT ({@link assertSeatRemains}) inside
|
|
2284
|
+
* the write lock, ahead of the write.
|
|
2155
2285
|
*/
|
|
2156
2286
|
async recordFormationUsage(params: {
|
|
2157
2287
|
token: string;
|
|
@@ -2167,17 +2297,16 @@ export class ControlDatabase {
|
|
|
2167
2297
|
validationKey?: string;
|
|
2168
2298
|
validationSignature?: string;
|
|
2169
2299
|
/**
|
|
2170
|
-
* Aborted when the caller has given up. Checked
|
|
2171
|
-
*
|
|
2172
|
-
*
|
|
2173
|
-
* once an insert has been issued.
|
|
2300
|
+
* Aborted when the caller has given up. Checked inside the write lock, before the insert
|
|
2301
|
+
* is issued, throwing {@link FormationAbortedError} with the invite unspent. Never
|
|
2302
|
+
* checked once the insert has been issued.
|
|
2174
2303
|
*/
|
|
2175
2304
|
signal?: AbortSignal;
|
|
2176
2305
|
/**
|
|
2177
2306
|
* The invite's seat budget, when the caller already has it in hand (e.g. it already read
|
|
2178
|
-
* the `FormationInvite` row to get here). Passed through to {@link
|
|
2179
|
-
*
|
|
2180
|
-
*
|
|
2307
|
+
* the `FormationInvite` row to get here). Passed through to {@link assertSeatRemains} so
|
|
2308
|
+
* the seat check needs no extra read. `undefined` falls back to a fresh
|
|
2309
|
+
* {@link queryFormationInvite} read; `null` means "no invite row" / unlimited.
|
|
2181
2310
|
*/
|
|
2182
2311
|
totalUses?: number | null;
|
|
2183
2312
|
}): Promise<FormationUsageResult> {
|
|
@@ -2188,109 +2317,40 @@ export class ControlDatabase {
|
|
|
2188
2317
|
totalUses,
|
|
2189
2318
|
} = params;
|
|
2190
2319
|
|
|
2191
|
-
//
|
|
2192
|
-
//
|
|
2320
|
+
// The host strand is pre-existing and owner-signed; its stamp names the strand ROW the
|
|
2321
|
+
// consent record authorizes.
|
|
2193
2322
|
const strandStampId = await this.queryStrandStampId(strandId);
|
|
2194
2323
|
if (strandStampId === null) {
|
|
2195
2324
|
throw new MissingHostStrandError(strandId, token);
|
|
2196
2325
|
}
|
|
2197
2326
|
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2327
|
+
// Ordering inside the locked body is contractual — abort check, seat check, write — see
|
|
2328
|
+
// the matching note in redeemInvitation.
|
|
2329
|
+
await this.lockedWithRetry(async () => {
|
|
2330
|
+
if (signal?.aborted) {
|
|
2331
|
+
throw new FormationAbortedError(token, 'usage recording');
|
|
2332
|
+
}
|
|
2333
|
+
await this.assertSeatRemains(token, totalUses);
|
|
2334
|
+
await this.execFormationUsageInsert({
|
|
2335
|
+
token, usageStampId, disclosure, strandId, strandStampId,
|
|
2201
2336
|
peerKey, peerSignature, nowMs: nowMs ?? Date.now(),
|
|
2202
2337
|
validationKey: validationKey ?? null, validationSignature: validationSignature ?? null,
|
|
2203
|
-
})
|
|
2338
|
+
});
|
|
2339
|
+
}, {}, 'formation-usage');
|
|
2204
2340
|
|
|
2205
|
-
log('Recorded formation usage: token=%s strand=%s (
|
|
2206
|
-
return {
|
|
2341
|
+
log('Recorded formation usage: token=%s strand=%s (stamp %s)', token, strandId, usageStampId);
|
|
2342
|
+
return { usageStampId };
|
|
2207
2343
|
}
|
|
2208
2344
|
|
|
2209
2345
|
/**
|
|
2210
|
-
*
|
|
2211
|
-
* writer took that number first.
|
|
2212
|
-
*
|
|
2213
|
-
* Each attempt takes {@link withWriteLock} and, inside it, reads {@link nextUseNumber}
|
|
2214
|
-
* afresh. Reading under the lock is the whole fix for the common case: two redemptions of
|
|
2215
|
-
* one token on the SAME node are serialized by the local write queue, so each computes a
|
|
2216
|
-
* number the other has already committed and neither collides. The retry covers the writers
|
|
2217
|
-
* this lock does not reach — another node of the cadre, another `Database` handle over the
|
|
2218
|
-
* same store — which can still take the number between our read and our commit.
|
|
2219
|
-
*
|
|
2220
|
-
* `write` receives ONLY the use number; every other parameter of the redemption (the nonce,
|
|
2221
|
-
* the approver's signature, the strand, the joining peer key, the disclosure) is closed over
|
|
2222
|
-
* by the caller and passed through untouched. So by construction a retry re-presents the
|
|
2223
|
-
* IDENTICAL approval and the four non-nonce fields inside the approver's signed digest
|
|
2224
|
-
* cannot drift. No second call to the approval hook is reachable from in here either:
|
|
2225
|
-
* `obtainApproval` lives a layer up in `ControlFormationUsageRecorder` and is never
|
|
2226
|
-
* re-entered.
|
|
2227
|
-
*
|
|
2228
|
-
* `write` must be a BARE private body ({@link inTransaction} /
|
|
2229
|
-
* {@link execFormationUsageInsert}) — {@link withWriteLock} is not re-entrant and a `write`
|
|
2230
|
-
* that called another locked public method would hang silently and strand the write queue.
|
|
2231
|
-
*
|
|
2232
|
-
* A failed attempt has written nothing (a bare insert never landed; a transactional one
|
|
2233
|
-
* rolled back), so the abort check at the top of each attempt stays inside
|
|
2234
|
-
* {@link FormationAbortedError}'s contract — it is only ever thrown before this attempt's
|
|
2235
|
-
* write is issued.
|
|
2236
|
-
*
|
|
2237
|
-
* @param operation - `'redemption'` | `'usage recording'`, for the abort error's text.
|
|
2238
|
-
* @param knownTotalUses - The invite's seat budget when the caller already has it in hand
|
|
2239
|
-
* (`undefined` to fall back to a fresh {@link queryFormationInvite} read; `null` for "no
|
|
2240
|
-
* invite row" / unlimited). Passed through to {@link assertSeatRemains} on every attempt.
|
|
2241
|
-
* @returns The `UseNumber` the successful attempt wrote under.
|
|
2242
|
-
*/
|
|
2243
|
-
private async withUseNumberRetry(
|
|
2244
|
-
token: string,
|
|
2245
|
-
operation: string,
|
|
2246
|
-
signal: AbortSignal | undefined,
|
|
2247
|
-
knownTotalUses: number | null | undefined,
|
|
2248
|
-
write: (useNumber: number) => Promise<void>,
|
|
2249
|
-
): Promise<number> {
|
|
2250
|
-
let lastError: unknown;
|
|
2251
|
-
for (let attempt = 1; attempt <= USE_NUMBER_ATTEMPTS; attempt++) {
|
|
2252
|
-
try {
|
|
2253
|
-
// lockedWithRetry carries the transient-cluster retry INSIDE this use-number
|
|
2254
|
-
// retry. The loops cannot multiply: a lost use number is a constraint failure,
|
|
2255
|
-
// which `isRetriableControlWriteFailure` never matches, and a transient cluster
|
|
2256
|
-
// failure is not a lost use number, so exactly one loop claims any given error.
|
|
2257
|
-
return await this.lockedWithRetry(async () => {
|
|
2258
|
-
// Inside the lock: a write still parked behind another writer when the caller's
|
|
2259
|
-
// budget expired is abandoned rather than executed.
|
|
2260
|
-
if (signal?.aborted) {
|
|
2261
|
-
throw new FormationAbortedError(token, operation);
|
|
2262
|
-
}
|
|
2263
|
-
const useNumber = await this.nextUseNumber(token);
|
|
2264
|
-
await this.assertSeatRemains(token, useNumber, knownTotalUses);
|
|
2265
|
-
await write(useNumber);
|
|
2266
|
-
return useNumber;
|
|
2267
|
-
});
|
|
2268
|
-
} catch (error) {
|
|
2269
|
-
if (!isLostUseNumberRace(error)) {
|
|
2270
|
-
throw error;
|
|
2271
|
-
}
|
|
2272
|
-
lastError = error;
|
|
2273
|
-
log('Use number for token %s was taken by another writer (attempt %d/%d): %s',
|
|
2274
|
-
token, attempt, USE_NUMBER_ATTEMPTS, error);
|
|
2275
|
-
}
|
|
2276
|
-
}
|
|
2277
|
-
// Out of attempts under sustained contention. Rethrowing the engine's own error keeps
|
|
2278
|
-
// today's behaviour: `StrandFormationManager.provisionAsResponder`'s catch-all maps it to
|
|
2279
|
-
// `Formation conflict, retry` and the joiner starts over with a fresh nonce.
|
|
2280
|
-
throw lastError;
|
|
2281
|
-
}
|
|
2282
|
-
|
|
2283
|
-
/**
|
|
2284
|
-
* Refuse an attempt that would consume a seat the invite does not have.
|
|
2346
|
+
* Refuse a redemption that would consume a seat the invite does not have.
|
|
2285
2347
|
*
|
|
2286
|
-
* `FormationUsage.Authorized`
|
|
2287
|
-
*
|
|
2288
|
-
*
|
|
2289
|
-
*
|
|
2290
|
-
*
|
|
2291
|
-
*
|
|
2292
|
-
* local write queue, so the loser's first attempt is already over budget, with no retry
|
|
2293
|
-
* involved.
|
|
2348
|
+
* The schema's count-based cap clause (`FormationUsage.Authorized`) refuses an over-limit
|
|
2349
|
+
* write at the database anyway — but it fails as a generic `CHECK constraint failed:
|
|
2350
|
+
* Authorized`, which the manager reports as a retryable conflict. Catching it here as
|
|
2351
|
+
* {@link InvitationExhaustedError} lets the joiner be told the invitation is spent instead
|
|
2352
|
+
* of being sent around a loop that can never close. Runs inside the write lock, so on a
|
|
2353
|
+
* same-node race the loser reads the winner's committed row and is refused by name.
|
|
2294
2354
|
*
|
|
2295
2355
|
* `knownTotalUses` lets a caller that already read the `FormationInvite` row (both
|
|
2296
2356
|
* production callers in `ControlFormationUsageRecorder` do) skip a second read on the common,
|
|
@@ -2300,11 +2360,10 @@ export class ControlDatabase {
|
|
|
2300
2360
|
* invite, before an outbound approval call). Safe today because `FormationInvite` is
|
|
2301
2361
|
* insert/delete only (its `Immutable` constraint), so the only way to stale it is an owner
|
|
2302
2362
|
* revoking the token and re-issuing it with MORE seats mid-redemption; if invites ever gain
|
|
2303
|
-
* an update path, drop the parameter and read here
|
|
2363
|
+
* an update path, drop the parameter and read here instead.
|
|
2304
2364
|
*/
|
|
2305
2365
|
private async assertSeatRemains(
|
|
2306
2366
|
token: string,
|
|
2307
|
-
useNumber: number,
|
|
2308
2367
|
knownTotalUses: number | null | undefined,
|
|
2309
2368
|
): Promise<void> {
|
|
2310
2369
|
// A missing invite is left to `Authorized` — it is not an exhaustion, and the CHECK's
|
|
@@ -2314,19 +2373,26 @@ export class ControlDatabase {
|
|
|
2314
2373
|
// is still open (tracked by `debt-composite-pk-point-lookup-unreliable-untracked`). A
|
|
2315
2374
|
// spurious empty result only costs the attempt its NAMED exhaustion error, reverting it to
|
|
2316
2375
|
// today's generic `Authorized` refusal — never a seat the invite does not have.
|
|
2376
|
+
// Both reads pass retry: false — assertSeatRemains only ever runs inside the locked
|
|
2377
|
+
// write bodies of redeemInvitation / recordFormationUsage, where a read retrying its
|
|
2378
|
+
// own backoff would sleep holding the write lock and the write funnel re-runs these
|
|
2379
|
+
// reads anyway (see the NOTE on readRows).
|
|
2317
2380
|
const totalUses = knownTotalUses !== undefined
|
|
2318
2381
|
? knownTotalUses
|
|
2319
|
-
: (await this.queryFormationInvite(token))?.totalUses ?? null;
|
|
2320
|
-
if (totalUses
|
|
2321
|
-
|
|
2382
|
+
: (await this.queryFormationInvite(token, false))?.totalUses ?? null;
|
|
2383
|
+
if (totalUses == null) {
|
|
2384
|
+
return;
|
|
2385
|
+
}
|
|
2386
|
+
const used = await this.countFormationUsage(token, false);
|
|
2387
|
+
if (used >= totalUses) {
|
|
2388
|
+
throw new InvitationExhaustedError(token, used, totalUses);
|
|
2322
2389
|
}
|
|
2323
2390
|
}
|
|
2324
2391
|
|
|
2325
2392
|
/** Parameterised `FormationUsage` insert shared by redeem + record paths. */
|
|
2326
2393
|
private async execFormationUsageInsert(opts: {
|
|
2327
2394
|
token: string;
|
|
2328
|
-
|
|
2329
|
-
/** Single-use nonce bound into the approver's signed digest; `unique`, so one approval spends once. */
|
|
2395
|
+
/** Single-use nonce bound into the approver's signed digest; the row's primary key, so one approval spends once. */
|
|
2330
2396
|
usageStampId: string;
|
|
2331
2397
|
disclosure: string;
|
|
2332
2398
|
strandId: string;
|
|
@@ -2348,12 +2414,12 @@ export class ControlDatabase {
|
|
|
2348
2414
|
// not a fix for an observable mis-ordering.
|
|
2349
2415
|
const nowCanonical = await canonicalDatetime(this.db!, opts.nowMs);
|
|
2350
2416
|
await this.db!.exec(`
|
|
2351
|
-
insert into CadreControl.FormationUsage (Token,
|
|
2417
|
+
insert into CadreControl.FormationUsage (Token, UsageStampId, PeerKey, PeerSig, Disclosure, StrandId, StrandStampId)
|
|
2352
2418
|
with context Now = ?, ValidationKey = ?, ValidationSignature = ?
|
|
2353
|
-
values (?, ?, ?, ?, ?, ?,
|
|
2419
|
+
values (?, ?, ?, ?, ?, ?, ?)
|
|
2354
2420
|
`, [
|
|
2355
2421
|
nowCanonical, opts.validationKey, opts.validationSignature,
|
|
2356
|
-
opts.token, opts.
|
|
2422
|
+
opts.token, opts.usageStampId, opts.peerKey,
|
|
2357
2423
|
opts.peerSignature, opts.disclosure, opts.strandId, opts.strandStampId,
|
|
2358
2424
|
]);
|
|
2359
2425
|
}
|
|
@@ -2362,8 +2428,12 @@ export class ControlDatabase {
|
|
|
2362
2428
|
* Read a `FormationInvite` row by token, or null when absent. `expiresAtMs` is
|
|
2363
2429
|
* the parsed epoch-ms of the stored `datetime` (null when the invite never
|
|
2364
2430
|
* expires); the caller compares it against the wall clock for freshness.
|
|
2431
|
+
*
|
|
2432
|
+
* `retry: false` is passed only by {@link assertSeatRemains}, which runs INSIDE a
|
|
2433
|
+
* locked write body — same per-call opt-out, and for the same reason, as
|
|
2434
|
+
* {@link queryStampId}'s. Every unlocked caller keeps the default retried path.
|
|
2365
2435
|
*/
|
|
2366
|
-
async queryFormationInvite(token: string): Promise<{
|
|
2436
|
+
async queryFormationInvite(token: string, retry = true): Promise<{
|
|
2367
2437
|
token: string;
|
|
2368
2438
|
sAppId: string;
|
|
2369
2439
|
expiresAtMs: number | null;
|
|
@@ -2372,10 +2442,8 @@ export class ControlDatabase {
|
|
|
2372
2442
|
strandId: string | null;
|
|
2373
2443
|
} | null> {
|
|
2374
2444
|
this.ensureInitialized();
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
[token]
|
|
2378
|
-
)) {
|
|
2445
|
+
const sql = 'select Token, sAppId, ExpiresAt, TotalUses, ValidationUrl, StrandId from CadreControl.FormationInvite where Token = ?';
|
|
2446
|
+
for (const row of await this.readRows(sql, [token], 'formation-invite', retry)) {
|
|
2379
2447
|
return {
|
|
2380
2448
|
token: row.Token as string,
|
|
2381
2449
|
sAppId: row.sAppId as string,
|
|
@@ -2390,13 +2458,32 @@ export class ControlDatabase {
|
|
|
2390
2458
|
|
|
2391
2459
|
/**
|
|
2392
2460
|
* Count `FormationUsage` rows recorded against a token (uses consumed so far).
|
|
2461
|
+
*
|
|
2462
|
+
* This count IS the seat cap (`enforceFormationUseCap`), so an under-report admits a seat
|
|
2463
|
+
* the invitation never paid for. It is served by a table SCAN, deliberately: a bare
|
|
2464
|
+
* equality on a non-key column is a shape the optimystic vtab declines to claim, so the
|
|
2465
|
+
* engine applies the filter itself over a full read of the table.
|
|
2466
|
+
*
|
|
2467
|
+
* NOTE: it was briefly an index seek, and that is what the scan is protecting against.
|
|
2468
|
+
* `formation-unique-token-redesign` took `Token` out of the primary key and declared
|
|
2469
|
+
* `FormationUsageByToken` purely to spare this scan, recording as a tripwire that an
|
|
2470
|
+
* under-reporting descent would admit an unpaid-for seat — "a failure mode a scan could
|
|
2471
|
+
* not produce". The tripwire fired. Across machines a secondary-index descent returns only
|
|
2472
|
+
* the rows the READING machine wrote, so this count under-reported without bound on a
|
|
2473
|
+
* multi-machine party and the cap over-admitted. The index was removed 2026-08-25; the
|
|
2474
|
+
* measurement is recorded in the block comment at the `FormationUsage` declaration in
|
|
2475
|
+
* `schemas/control.qsql`, and the engine defect in
|
|
2476
|
+
* `blocked/secondary-index-seek-blind-to-sibling-rows`. Do NOT re-declare the index to
|
|
2477
|
+
* speed this up while that ticket is open.
|
|
2478
|
+
*
|
|
2479
|
+
* `retry: false` is passed only by {@link assertSeatRemains}, which runs INSIDE a
|
|
2480
|
+
* locked write body — same per-call opt-out, and for the same reason, as
|
|
2481
|
+
* {@link queryStampId}'s. Every unlocked caller keeps the default retried path.
|
|
2393
2482
|
*/
|
|
2394
|
-
async countFormationUsage(token: string): Promise<number> {
|
|
2483
|
+
async countFormationUsage(token: string, retry = true): Promise<number> {
|
|
2395
2484
|
this.ensureInitialized();
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
[token]
|
|
2399
|
-
)) {
|
|
2485
|
+
const sql = 'select count(1) as Count from CadreControl.FormationUsage where Token = ?';
|
|
2486
|
+
for (const row of await this.readRows(sql, [token], 'formation-usage-count', retry)) {
|
|
2400
2487
|
return (row.Count as number) ?? 0;
|
|
2401
2488
|
}
|
|
2402
2489
|
return 0;
|
|
@@ -2428,8 +2515,10 @@ export class ControlDatabase {
|
|
|
2428
2515
|
// prune redeemed/expired rows.
|
|
2429
2516
|
const metered: Array<{ token: string; totalUses: number }> = [];
|
|
2430
2517
|
let unlimitedOutstanding = false;
|
|
2431
|
-
for
|
|
2432
|
-
'select Token, ExpiresAt, TotalUses from CadreControl.FormationInvite'
|
|
2518
|
+
for (const row of await this.readRows(
|
|
2519
|
+
'select Token, ExpiresAt, TotalUses from CadreControl.FormationInvite',
|
|
2520
|
+
undefined,
|
|
2521
|
+
'outstanding-invites'
|
|
2433
2522
|
)) {
|
|
2434
2523
|
const expiresAtMs = parseNullableStoredDatetimeMs(row.ExpiresAt as string | number | null);
|
|
2435
2524
|
if (expiresAtMs !== null && expiresAtMs <= nowMs) {
|
|
@@ -2445,6 +2534,12 @@ export class ControlDatabase {
|
|
|
2445
2534
|
if (unlimitedOutstanding) {
|
|
2446
2535
|
return true;
|
|
2447
2536
|
}
|
|
2537
|
+
// NOTE: this loop issues one retried read per metered invite, each carrying its own
|
|
2538
|
+
// CONTROL_READ_RETRY_BUDGET_MS, on a path the inbound admission gate awaits under a
|
|
2539
|
+
// 2 s FAIL-OPEN deadline — so with N unexpired metered invites the worst case is
|
|
2540
|
+
// (1 + N) budgets, not one. Fine while a cadre holds a couple of live invites and
|
|
2541
|
+
// transient read failures are rare; if a party ever keeps many metered invites open,
|
|
2542
|
+
// give this whole method one shared deadline rather than raising the per-read budget.
|
|
2448
2543
|
for (const invite of metered) {
|
|
2449
2544
|
if (await this.countFormationUsage(invite.token) < invite.totalUses) {
|
|
2450
2545
|
return true;
|
|
@@ -2453,23 +2548,6 @@ export class ControlDatabase {
|
|
|
2453
2548
|
return false;
|
|
2454
2549
|
}
|
|
2455
2550
|
|
|
2456
|
-
/**
|
|
2457
|
-
* Next `UseNumber` for a token = max(existing)+1, per the `Monotonic` constraint.
|
|
2458
|
-
*
|
|
2459
|
-
* Called only from inside {@link withUseNumberRetry}'s locked body — a read takes no
|
|
2460
|
-
* transaction of its own, so it is safe there, and reading it under the lock is what stops
|
|
2461
|
-
* two local redemptions of one token from picking the same number.
|
|
2462
|
-
*/
|
|
2463
|
-
private async nextUseNumber(token: string): Promise<number> {
|
|
2464
|
-
for await (const row of this.db!.eval(
|
|
2465
|
-
'select coalesce(max(UseNumber), 0) as MaxUse from CadreControl.FormationUsage where Token = ?',
|
|
2466
|
-
[token]
|
|
2467
|
-
)) {
|
|
2468
|
-
return (row.MaxUse as number) + 1;
|
|
2469
|
-
}
|
|
2470
|
-
return 1;
|
|
2471
|
-
}
|
|
2472
|
-
|
|
2473
2551
|
/**
|
|
2474
2552
|
* Close the database and cleanup resources.
|
|
2475
2553
|
*
|