@serfab/cadre-core 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +255 -255
- package/dist/announce-addrs.d.ts +49 -0
- package/dist/announce-addrs.js +78 -0
- package/dist/announce-addrs.js.map +1 -0
- package/dist/cadre-node.d.ts +484 -58
- package/dist/cadre-node.js +953 -182
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-cohort.d.ts +26 -0
- package/dist/control-cohort.js +26 -0
- package/dist/control-cohort.js.map +1 -1
- package/dist/control-database.d.ts +222 -125
- package/dist/control-database.js +356 -266
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +8 -7
- package/dist/control-formation-recorder.js +8 -7
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-read-retry.d.ts +103 -0
- package/dist/control-read-retry.js +145 -0
- package/dist/control-read-retry.js.map +1 -0
- package/dist/control-retry.d.ts +101 -0
- package/dist/control-retry.js +108 -0
- package/dist/control-retry.js.map +1 -0
- package/dist/control-schema.d.ts +1 -1
- package/dist/control-schema.js +805 -774
- package/dist/control-schema.js.map +1 -1
- package/dist/control-write-retry.d.ts +65 -31
- package/dist/control-write-retry.js +33 -103
- package/dist/control-write-retry.js.map +1 -1
- package/dist/delegate-admission.d.ts +17 -15
- package/dist/delegate-admission.js +17 -15
- package/dist/delegate-admission.js.map +1 -1
- package/dist/index.d.ts +7 -5
- package/dist/index.js +18 -9
- package/dist/index.js.map +1 -1
- package/dist/membership-connection-gater.d.ts +211 -25
- package/dist/membership-connection-gater.js +309 -29
- package/dist/membership-connection-gater.js.map +1 -1
- package/dist/peer-addr-book.d.ts +92 -0
- package/dist/peer-addr-book.js +220 -0
- package/dist/peer-addr-book.js.map +1 -0
- package/dist/peer-record.d.ts +26 -0
- package/dist/peer-record.js +32 -0
- package/dist/peer-record.js.map +1 -1
- package/dist/relay-addrs.d.ts +76 -26
- package/dist/relay-addrs.js +115 -27
- package/dist/relay-addrs.js.map +1 -1
- package/dist/relay-reservation.d.ts +20 -15
- package/dist/relay-reservation.js +18 -14
- package/dist/relay-reservation.js.map +1 -1
- package/dist/seed-bootstrap.d.ts +7 -0
- package/dist/seed-bootstrap.js +15 -13
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/strand-addr-protocol.js +16 -0
- package/dist/strand-addr-protocol.js.map +1 -1
- package/dist/strand-backfill.js +13 -4
- package/dist/strand-backfill.js.map +1 -1
- package/dist/strand-cohort.d.ts +4 -37
- package/dist/strand-cohort.js +4 -15
- package/dist/strand-cohort.js.map +1 -1
- package/dist/strand-database.d.ts +11 -16
- package/dist/strand-database.js +19 -7
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-formation-manager.d.ts +18 -17
- package/dist/strand-formation-manager.js +25 -27
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +7 -7
- package/dist/strand-formation-protocol.js +8 -8
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +53 -21
- package/dist/strand-instance-manager.js +128 -37
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +113 -19
- package/dist/strand-membership-writer.js +219 -51
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-solicitation.d.ts +3 -3
- package/dist/strand-solicitation.js +4 -5
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-wake-protocol.d.ts +47 -2
- package/dist/strand-wake-protocol.js +76 -6
- package/dist/strand-wake-protocol.js.map +1 -1
- package/dist/types.d.ts +107 -54
- package/dist/types.js.map +1 -1
- package/package.json +8 -7
- package/src/announce-addrs.ts +93 -0
- package/src/arachnode-stub.ts +159 -159
- package/src/bootstrap-peer-store-file.ts +44 -44
- package/src/bootstrap-peer-store.ts +172 -172
- package/src/cadre-node.ts +1026 -194
- package/src/canonical-json.ts +30 -30
- package/src/control-authorization.ts +114 -114
- package/src/control-cohort.ts +130 -103
- package/src/control-database.ts +376 -298
- package/src/control-formation-recorder.ts +305 -304
- package/src/control-read-retry.ts +167 -0
- package/src/control-retry.ts +190 -0
- package/src/control-schema.ts +817 -786
- package/src/control-stream.ts +205 -205
- package/src/control-write-retry.ts +383 -473
- package/src/delegate-admission.ts +304 -302
- package/src/device-token.ts +94 -94
- package/src/diagnostics/connection-path.ts +277 -277
- package/src/diagnostics/webrtc-turn-tracker.ts +222 -222
- package/src/ed25519-key.ts +134 -134
- package/src/enrollment.ts +199 -199
- package/src/file-durable-slot.ts +61 -61
- package/src/formation-approval.ts +604 -604
- package/src/fs-atomic.ts +107 -107
- package/src/hibernation-manager.ts +437 -437
- package/src/identity-key.ts +109 -109
- package/src/index.ts +435 -406
- package/src/key-store-file.ts +132 -132
- package/src/key-store.ts +99 -99
- package/src/membership-connection-gater.ts +368 -39
- package/src/node-local-snapshot.ts +260 -260
- package/src/peer-addr-book.ts +250 -0
- package/src/peer-authorization.ts +259 -259
- package/src/peer-record.ts +197 -164
- package/src/push-credentials.ts +84 -84
- package/src/push-node.ts +64 -64
- package/src/push-notifier-apns.ts +247 -247
- package/src/push-notifier-fcm.ts +220 -220
- package/src/push-notifier-shared.ts +28 -28
- package/src/push-notifier.ts +52 -52
- package/src/relay-addrs.ts +134 -27
- package/src/relay-reservation.ts +22 -17
- package/src/seed-bootstrap.ts +15 -15
- package/src/seed-trust-policy.ts +140 -140
- package/src/strand-addr-protocol.ts +458 -441
- package/src/strand-backfill.ts +28 -5
- package/src/strand-cohort.ts +38 -77
- package/src/strand-database.ts +202 -204
- package/src/strand-formation-manager.ts +26 -28
- package/src/strand-formation-protocol.ts +692 -692
- package/src/strand-instance-manager.ts +636 -553
- package/src/strand-member-key.ts +39 -39
- package/src/strand-member-registry.ts +182 -182
- package/src/strand-membership-writer.ts +1627 -1430
- package/src/strand-solicitation.ts +471 -472
- package/src/strand-transport-key.ts +58 -58
- package/src/strand-wake-payload.ts +32 -32
- package/src/strand-wake-protocol.ts +400 -314
- package/src/strand-watcher.ts +314 -314
- package/src/trusted-owner-store-file.ts +41 -41
- package/src/trusted-owner-store.ts +171 -171
- package/src/types.ts +1354 -1303
package/dist/control-database.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import debug from 'debug';
|
|
2
2
|
import { toString as uint8ArrayToString } from 'uint8arrays';
|
|
3
|
-
import { Database, registerPlugin
|
|
3
|
+
import { Database, registerPlugin } from '@quereus/quereus';
|
|
4
4
|
import cryptoPlugin from '@optimystic/quereus-plugin-crypto/plugin';
|
|
5
5
|
import optimysticPlugin from '@optimystic/quereus-plugin-optimystic/plugin';
|
|
6
6
|
import { digest, randomBytes } from '@optimystic/quereus-plugin-crypto';
|
|
@@ -9,6 +9,7 @@ import { canonicalDatetime } from './canonical-datetime.js';
|
|
|
9
9
|
import { controlAuthorizationFields, CONTROL_TABLES } from './control-authorization.js';
|
|
10
10
|
import { requireEd25519PublicKeyB64 } from './ed25519-key.js';
|
|
11
11
|
import { retryControlWrite, SCHEMA_INIT_RETRY_POLICY } from './control-write-retry.js';
|
|
12
|
+
import { retryControlRead } from './control-read-retry.js';
|
|
12
13
|
const log = debug('sereus:cadre:control-db');
|
|
13
14
|
const timing = debug('sereus:cadre:timing');
|
|
14
15
|
/**
|
|
@@ -73,94 +74,30 @@ export class FormationAbortedError extends Error {
|
|
|
73
74
|
}
|
|
74
75
|
}
|
|
75
76
|
/**
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
77
|
+
* The invitation's seat budget is already spent: the count of recorded `FormationUsage` rows
|
|
78
|
+
* for the token has reached `FormationInvite.TotalUses`, so this redemption cannot be given a
|
|
79
|
+
* seat.
|
|
79
80
|
*
|
|
80
|
-
* Raised
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
*
|
|
81
|
+
* Raised off the committed-count check ({@link ControlDatabase.assertSeatRemains}) ahead of
|
|
82
|
+
* the write, inside {@link ControlDatabase.redeemInvitation} /
|
|
83
|
+
* {@link ControlDatabase.recordFormationUsage} — and off nothing else. It exists because
|
|
84
|
+
* without it the refusal surfaces as a generic `CHECK constraint failed: Authorized` from the
|
|
85
|
+
* schema's own count-based cap clause, which the manager reports as a retryable
|
|
86
|
+
* `Formation conflict, retry` — telling the joiner to retry something that can never succeed.
|
|
87
|
+
* Two redemptions of one token racing on the SAME node serialize behind the local write
|
|
88
|
+
* queue, so the loser reads the winner's committed row and is refused here by name —
|
|
89
|
+
* `StrandFormationManager.validateToken`'s `isTokenUsed` check cannot catch that, since it
|
|
90
|
+
* ran before either redemption wrote anything.
|
|
88
91
|
*/
|
|
89
92
|
export class InvitationExhaustedError extends Error {
|
|
90
|
-
constructor(token,
|
|
91
|
-
super(`Invitation ${token} is exhausted:
|
|
93
|
+
constructor(token, usesRecorded, totalUses) {
|
|
94
|
+
super(`Invitation ${token} is exhausted: ${usesRecorded} of ${totalUses} use(s) already recorded`);
|
|
92
95
|
this.token = token;
|
|
93
|
-
this.
|
|
96
|
+
this.usesRecorded = usesRecorded;
|
|
94
97
|
this.totalUses = totalUses;
|
|
95
98
|
this.name = 'InvitationExhaustedError';
|
|
96
99
|
}
|
|
97
100
|
}
|
|
98
|
-
/**
|
|
99
|
-
* Attempts allowed per redemption: the first, plus two retries of a use number another
|
|
100
|
-
* writer took. Bounded so sustained contention terminates in a clean rejection rather than
|
|
101
|
-
* spinning; two retries is enough for the realistic case (a handful of nodes redeeming one
|
|
102
|
-
* multi-use token at once) without holding the write lock hostage.
|
|
103
|
-
*
|
|
104
|
-
* NOTE: the attempts fire back-to-back, with no backoff and no jitter — each one only takes
|
|
105
|
-
* and releases the write lock. Fine while cross-node contention on a single token is a
|
|
106
|
-
* handful of writers; if one hot multi-use token ever draws sustained concurrent redemption
|
|
107
|
-
* (a broadcast invite, a bulk onboarding run), all three attempts can burn against the same
|
|
108
|
-
* competitor and the caller gets the engine's error. Add a short randomised delay between
|
|
109
|
-
* attempts then — NOT a longer attempt budget, which would hold the write queue.
|
|
110
|
-
*/
|
|
111
|
-
const USE_NUMBER_ATTEMPTS = 3;
|
|
112
|
-
/**
|
|
113
|
-
* The two error messages that mean "another writer took the `UseNumber` we picked", and
|
|
114
|
-
* nothing else.
|
|
115
|
-
*
|
|
116
|
-
* - the `(Token, UseNumber)` primary-key collision, which the optimystic vtab reports on the
|
|
117
|
-
* insert itself (`uniqueConstraintMessage` renders the columns table-qualified, no schema
|
|
118
|
-
* prefix);
|
|
119
|
-
* - `CHECK constraint failed: Monotonic` — `FormationUsage.Monotonic` contains a subquery, so
|
|
120
|
-
* Quereus defers it to commit where it reads `committed.FormationUsage`. A concurrent row
|
|
121
|
-
* that our insert's key probe did not see still trips it there. Same meaning, different
|
|
122
|
-
* surface.
|
|
123
|
-
*
|
|
124
|
-
* Deliberately NOT matched: `FormationUsage.UsageStampId`'s `unique` (a repeated nonce is
|
|
125
|
-
* replay, never a race — retrying it would re-present a spent approval), and every named
|
|
126
|
-
* CHECK other than `Monotonic`. `Authorized` in particular renders with the same
|
|
127
|
-
* `CHECK constraint failed:` prefix, so the constraint NAME is anchored with `\b` exactly as
|
|
128
|
-
* `expectConstraintFailure` does rather than matched by prefix.
|
|
129
|
-
*
|
|
130
|
-
* NOTE: a CHECK failure names only the CONSTRAINT, never its table, and `Monotonic` is a
|
|
131
|
-
* generic enough name that a second table could plausibly claim it. Today it is unique across
|
|
132
|
-
* `schemas/control.qsql`, so the pattern cannot mis-fire; if another table ever declares a
|
|
133
|
-
* `Monotonic` constraint, a failure of THAT constraint would be read here as a lost use-number
|
|
134
|
-
* race and pointlessly retried up to {@link USE_NUMBER_ATTEMPTS} times. Rename the other
|
|
135
|
-
* constraint (cheapest) rather than trying to recover the table from the message.
|
|
136
|
-
*/
|
|
137
|
-
const LOST_USE_NUMBER_PATTERNS = [
|
|
138
|
-
/UNIQUE constraint failed: FormationUsage\.Token, FormationUsage\.UseNumber/i,
|
|
139
|
-
/CHECK constraint failed: Monotonic\b/,
|
|
140
|
-
];
|
|
141
|
-
/**
|
|
142
|
-
* Did this write fail purely because another writer took the `UseNumber` it picked — i.e. is
|
|
143
|
-
* re-presenting the SAME approval under the next free use number the right response?
|
|
144
|
-
*
|
|
145
|
-
* Classifies by MESSAGE, walking the `cause` chain with Quereus' own {@link unwrapError}:
|
|
146
|
-
* neither surface can be recognised by type. The deferred `Monotonic` failure is thrown as a
|
|
147
|
-
* bare `QuereusError`, not a `ConstraintError` (see
|
|
148
|
-
* `quereus/src/runtime/deferred-constraint-queue.ts`), so an `instanceof ConstraintError` gate
|
|
149
|
-
* would silently drop the branch this retry exists for. Messages also carry an
|
|
150
|
-
* ` (at line N, column M)` suffix and arrive wrapped when the write ran inside a transaction,
|
|
151
|
-
* so the match is a substring test rather than an equality one.
|
|
152
|
-
*
|
|
153
|
-
* NOTE: this is the one piece of the retry that depends on engine error TEXT. Its own test
|
|
154
|
-
* (`control-formation-use-number-retry.spec.ts`) produces all three messages from the real
|
|
155
|
-
* engine rather than from string literals, so a reworded storage-layer error fails that spec
|
|
156
|
-
* instead of silently disabling the retry.
|
|
157
|
-
*/
|
|
158
|
-
export function isLostUseNumberRace(error) {
|
|
159
|
-
if (!(error instanceof Error)) {
|
|
160
|
-
return false;
|
|
161
|
-
}
|
|
162
|
-
return unwrapError(error).some(({ message }) => LOST_USE_NUMBER_PATTERNS.some(pattern => pattern.test(message)));
|
|
163
|
-
}
|
|
164
101
|
/**
|
|
165
102
|
* Parse a stored Quereus `datetime` value into epoch milliseconds.
|
|
166
103
|
*
|
|
@@ -312,6 +249,13 @@ export class ControlDatabase {
|
|
|
312
249
|
* having to restate the policy's attempts or classifier.
|
|
313
250
|
*/
|
|
314
251
|
this.controlWriteRetryPacing = {};
|
|
252
|
+
/**
|
|
253
|
+
* Pacing seams for {@link readRows}'s retry — the read twin of
|
|
254
|
+
* {@link controlWriteRetryPacing}, with the same contract: production leaves this
|
|
255
|
+
* empty, specs inject a recorded `sleep` / fake `now` so no test waits out a real
|
|
256
|
+
* backoff. Applied OVER the read policy's defaults but UNDER the call site's `label`.
|
|
257
|
+
*/
|
|
258
|
+
this.controlReadRetryPacing = {};
|
|
315
259
|
this.config = config;
|
|
316
260
|
}
|
|
317
261
|
/**
|
|
@@ -377,6 +321,20 @@ export class ControlDatabase {
|
|
|
377
321
|
timing('[controlDb] hydrate: %dms (tables=%d, indexes=%d)', Math.round(performance.now() - t0), hydrated.tables, hydrated.indexes);
|
|
378
322
|
log('Hydrated control catalog (tables=%d, indexes=%d)', hydrated.tables, hydrated.indexes);
|
|
379
323
|
// Load and execute the schema
|
|
324
|
+
// NOTE: this is where a slow launch is felt. Duration is (raw-storage operations
|
|
325
|
+
// issued) × per-operation storage latency: ~1ms/op on an idle machine, but
|
|
326
|
+
// 50-90ms/op on a loaded disk or a phone's flash under launch contention. A cold
|
|
327
|
+
// start now reaches the backend 172 times (8 tables + 1 index, 21 distinct blocks
|
|
328
|
+
// — dominated by its 130 genuine writes), a warm restart 52, because cadre-core
|
|
329
|
+
// wraps every embedder storage in `@optimystic/db-p2p`'s write-through cache
|
|
330
|
+
// (@serfab/quereus-plugin-sereus's cached-storage.ts). Uncached the same start issued ~2000 operations — the
|
|
331
|
+
// upstream re-read amplification measured in
|
|
332
|
+
// tickets/complete/optimystic-block-read-amplification-on-control-start.md (and
|
|
333
|
+
// its follow-up ...-schema-catalog-reread-per-write-blows-storage-budgets.md),
|
|
334
|
+
// which also rules out the retry policy and cluster deadlines. The counts are
|
|
335
|
+
// pinned by packages/cadre-core/test/control-start-storage-op-budget.spec.ts; if
|
|
336
|
+
// they grow — e.g. the cache leaves this path — that spec fails before anyone
|
|
337
|
+
// has to debug a launch.
|
|
380
338
|
t0 = performance.now();
|
|
381
339
|
await this.loadSchema();
|
|
382
340
|
timing('[controlDb] loadSchema: %dms', Math.round(performance.now() - t0));
|
|
@@ -443,16 +401,129 @@ export class ControlDatabase {
|
|
|
443
401
|
// also what lets schema init absorb optimystic's self-coordination grace refusal, which
|
|
444
402
|
// is unsafe for control writes in general. Why, on
|
|
445
403
|
// `RETRIABLE_SCHEMA_INIT_MATCHERS` in `control-write-retry.ts`.
|
|
446
|
-
await this.lockedWithRetry(() => this.db.exec(schemaContent), SCHEMA_INIT_RETRY_POLICY);
|
|
404
|
+
await this.lockedWithRetry(() => this.db.exec(schemaContent), SCHEMA_INIT_RETRY_POLICY, 'schema-init');
|
|
447
405
|
log('Schema loaded and executed');
|
|
448
406
|
}
|
|
407
|
+
/**
|
|
408
|
+
* One UNRETRIED drain of a control read to rows — every read in this class bottoms out
|
|
409
|
+
* here (almost all via {@link readRows}, which adds the transient-failure retry), and
|
|
410
|
+
* this is the ONE place the committed-read opt-in is spelled.
|
|
411
|
+
*
|
|
412
|
+
* Reached only through {@link readRows}, which drops straight here for the call sites
|
|
413
|
+
* that pass `retry: false` — reads issued from INSIDE a locked write body, where a
|
|
414
|
+
* backoff sleep would hold the write lock and the write funnel re-runs the read anyway
|
|
415
|
+
* (see the NOTE on {@link readRows}).
|
|
416
|
+
*
|
|
417
|
+
* A control read normally queues on the database's exec mutex, so it does not answer
|
|
418
|
+
* until whatever statement holds that mutex finishes. When a control WRITE is parked
|
|
419
|
+
* against an unresponsive cohort member that is minutes, and a read that only wants
|
|
420
|
+
* the last committed state waits the whole time for no reason
|
|
421
|
+
* (`complete/control-reads-blocked-by-stalled-write`).
|
|
422
|
+
*
|
|
423
|
+
* Quereus's remedy is `readConcurrency: 'committed'` — a PER-CALL opt-in
|
|
424
|
+
* (`Statement.tryRouteConcurrent` refuses anything that did not ask for it by name)
|
|
425
|
+
* that runs an eligible read off the mutex against each table's last committed state.
|
|
426
|
+
* Routing is best-effort and never an error: an ineligible statement falls back to
|
|
427
|
+
* the serialized path silently — any side-effecting node, any table-valued function
|
|
428
|
+
* call (pure ones included), any table whose module does not declare
|
|
429
|
+
* `readCommittedSnapshot`, or an EXPLICIT transaction being open on this database.
|
|
430
|
+
*
|
|
431
|
+
* **It is asked for only while a write is in flight, and that condition is the whole
|
|
432
|
+
* point.** The opt-in does not merely move the read off the mutex: it connects the
|
|
433
|
+
* table with `_readCommitted`, and the optimystic vtab serves such a connection from
|
|
434
|
+
* a pinned pre-transaction snapshot that NEVER refreshes from the network, where an
|
|
435
|
+
* ordinary read calls `update()` on the collection first. Taking that path
|
|
436
|
+
* unconditionally therefore trades "reads block behind a stalled write" for "reads
|
|
437
|
+
* stop seeing what other machines wrote" — measured, not theorised: with every read
|
|
438
|
+
* opted in, `control-write-degraded-cohort-member` could no longer observe a sibling
|
|
439
|
+
* publishing its own `CadrePeer` row and failed at suite setup on two runs out of two.
|
|
440
|
+
*
|
|
441
|
+
* `getAutocommit()` reports the whole `Database` rather than this call (the same
|
|
442
|
+
* property {@link assertCommitBoundary} leans on), which is exactly what is wanted:
|
|
443
|
+
* false means some writer's transaction is open, which is the only situation where a
|
|
444
|
+
* read would have blocked. The check races that writer harmlessly in both directions
|
|
445
|
+
* — losing it either reads normally (today's behaviour) or reads committed state.
|
|
446
|
+
*
|
|
447
|
+
* What a committed read gives up is observing a write that has not finished
|
|
448
|
+
* committing, and that is safe for every caller here: an AWAITED write is committed by
|
|
449
|
+
* the time it returns, and a read inside {@link inTransaction} is disqualified by the
|
|
450
|
+
* open explicit transaction and stays serialized, so it still sees its own
|
|
451
|
+
* transaction's rows. A read after an UNAWAITED write on this database would be wrong
|
|
452
|
+
* — no such caller exists, and {@link withWriteLock} is what keeps it that way.
|
|
453
|
+
*/
|
|
454
|
+
async readRowsOnce(sql, params) {
|
|
455
|
+
let iterator;
|
|
456
|
+
if (this.db.getAutocommit()) {
|
|
457
|
+
iterator = this.db.eval(sql, params);
|
|
458
|
+
}
|
|
459
|
+
else {
|
|
460
|
+
log('read-eval: a write is in flight, asking for a committed read: %s', sql);
|
|
461
|
+
iterator = this.db.eval(sql, params, { readConcurrency: 'committed' });
|
|
462
|
+
}
|
|
463
|
+
const rows = [];
|
|
464
|
+
for await (const row of iterator) {
|
|
465
|
+
rows.push(row);
|
|
466
|
+
}
|
|
467
|
+
return rows;
|
|
468
|
+
}
|
|
469
|
+
/**
|
|
470
|
+
* {@link readRowsOnce} plus the bounded transient-failure retry
|
|
471
|
+
* ({@link retryControlRead}) — the funnel every UNLOCKED control read goes through, so a
|
|
472
|
+
* read that failed because the cluster could not be asked properly (a stream reset
|
|
473
|
+
* mid-scan, a partially unreachable cohort) is re-presented a moment later instead of
|
|
474
|
+
* surfacing to every reader.
|
|
475
|
+
*
|
|
476
|
+
* MATERIALIZES the read rather than returning a lazy iterator, and that shape is what
|
|
477
|
+
* makes the retry possible at all: the failure happens during ITERATION, not at call
|
|
478
|
+
* time, and a half-consumed iterator cannot be retried without re-yielding rows the
|
|
479
|
+
* caller already saw. Materializing is safe here — every caller either drains to an
|
|
480
|
+
* array anyway or takes the first row of a statement that yields at most one (a
|
|
481
|
+
* primary-key lookup or a `count(1)`); nothing streams.
|
|
482
|
+
*
|
|
483
|
+
* Each ATTEMPT is a fresh {@link readRowsOnce} call, so the committed-read opt-in
|
|
484
|
+
* (`getAutocommit()`) is re-evaluated per attempt — a write can finish between attempts
|
|
485
|
+
* and change the right answer.
|
|
486
|
+
*
|
|
487
|
+
* The retry's budget ({@link CONTROL_READ_RETRY_BUDGET_MS}, 1.5 s) is deliberately far
|
|
488
|
+
* under the write funnel's: the tightest caller deadline over a control read is the
|
|
489
|
+
* inbound admission gate's 2 s FAIL-OPEN timeout, and a read that outlives it spends its
|
|
490
|
+
* retries after the gate has already admitted (see `control-read-retry.ts`).
|
|
491
|
+
*
|
|
492
|
+
* `label` names the read in the retry loop's `Control read [<label>] …` debug lines and
|
|
493
|
+
* nothing else; several reads are in flight concurrently in a real party, so an
|
|
494
|
+
* unlabelled line cannot be attributed — keep new call sites labelled.
|
|
495
|
+
*
|
|
496
|
+
* `retry: false` drops straight to {@link readRowsOnce}: reads issued from INSIDE a
|
|
497
|
+
* locked write body must not retry on their own. `retryControlWrite`'s contract is that
|
|
498
|
+
* backoff sleeps happen with NO lock held; a read retrying its own backoff inside a
|
|
499
|
+
* locked body would sleep holding the write lock (stalling every other local writer),
|
|
500
|
+
* and the write funnel already re-runs the body's reads when it re-runs the body. See
|
|
501
|
+
* {@link queryStampId} / {@link assertSeatRemains} / {@link queryCadrePeers} for the
|
|
502
|
+
* per-call opt-outs.
|
|
503
|
+
*
|
|
504
|
+
* NOTE: nothing structural enforces that opt-out — it is per-call because an unlocked
|
|
505
|
+
* read runs CONCURRENTLY with a locked body, so `this`-state cannot tell the two apart,
|
|
506
|
+
* and there is no async-context primitive available on every target platform (browser,
|
|
507
|
+
* React Native) to carry the answer. Reads reached through a CALLBACK invoked under the
|
|
508
|
+
* lock are the easy miss: `notifyMembershipChanged` runs its listener with the lock
|
|
509
|
+
* held, and its membership read is opted out at {@link queryCadrePeers}. If another
|
|
510
|
+
* under-lock callback seam is ever added, audit its read graph the same way.
|
|
511
|
+
*/
|
|
512
|
+
readRows(sql, params, label, retry = true) {
|
|
513
|
+
if (!retry) {
|
|
514
|
+
return this.readRowsOnce(sql, params);
|
|
515
|
+
}
|
|
516
|
+
// The label lands LAST so the call site's name survives the spec-injected pacing —
|
|
517
|
+
// it changes log attribution only, never behaviour (same layering as lockedWithRetry).
|
|
518
|
+
return retryControlRead(() => this.readRowsOnce(sql, params), { ...this.controlReadRetryPacing, label });
|
|
519
|
+
}
|
|
449
520
|
/**
|
|
450
521
|
* Query all strands from the control database
|
|
451
522
|
*/
|
|
452
523
|
async queryStrands() {
|
|
453
524
|
this.ensureInitialized();
|
|
454
525
|
const results = [];
|
|
455
|
-
for
|
|
526
|
+
for (const row of await this.readRows('select Id, MemberPrivateKey, Type from CadreControl.Strand', undefined, 'strands')) {
|
|
456
527
|
results.push({
|
|
457
528
|
Id: row.Id,
|
|
458
529
|
MemberPrivateKey: row.MemberPrivateKey,
|
|
@@ -469,7 +540,7 @@ export class ControlDatabase {
|
|
|
469
540
|
*/
|
|
470
541
|
async queryStrand(strandId) {
|
|
471
542
|
this.ensureInitialized();
|
|
472
|
-
for
|
|
543
|
+
for (const row of await this.readRows('select Id, MemberPrivateKey, Type from CadreControl.Strand where Id = ?', [strandId], 'strand')) {
|
|
473
544
|
return {
|
|
474
545
|
Id: row.Id,
|
|
475
546
|
MemberPrivateKey: row.MemberPrivateKey,
|
|
@@ -494,7 +565,7 @@ export class ControlDatabase {
|
|
|
494
565
|
if (!CONTROL_TABLE_SET.has(table)) {
|
|
495
566
|
throw new Error(`Unknown CadreControl table: ${table}`);
|
|
496
567
|
}
|
|
497
|
-
for
|
|
568
|
+
for (const row of await this.readRows(`select count(1) as Count from CadreControl.${table}`, undefined, 'count-rows')) {
|
|
498
569
|
return row.Count ?? 0;
|
|
499
570
|
}
|
|
500
571
|
return 0;
|
|
@@ -512,7 +583,7 @@ export class ControlDatabase {
|
|
|
512
583
|
*/
|
|
513
584
|
async hasOwnerKey() {
|
|
514
585
|
this.ensureInitialized();
|
|
515
|
-
for
|
|
586
|
+
for (const row of await this.readRows('select count(1) as Count from CadreControl.OwnerKey', undefined, 'owner-key-exists')) {
|
|
516
587
|
return row.Count > 0;
|
|
517
588
|
}
|
|
518
589
|
return false;
|
|
@@ -543,7 +614,7 @@ export class ControlDatabase {
|
|
|
543
614
|
async getOwnerKeys() {
|
|
544
615
|
this.ensureInitialized();
|
|
545
616
|
const keys = new Set();
|
|
546
|
-
for
|
|
617
|
+
for (const row of await this.readRows('select Key from CadreControl.OwnerKey', undefined, 'owner-keys')) {
|
|
547
618
|
keys.add(row.Key);
|
|
548
619
|
}
|
|
549
620
|
return keys;
|
|
@@ -564,12 +635,29 @@ export class ControlDatabase {
|
|
|
564
635
|
* membership-gate refresh path; control tables are a handful of rows today, so this
|
|
565
636
|
* is cheap — if they ever grow, fold the exclusion into one statement or cache the
|
|
566
637
|
* retired-stamp set.
|
|
638
|
+
*
|
|
639
|
+
* NOTE: this filter is HAND-MIRRORED by the gate suites' fake control databases
|
|
640
|
+
* (`test/membership-gate-helpers.ts` and `test/cadre-node-authorized-surface.spec.ts`),
|
|
641
|
+
* which feed the node rows directly. Nothing enforces that the copies stay true, so a
|
|
642
|
+
* change to the filter's SHAPE here (what it keys on, which table it consults) must be
|
|
643
|
+
* carried into both, or those suites keep passing against a contract the database no
|
|
644
|
+
* longer honours. Fine while the filter is one `Set.has` on `StampId`; if it grows a
|
|
645
|
+
* second dimension, give the fakes a shared fixture or a contract test instead.
|
|
646
|
+
*
|
|
647
|
+
* `retry: false` is passed by exactly one caller —
|
|
648
|
+
* `CadreNode.refreshAuthorizedControlPeers`, the membership-gate snapshot refresh —
|
|
649
|
+
* because {@link notifyMembershipChanged} invokes that listener with the write lock
|
|
650
|
+
* HELD, and a retrying read there would sleep its backoff holding the lock (see the
|
|
651
|
+
* NOTE on {@link readRows}). It costs nothing: that refresh is best-effort, keeps the
|
|
652
|
+
* previous snapshot on failure, and is re-driven by the next write and by the timed
|
|
653
|
+
* reconcile. BOTH of this method's reads take the flag — retrying either one under the
|
|
654
|
+
* lock is the same defect.
|
|
567
655
|
*/
|
|
568
|
-
async queryCadrePeers() {
|
|
656
|
+
async queryCadrePeers(retry = true) {
|
|
569
657
|
this.ensureInitialized();
|
|
570
|
-
const revoked = await this.queryRevokedStamps('CadrePeer');
|
|
658
|
+
const revoked = await this.queryRevokedStamps('CadrePeer', retry);
|
|
571
659
|
const rows = [];
|
|
572
|
-
for
|
|
660
|
+
for (const row of await this.readRows('select PeerId, Multiaddr, StampId, VouchOwner, VouchSig from CadreControl.CadrePeer', undefined, 'cadre-peers', retry)) {
|
|
573
661
|
const stampId = row.StampId ?? null;
|
|
574
662
|
if (stampId !== null && revoked.has(stampId)) {
|
|
575
663
|
continue;
|
|
@@ -596,7 +684,7 @@ export class ControlDatabase {
|
|
|
596
684
|
async queryValidationKeys() {
|
|
597
685
|
this.ensureInitialized();
|
|
598
686
|
const keys = [];
|
|
599
|
-
for
|
|
687
|
+
for (const row of await this.readRows('select Key from CadreControl.ValidationKey', undefined, 'validation-keys')) {
|
|
600
688
|
keys.push(row.Key);
|
|
601
689
|
}
|
|
602
690
|
return keys.sort();
|
|
@@ -615,11 +703,21 @@ export class ControlDatabase {
|
|
|
615
703
|
* `table` and its {@link GUARDED_KEY_COLUMN} column are interpolated into the SQL; both
|
|
616
704
|
* come from closed literal unions — no caller-supplied string can reach the statement
|
|
617
705
|
* (same injection-surface discipline as {@link countRows}'s `CONTROL_TABLE_SET` guard).
|
|
706
|
+
*
|
|
707
|
+
* NOTE: `retry: false` is passed by exactly the call sites that run INSIDE a locked
|
|
708
|
+
* write body — {@link deleteGuardedRow} and {@link insertCadrePeer}'s insert-if-absent
|
|
709
|
+
* guard. A read retrying there would sleep its backoff HOLDING the write lock
|
|
710
|
+
* (`retryControlWrite`'s contract is that backoff sleeps happen with no lock held), and
|
|
711
|
+
* the write funnel already re-runs the body's reads when it re-runs the body. The
|
|
712
|
+
* unlocked callers — the public stamp readers, {@link reauthorizeCadrePeer} and
|
|
713
|
+
* {@link reapRevokedRow}, whose stamp reads run BEFORE their locks are taken — keep the
|
|
714
|
+
* default retried path. Explicit per call rather than an ambient flag: unlocked reads
|
|
715
|
+
* run concurrently with a locked body, so `this`-state cannot tell the two apart.
|
|
618
716
|
*/
|
|
619
|
-
async queryStampId(table, keyValue) {
|
|
717
|
+
async queryStampId(table, keyValue, retry = true) {
|
|
620
718
|
this.ensureInitialized();
|
|
621
719
|
const sql = `select StampId from CadreControl.${table} where ${GUARDED_KEY_COLUMN[table]} = ?`;
|
|
622
|
-
for
|
|
720
|
+
for (const row of await this.readRows(sql, [keyValue], `stamp-${table}`, retry)) {
|
|
623
721
|
return row.StampId ?? null;
|
|
624
722
|
}
|
|
625
723
|
return null;
|
|
@@ -664,11 +762,14 @@ export class ControlDatabase {
|
|
|
664
762
|
* NOTE: re-reads the whole retired set on every call, and the caller runs per inbound
|
|
665
763
|
* gate request while the table only ever grows. Cheap today (a cadre removes peers
|
|
666
764
|
* rarely); if removals ever become routine, cache the set and invalidate it on write.
|
|
765
|
+
*
|
|
766
|
+
* `retry: false` is forwarded by {@link queryCadrePeers} for the one caller that reads
|
|
767
|
+
* under the write lock — see that method's note.
|
|
667
768
|
*/
|
|
668
|
-
async queryRevokedStamps(tableName) {
|
|
769
|
+
async queryRevokedStamps(tableName, retry = true) {
|
|
669
770
|
this.ensureInitialized();
|
|
670
771
|
const stamps = new Set();
|
|
671
|
-
for
|
|
772
|
+
for (const row of await this.readRows('select StampId from CadreControl.Revocation where TableName = ?', [tableName], 'revoked-stamps', retry)) {
|
|
672
773
|
stamps.add(row.StampId);
|
|
673
774
|
}
|
|
674
775
|
return stamps;
|
|
@@ -684,7 +785,7 @@ export class ControlDatabase {
|
|
|
684
785
|
async queryRevocations() {
|
|
685
786
|
this.ensureInitialized();
|
|
686
787
|
const rows = [];
|
|
687
|
-
for
|
|
788
|
+
for (const row of await this.readRows('select TableName, RowKey, StampId, ReissuedAt from CadreControl.Revocation', undefined, 'revocations')) {
|
|
688
789
|
rows.push({
|
|
689
790
|
tableName: row.TableName,
|
|
690
791
|
rowKey: row.RowKey,
|
|
@@ -705,7 +806,7 @@ export class ControlDatabase {
|
|
|
705
806
|
* {@link queryRevokedStamps} rather than inlining the SQL, so tests can interpose on
|
|
706
807
|
* that seam (see {@link queryCadrePeers}).
|
|
707
808
|
*
|
|
708
|
-
* Missing/
|
|
809
|
+
* Missing/null column values are coalesced
|
|
709
810
|
* to their empty form (`''` key/sig, `[]` addrs, `0` stamp) so the caller's
|
|
710
811
|
* verify/freshness gates uniformly reject an unpublished or malformed row.
|
|
711
812
|
* The split `addrs` re-join to the exact stored `Multiaddr` (split-on-`,` is
|
|
@@ -715,7 +816,7 @@ export class ControlDatabase {
|
|
|
715
816
|
async queryPeerRecord(peerId) {
|
|
716
817
|
this.ensureInitialized();
|
|
717
818
|
const revoked = await this.queryRevokedStamps('CadrePeer');
|
|
718
|
-
for
|
|
819
|
+
for (const row of await this.readRows('select PeerId, PublicKey, Multiaddr, UpdatedAt, Sig, StampId from CadreControl.CadrePeer where PeerId = ?', [peerId], 'peer-record')) {
|
|
719
820
|
const stampId = row.StampId ?? null;
|
|
720
821
|
if (stampId !== null && revoked.has(stampId)) {
|
|
721
822
|
return null;
|
|
@@ -762,13 +863,13 @@ export class ControlDatabase {
|
|
|
762
863
|
with context OwnerKey = null, Signature = ?
|
|
763
864
|
set Multiaddr = ?, UpdatedAt = ?, Sig = ?
|
|
764
865
|
where PeerId = ?
|
|
765
|
-
`, [record.sig, multiaddr, record.updatedAt, record.sig, record.peerId]);
|
|
866
|
+
`, [record.sig, multiaddr, record.updatedAt, record.sig, record.peerId], 'self-record-update');
|
|
766
867
|
log('Self peer record updated: %s (updatedAt=%d)', record.peerId, record.updatedAt);
|
|
767
868
|
}
|
|
768
869
|
/**
|
|
769
870
|
* Read a single peer's device push token (the full `DeviceToken` row) by PeerId.
|
|
770
871
|
*
|
|
771
|
-
* Returns null when no row exists. Missing/
|
|
872
|
+
* Returns null when no row exists. Missing/null column values are coalesced to
|
|
772
873
|
* their empty form (`''` token/sig, `0` stamp) so the caller's verify/freshness
|
|
773
874
|
* gates uniformly reject an unpublished or malformed row. `platform` is returned
|
|
774
875
|
* verbatim (the resolver validates it against {@link PushPlatform} and re-verifies
|
|
@@ -780,7 +881,7 @@ export class ControlDatabase {
|
|
|
780
881
|
*/
|
|
781
882
|
async queryDeviceToken(peerId) {
|
|
782
883
|
this.ensureInitialized();
|
|
783
|
-
for
|
|
884
|
+
for (const row of await this.readRows('select PeerId, Platform, Token, UpdatedAt, Sig, StampId from CadreControl.DeviceToken where PeerId = ?', [peerId], 'device-token')) {
|
|
784
885
|
return {
|
|
785
886
|
peerId: row.PeerId,
|
|
786
887
|
platform: (row.Platform ?? ''),
|
|
@@ -810,7 +911,7 @@ export class ControlDatabase {
|
|
|
810
911
|
with context OwnerKey = null, Signature = ?
|
|
811
912
|
set Platform = ?, Token = ?, UpdatedAt = ?, Sig = ?
|
|
812
913
|
where PeerId = ?
|
|
813
|
-
`, [record.sig, record.platform, record.token, record.updatedAt, record.sig, record.peerId]);
|
|
914
|
+
`, [record.sig, record.platform, record.token, record.updatedAt, record.sig, record.peerId], 'device-token-update');
|
|
814
915
|
log('Self device token updated: %s (platform=%s, updatedAt=%d)', record.peerId, record.platform, record.updatedAt);
|
|
815
916
|
}
|
|
816
917
|
/**
|
|
@@ -832,7 +933,7 @@ export class ControlDatabase {
|
|
|
832
933
|
insert into CadreControl.OwnerKey (Key, StampId)
|
|
833
934
|
with context OwnerKey = null, Signature = null
|
|
834
935
|
values (?, ?)
|
|
835
|
-
`, [key, stampId]);
|
|
936
|
+
`, [key, stampId], 'owner-key-insert');
|
|
836
937
|
log('Owner key inserted');
|
|
837
938
|
}
|
|
838
939
|
/**
|
|
@@ -865,7 +966,7 @@ export class ControlDatabase {
|
|
|
865
966
|
insert into CadreControl.Strand (Id, Type, MemberPrivateKey, StampId)
|
|
866
967
|
with context OwnerKey = ?, Signature = ?
|
|
867
968
|
values (?, ?, ?, ?)
|
|
868
|
-
`, [ownerKey, signature, strandId, type, memberPrivateKey ?? null, stampId]);
|
|
969
|
+
`, [ownerKey, signature, strandId, type, memberPrivateKey ?? null, stampId], 'strand-insert');
|
|
869
970
|
log('Strand inserted: %s', strandId);
|
|
870
971
|
}
|
|
871
972
|
/**
|
|
@@ -887,7 +988,7 @@ export class ControlDatabase {
|
|
|
887
988
|
* when a row was actually removed.
|
|
888
989
|
*/
|
|
889
990
|
deleteStrand(strandId, ownerKey, signMessage) {
|
|
890
|
-
return this.lockedWithRetry(() => this.deleteGuardedRow('Strand', strandId, ownerKey, signMessage));
|
|
991
|
+
return this.lockedWithRetry(() => this.deleteGuardedRow('Strand', strandId, ownerKey, signMessage), {}, 'strand-delete');
|
|
891
992
|
}
|
|
892
993
|
/**
|
|
893
994
|
* Insert a validation key into the control database using an owner signature.
|
|
@@ -914,7 +1015,7 @@ export class ControlDatabase {
|
|
|
914
1015
|
insert into CadreControl.ValidationKey (Key, StampId)
|
|
915
1016
|
with context OwnerKey = ?, Signature = ?
|
|
916
1017
|
values (?, ?)
|
|
917
|
-
`, [ownerKey, signature, key, stampId]);
|
|
1018
|
+
`, [ownerKey, signature, key, stampId], 'validation-key-insert');
|
|
918
1019
|
log('Validation key inserted: %s', key);
|
|
919
1020
|
}
|
|
920
1021
|
/**
|
|
@@ -934,7 +1035,7 @@ export class ControlDatabase {
|
|
|
934
1035
|
* when a row was actually removed.
|
|
935
1036
|
*/
|
|
936
1037
|
deleteValidationKey(key, ownerKey, signMessage) {
|
|
937
|
-
return this.lockedWithRetry(() => this.deleteGuardedRow('ValidationKey', key, ownerKey, signMessage));
|
|
1038
|
+
return this.lockedWithRetry(() => this.deleteGuardedRow('ValidationKey', key, ownerKey, signMessage), {}, 'validation-key-delete');
|
|
938
1039
|
}
|
|
939
1040
|
/**
|
|
940
1041
|
* Owner-vouched INSERT of one `CadrePeer` membership row, notifying the membership
|
|
@@ -980,7 +1081,8 @@ export class ControlDatabase {
|
|
|
980
1081
|
const stampId = generateStampId(row.peerId);
|
|
981
1082
|
const signature = signMessage(buildAuthorizationMessage('CadreControl.CadrePeer', 'vouch', [row.peerId, stampId]));
|
|
982
1083
|
return await this.mutateCadrePeer('peer-insert', async () => {
|
|
983
|
-
|
|
1084
|
+
// retry: false — this guard runs inside the locked write body (see queryStampId's NOTE).
|
|
1085
|
+
if (await this.queryStampId('CadrePeer', row.peerId, false) !== null) {
|
|
984
1086
|
log('CadrePeer row already present for %s; insert skipped (already a member)', row.peerId);
|
|
985
1087
|
return false;
|
|
986
1088
|
}
|
|
@@ -1102,7 +1204,7 @@ export class ControlDatabase {
|
|
|
1102
1204
|
* when a row was actually removed.
|
|
1103
1205
|
*/
|
|
1104
1206
|
deleteDeviceToken(peerId, ownerKey, signMessage) {
|
|
1105
|
-
return this.lockedWithRetry(() => this.deleteGuardedRow('DeviceToken', peerId, ownerKey, signMessage));
|
|
1207
|
+
return this.lockedWithRetry(() => this.deleteGuardedRow('DeviceToken', peerId, ownerKey, signMessage), {}, 'device-token-delete');
|
|
1106
1208
|
}
|
|
1107
1209
|
/**
|
|
1108
1210
|
* Owner-signed delete of one guarded row plus the `Revocation` tombstone retiring its
|
|
@@ -1139,7 +1241,8 @@ export class ControlDatabase {
|
|
|
1139
1241
|
*/
|
|
1140
1242
|
async deleteGuardedRow(table, keyValue, ownerKey, signMessage) {
|
|
1141
1243
|
this.ensureInitialized();
|
|
1142
|
-
|
|
1244
|
+
// retry: false — this runs inside the locked write body (see queryStampId's NOTE).
|
|
1245
|
+
const stampId = await this.queryStampId(table, keyValue, false);
|
|
1143
1246
|
if (stampId === null) {
|
|
1144
1247
|
log('delete %s: no row for %s (already absent)', table, keyValue);
|
|
1145
1248
|
return false;
|
|
@@ -1259,7 +1362,7 @@ export class ControlDatabase {
|
|
|
1259
1362
|
await this.mutateCadrePeer('peer-reap', () => this.db.exec(sql, [rowKey, stampId]));
|
|
1260
1363
|
}
|
|
1261
1364
|
else {
|
|
1262
|
-
await this.execWrite(sql, [rowKey, stampId]);
|
|
1365
|
+
await this.execWrite(sql, [rowKey, stampId], `reap-${table}`);
|
|
1263
1366
|
}
|
|
1264
1367
|
// A row leaving the control plane without an owner signature at the write site
|
|
1265
1368
|
// should be visible in a log.
|
|
@@ -1411,7 +1514,7 @@ export class ControlDatabase {
|
|
|
1411
1514
|
});
|
|
1412
1515
|
log('Reissued %d revocation tombstone(s) at %d', executed, reissuedAt);
|
|
1413
1516
|
return executed;
|
|
1414
|
-
});
|
|
1517
|
+
}, {}, 'revocation-reissue');
|
|
1415
1518
|
}
|
|
1416
1519
|
/**
|
|
1417
1520
|
* Run `body` between `beginTransaction` and `commit`, rolling back on failure.
|
|
@@ -1514,7 +1617,7 @@ export class ControlDatabase {
|
|
|
1514
1617
|
this.assertCommitBoundary(reason, 'on return from');
|
|
1515
1618
|
await this.notifyMembershipChanged(reason);
|
|
1516
1619
|
return result;
|
|
1517
|
-
});
|
|
1620
|
+
}, {}, reason);
|
|
1518
1621
|
}
|
|
1519
1622
|
/**
|
|
1520
1623
|
* Run one local write under the database-wide write lock, serializing it against
|
|
@@ -1526,7 +1629,8 @@ export class ControlDatabase {
|
|
|
1526
1629
|
* {@link assertCommitBoundary} (any `CadrePeer` path) or silently join each other's
|
|
1527
1630
|
* open transaction — a torn commit if either side rolls back. This lock closes that
|
|
1528
1631
|
* class: every public write method of this class runs its statement(s) under it, and
|
|
1529
|
-
* `SeedBootstrapService`
|
|
1632
|
+
* `SeedBootstrapService` reaches it through {@link execWrite} rather than touching the
|
|
1633
|
+
* `Database` directly, so it is covered too. The race is
|
|
1530
1634
|
* real, not theoretical: the background self-record publish on a control connection
|
|
1531
1635
|
* opening (`CadreNode.drainPendingControlReplication` → `registerSelf`) collided with
|
|
1532
1636
|
* a foreground `authorizePeer` — both `CadrePeer` inserts — and tripped the boundary
|
|
@@ -1569,10 +1673,8 @@ export class ControlDatabase {
|
|
|
1569
1673
|
*
|
|
1570
1674
|
* The retry wraps the LOCK, it does not live inside it: each attempt takes and
|
|
1571
1675
|
* releases the lock, and the backoff sleeps with NO lock held, so a write parked in
|
|
1572
|
-
* backoff never stalls the other local writers queued behind it.
|
|
1573
|
-
*
|
|
1574
|
-
* `USE_NUMBER_ATTEMPTS`. Safe because every locked body is atomic and re-runnable
|
|
1575
|
-
* (the contract on {@link withWriteLock}).
|
|
1676
|
+
* backoff never stalls the other local writers queued behind it. Safe because every
|
|
1677
|
+
* locked body is atomic and re-runnable (the contract on {@link withWriteLock}).
|
|
1576
1678
|
*
|
|
1577
1679
|
* Also carries {@link loadSchema}'s distributed DDL, the one caller that runs BEFORE
|
|
1578
1680
|
* `initialized` is set — so it reaches this method directly rather than through
|
|
@@ -1581,9 +1683,16 @@ export class ControlDatabase {
|
|
|
1581
1683
|
* rather than carving out a lockless path. It is also the ONLY caller that passes a
|
|
1582
1684
|
* `policy` — everything else takes the default, whose classifier and attempt count are
|
|
1583
1685
|
* unchanged by that call site existing.
|
|
1686
|
+
*
|
|
1687
|
+
* `label` names the operation in this loop's debug lines and nothing else
|
|
1688
|
+
* ({@link ControlWriteRetryOptions.label}). Every call site in this class supplies one:
|
|
1689
|
+
* several writes retry CONCURRENTLY in a real party, so an unlabelled line cannot be
|
|
1690
|
+
* attributed to a write — keep new call sites labelled.
|
|
1584
1691
|
*/
|
|
1585
|
-
lockedWithRetry(fn, policy = {}) {
|
|
1586
|
-
|
|
1692
|
+
lockedWithRetry(fn, policy = {}, label) {
|
|
1693
|
+
// The label lands LAST so a call site's own name survives both the policy and the
|
|
1694
|
+
// spec-injected pacing; it changes log attribution only, never behaviour.
|
|
1695
|
+
return retryControlWrite(() => this.withWriteLock(fn), { ...policy, ...this.controlWriteRetryPacing, ...(label !== undefined ? { label } : {}) });
|
|
1587
1696
|
}
|
|
1588
1697
|
/**
|
|
1589
1698
|
* One-statement local write, serialized by {@link withWriteLock} and retried on a
|
|
@@ -1594,10 +1703,13 @@ export class ControlDatabase {
|
|
|
1594
1703
|
* Public so `SeedBootstrapService`'s direct `CadrePeer`/`DeviceToken` SQL writes go
|
|
1595
1704
|
* through the same seam. Must NOT be called from inside an already-locked body (the
|
|
1596
1705
|
* lock is not re-entrant); such a caller uses a bare `getDatabase().exec` instead.
|
|
1706
|
+
*
|
|
1707
|
+
* `label` names the write in the retry loop's debug lines (see {@link lockedWithRetry});
|
|
1708
|
+
* optional only because it is log-only, but every caller in this repo passes one.
|
|
1597
1709
|
*/
|
|
1598
|
-
execWrite(sql, params) {
|
|
1710
|
+
execWrite(sql, params, label) {
|
|
1599
1711
|
this.ensureInitialized();
|
|
1600
|
-
return this.lockedWithRetry(() => this.db.exec(sql, params));
|
|
1712
|
+
return this.lockedWithRetry(() => this.db.exec(sql, params), {}, label);
|
|
1601
1713
|
}
|
|
1602
1714
|
/**
|
|
1603
1715
|
* Throw unless the database sits at a committed boundary — `getAutocommit()` is false
|
|
@@ -1666,7 +1778,8 @@ export class ControlDatabase {
|
|
|
1666
1778
|
* `StrandId` binds the invite to a pre-existing host strand (provision-then-record):
|
|
1667
1779
|
* when set, a responder redeeming this token records a `FormationUsage` against that
|
|
1668
1780
|
* strand and returns it (see {@link ControlFormationUsageRecorder.resolveStrand}); a
|
|
1669
|
-
* null `StrandId` (the default)
|
|
1781
|
+
* null `StrandId` (the default) takes the unbound responder-provisions path: the
|
|
1782
|
+
* responder provisions a fresh open strand and atomically records its one consent row.
|
|
1670
1783
|
* Like ValidationUrl it is a nullable bound field, signed as `''` when absent.
|
|
1671
1784
|
*
|
|
1672
1785
|
* The ExpiresAt and TotalUses message fields must byte-match what the (auto-deferred,
|
|
@@ -1723,7 +1836,7 @@ export class ControlDatabase {
|
|
|
1723
1836
|
options.validationUrl ?? null,
|
|
1724
1837
|
options.strandId ?? null,
|
|
1725
1838
|
stampId,
|
|
1726
|
-
]);
|
|
1839
|
+
], 'formation-invite-insert');
|
|
1727
1840
|
log('Formation invite inserted: %s', token);
|
|
1728
1841
|
}
|
|
1729
1842
|
/**
|
|
@@ -1751,50 +1864,60 @@ export class ControlDatabase {
|
|
|
1751
1864
|
* takes an owner re-seat ({@link insertStrand}) plus a bound invite, never another
|
|
1752
1865
|
* redemption.
|
|
1753
1866
|
*
|
|
1754
|
-
*
|
|
1755
|
-
*
|
|
1756
|
-
*
|
|
1757
|
-
*
|
|
1758
|
-
*
|
|
1759
|
-
*
|
|
1760
|
-
* cannot drift, and no second trip through the approval hook is possible from in here.
|
|
1867
|
+
* The usage row is keyed by the joiner's own nonce (`UsageStampId` is the primary key), so
|
|
1868
|
+
* concurrent redemptions of one token never contend for a shared row key — there is no
|
|
1869
|
+
* lost race to retry and no second trip through the approval hook. The invite's seat
|
|
1870
|
+
* budget is checked by COUNT ({@link assertSeatRemains}) inside the write lock, ahead of
|
|
1871
|
+
* the write, so a spent invite is refused by name (`InvitationExhaustedError`) instead of
|
|
1872
|
+
* as the schema cap clause's generic `Authorized` CHECK failure.
|
|
1761
1873
|
*/
|
|
1762
1874
|
async redeemInvitation(params) {
|
|
1763
1875
|
this.ensureInitialized();
|
|
1764
1876
|
const { token, strandId, disclosure = '', peerKey, peerSignature, usageStampId, nowMs, validationKey, validationSignature, signal, totalUses, } = params;
|
|
1765
1877
|
log('Redeeming invitation %s -> strand %s', token, strandId);
|
|
1766
1878
|
const localPeerId = this.config.libp2pNode.peerId.toString();
|
|
1767
|
-
//
|
|
1768
|
-
//
|
|
1769
|
-
// approver's signed digest. Re-minting per attempt would only churn.
|
|
1879
|
+
// A failed attempt rolls its whole transaction back, so nothing was ever persisted under
|
|
1880
|
+
// this stamp, and it is deliberately not part of the approver's signed digest.
|
|
1770
1881
|
const strandStampId = generateStampId(localPeerId);
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1882
|
+
// Ordering inside the locked body is contractual: the abort check, then the seat check,
|
|
1883
|
+
// then the write — FormationAbortedError may only ever be thrown before the write is
|
|
1884
|
+
// issued, and a transient-failure retry (lockedWithRetry re-runs the whole body) must
|
|
1885
|
+
// re-check both before re-presenting the write.
|
|
1886
|
+
await this.lockedWithRetry(async () => {
|
|
1887
|
+
// Inside the lock: a write still parked behind another writer when the caller's
|
|
1888
|
+
// budget expired is abandoned rather than executed.
|
|
1889
|
+
if (signal?.aborted) {
|
|
1890
|
+
throw new FormationAbortedError(token, 'redemption');
|
|
1891
|
+
}
|
|
1892
|
+
await this.assertSeatRemains(token, totalUses);
|
|
1893
|
+
await this.inTransaction('redemption', async () => {
|
|
1894
|
+
// 1. Strand row — authorised by the FormationUsage branch (no owner sig),
|
|
1895
|
+
// still carrying a fresh unique StampId for the anti-replay column.
|
|
1896
|
+
// Hard-coded open + keyless: the consent branch admits no other shape.
|
|
1897
|
+
await this.db.exec(`
|
|
1776
1898
|
insert into CadreControl.Strand (Id, Type, MemberPrivateKey, StampId)
|
|
1777
1899
|
with context OwnerKey = null, Signature = null
|
|
1778
1900
|
values (?, 'o', null, ?)
|
|
1779
1901
|
`, [strandId, strandStampId]);
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1902
|
+
// 2. FormationUsage row — authorised by the matching FormationInvite, and
|
|
1903
|
+
// carrying the strand's stamp so it authorizes THIS row and no other.
|
|
1904
|
+
await this.execFormationUsageInsert({
|
|
1905
|
+
token, usageStampId, disclosure, strandId, strandStampId,
|
|
1906
|
+
peerKey, peerSignature, nowMs: nowMs ?? Date.now(),
|
|
1907
|
+
validationKey: validationKey ?? null, validationSignature: validationSignature ?? null,
|
|
1908
|
+
});
|
|
1786
1909
|
});
|
|
1787
|
-
})
|
|
1788
|
-
log('Redeemed invitation %s -> strand %s (
|
|
1789
|
-
return {
|
|
1910
|
+
}, {}, 'formation-usage');
|
|
1911
|
+
log('Redeemed invitation %s -> strand %s (stamp %s)', token, strandId, usageStampId);
|
|
1912
|
+
return { usageStampId };
|
|
1790
1913
|
}
|
|
1791
1914
|
/**
|
|
1792
1915
|
* Record a `FormationUsage` against an **already-existing** `Strand` (no strand
|
|
1793
1916
|
* insert). This is the redemption path when the strand was provisioned
|
|
1794
1917
|
* separately (e.g. owner-signed) and the consent record is added after the
|
|
1795
1918
|
* fact: the single insert auto-commits, and the deferred `StrandExists` CHECK
|
|
1796
|
-
* is satisfied by the pre-existing committed strand row.
|
|
1797
|
-
* `
|
|
1919
|
+
* is satisfied by the pre-existing committed strand row. Echoes back the
|
|
1920
|
+
* redemption's `usageStampId` (the row's primary key).
|
|
1798
1921
|
*
|
|
1799
1922
|
* Use {@link redeemInvitation} instead when the strand must be created by
|
|
1800
1923
|
* consent atomically with the usage.
|
|
@@ -1808,108 +1931,47 @@ export class ControlDatabase {
|
|
|
1808
1931
|
* by {@link ControlFormationUsageRecorder.resolveStrand}, so an absent row at this
|
|
1809
1932
|
* point is a genuine race and deserves a named error, not a silent rollback.
|
|
1810
1933
|
*
|
|
1811
|
-
*
|
|
1812
|
-
*
|
|
1813
|
-
*
|
|
1814
|
-
*
|
|
1815
|
-
*
|
|
1816
|
-
*
|
|
1934
|
+
* The row is keyed by the joiner's own nonce (`UsageStampId` is the primary key), so this
|
|
1935
|
+
* path — the one production actually races on (a bound invite published by `cadre-web` /
|
|
1936
|
+
* `cadre-phone` is always record-only) — never contends with another redemption for a
|
|
1937
|
+
* shared row key: there is no lost race to retry and no second trip through the approval
|
|
1938
|
+
* hook. The invite's seat budget is checked by COUNT ({@link assertSeatRemains}) inside
|
|
1939
|
+
* the write lock, ahead of the write.
|
|
1817
1940
|
*/
|
|
1818
1941
|
async recordFormationUsage(params) {
|
|
1819
1942
|
this.ensureInitialized();
|
|
1820
1943
|
const { token, strandId, disclosure = '', peerKey, peerSignature, usageStampId, nowMs, validationKey, validationSignature, signal, totalUses, } = params;
|
|
1821
|
-
//
|
|
1822
|
-
//
|
|
1944
|
+
// The host strand is pre-existing and owner-signed; its stamp names the strand ROW the
|
|
1945
|
+
// consent record authorizes.
|
|
1823
1946
|
const strandStampId = await this.queryStrandStampId(strandId);
|
|
1824
1947
|
if (strandStampId === null) {
|
|
1825
1948
|
throw new MissingHostStrandError(strandId, token);
|
|
1826
1949
|
}
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
log('Recorded formation usage: token=%s strand=%s (use #%d)', token, strandId, useNumber);
|
|
1833
|
-
return { useNumber, usageStampId };
|
|
1834
|
-
}
|
|
1835
|
-
/**
|
|
1836
|
-
* Assign a `UseNumber` and run `write` under it, re-picking and retrying when another
|
|
1837
|
-
* writer took that number first.
|
|
1838
|
-
*
|
|
1839
|
-
* Each attempt takes {@link withWriteLock} and, inside it, reads {@link nextUseNumber}
|
|
1840
|
-
* afresh. Reading under the lock is the whole fix for the common case: two redemptions of
|
|
1841
|
-
* one token on the SAME node are serialized by the local write queue, so each computes a
|
|
1842
|
-
* number the other has already committed and neither collides. The retry covers the writers
|
|
1843
|
-
* this lock does not reach — another node of the cadre, another `Database` handle over the
|
|
1844
|
-
* same store — which can still take the number between our read and our commit.
|
|
1845
|
-
*
|
|
1846
|
-
* `write` receives ONLY the use number; every other parameter of the redemption (the nonce,
|
|
1847
|
-
* the approver's signature, the strand, the joining peer key, the disclosure) is closed over
|
|
1848
|
-
* by the caller and passed through untouched. So by construction a retry re-presents the
|
|
1849
|
-
* IDENTICAL approval and the four non-nonce fields inside the approver's signed digest
|
|
1850
|
-
* cannot drift. No second call to the approval hook is reachable from in here either:
|
|
1851
|
-
* `obtainApproval` lives a layer up in `ControlFormationUsageRecorder` and is never
|
|
1852
|
-
* re-entered.
|
|
1853
|
-
*
|
|
1854
|
-
* `write` must be a BARE private body ({@link inTransaction} /
|
|
1855
|
-
* {@link execFormationUsageInsert}) — {@link withWriteLock} is not re-entrant and a `write`
|
|
1856
|
-
* that called another locked public method would hang silently and strand the write queue.
|
|
1857
|
-
*
|
|
1858
|
-
* A failed attempt has written nothing (a bare insert never landed; a transactional one
|
|
1859
|
-
* rolled back), so the abort check at the top of each attempt stays inside
|
|
1860
|
-
* {@link FormationAbortedError}'s contract — it is only ever thrown before this attempt's
|
|
1861
|
-
* write is issued.
|
|
1862
|
-
*
|
|
1863
|
-
* @param operation - `'redemption'` | `'usage recording'`, for the abort error's text.
|
|
1864
|
-
* @param knownTotalUses - The invite's seat budget when the caller already has it in hand
|
|
1865
|
-
* (`undefined` to fall back to a fresh {@link queryFormationInvite} read; `null` for "no
|
|
1866
|
-
* invite row" / unlimited). Passed through to {@link assertSeatRemains} on every attempt.
|
|
1867
|
-
* @returns The `UseNumber` the successful attempt wrote under.
|
|
1868
|
-
*/
|
|
1869
|
-
async withUseNumberRetry(token, operation, signal, knownTotalUses, write) {
|
|
1870
|
-
let lastError;
|
|
1871
|
-
for (let attempt = 1; attempt <= USE_NUMBER_ATTEMPTS; attempt++) {
|
|
1872
|
-
try {
|
|
1873
|
-
// lockedWithRetry carries the transient-cluster retry INSIDE this use-number
|
|
1874
|
-
// retry. The loops cannot multiply: a lost use number is a constraint failure,
|
|
1875
|
-
// which `isRetriableControlWriteFailure` never matches, and a transient cluster
|
|
1876
|
-
// failure is not a lost use number, so exactly one loop claims any given error.
|
|
1877
|
-
return await this.lockedWithRetry(async () => {
|
|
1878
|
-
// Inside the lock: a write still parked behind another writer when the caller's
|
|
1879
|
-
// budget expired is abandoned rather than executed.
|
|
1880
|
-
if (signal?.aborted) {
|
|
1881
|
-
throw new FormationAbortedError(token, operation);
|
|
1882
|
-
}
|
|
1883
|
-
const useNumber = await this.nextUseNumber(token);
|
|
1884
|
-
await this.assertSeatRemains(token, useNumber, knownTotalUses);
|
|
1885
|
-
await write(useNumber);
|
|
1886
|
-
return useNumber;
|
|
1887
|
-
});
|
|
1950
|
+
// Ordering inside the locked body is contractual — abort check, seat check, write — see
|
|
1951
|
+
// the matching note in redeemInvitation.
|
|
1952
|
+
await this.lockedWithRetry(async () => {
|
|
1953
|
+
if (signal?.aborted) {
|
|
1954
|
+
throw new FormationAbortedError(token, 'usage recording');
|
|
1888
1955
|
}
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
// today's behaviour: `StrandFormationManager.provisionAsResponder`'s catch-all maps it to
|
|
1899
|
-
// `Formation conflict, retry` and the joiner starts over with a fresh nonce.
|
|
1900
|
-
throw lastError;
|
|
1956
|
+
await this.assertSeatRemains(token, totalUses);
|
|
1957
|
+
await this.execFormationUsageInsert({
|
|
1958
|
+
token, usageStampId, disclosure, strandId, strandStampId,
|
|
1959
|
+
peerKey, peerSignature, nowMs: nowMs ?? Date.now(),
|
|
1960
|
+
validationKey: validationKey ?? null, validationSignature: validationSignature ?? null,
|
|
1961
|
+
});
|
|
1962
|
+
}, {}, 'formation-usage');
|
|
1963
|
+
log('Recorded formation usage: token=%s strand=%s (stamp %s)', token, strandId, usageStampId);
|
|
1964
|
+
return { usageStampId };
|
|
1901
1965
|
}
|
|
1902
1966
|
/**
|
|
1903
|
-
* Refuse
|
|
1967
|
+
* Refuse a redemption that would consume a seat the invite does not have.
|
|
1904
1968
|
*
|
|
1905
|
-
* `FormationUsage.Authorized`
|
|
1906
|
-
*
|
|
1907
|
-
*
|
|
1908
|
-
*
|
|
1909
|
-
*
|
|
1910
|
-
*
|
|
1911
|
-
* local write queue, so the loser's first attempt is already over budget, with no retry
|
|
1912
|
-
* involved.
|
|
1969
|
+
* The schema's count-based cap clause (`FormationUsage.Authorized`) refuses an over-limit
|
|
1970
|
+
* write at the database anyway — but it fails as a generic `CHECK constraint failed:
|
|
1971
|
+
* Authorized`, which the manager reports as a retryable conflict. Catching it here as
|
|
1972
|
+
* {@link InvitationExhaustedError} lets the joiner be told the invitation is spent instead
|
|
1973
|
+
* of being sent around a loop that can never close. Runs inside the write lock, so on a
|
|
1974
|
+
* same-node race the loser reads the winner's committed row and is refused by name.
|
|
1913
1975
|
*
|
|
1914
1976
|
* `knownTotalUses` lets a caller that already read the `FormationInvite` row (both
|
|
1915
1977
|
* production callers in `ControlFormationUsageRecorder` do) skip a second read on the common,
|
|
@@ -1919,9 +1981,9 @@ export class ControlDatabase {
|
|
|
1919
1981
|
* invite, before an outbound approval call). Safe today because `FormationInvite` is
|
|
1920
1982
|
* insert/delete only (its `Immutable` constraint), so the only way to stale it is an owner
|
|
1921
1983
|
* revoking the token and re-issuing it with MORE seats mid-redemption; if invites ever gain
|
|
1922
|
-
* an update path, drop the parameter and read here
|
|
1984
|
+
* an update path, drop the parameter and read here instead.
|
|
1923
1985
|
*/
|
|
1924
|
-
async assertSeatRemains(token,
|
|
1986
|
+
async assertSeatRemains(token, knownTotalUses) {
|
|
1925
1987
|
// A missing invite is left to `Authorized` — it is not an exhaustion, and the CHECK's
|
|
1926
1988
|
// rejection is already the right, non-retryable answer for it. That fallback is also what
|
|
1927
1989
|
// makes this read's failure mode safe: it is a full-primary-key point lookup
|
|
@@ -1929,11 +1991,19 @@ export class ControlDatabase {
|
|
|
1929
1991
|
// is still open (tracked by `debt-composite-pk-point-lookup-unreliable-untracked`). A
|
|
1930
1992
|
// spurious empty result only costs the attempt its NAMED exhaustion error, reverting it to
|
|
1931
1993
|
// today's generic `Authorized` refusal — never a seat the invite does not have.
|
|
1994
|
+
// Both reads pass retry: false — assertSeatRemains only ever runs inside the locked
|
|
1995
|
+
// write bodies of redeemInvitation / recordFormationUsage, where a read retrying its
|
|
1996
|
+
// own backoff would sleep holding the write lock and the write funnel re-runs these
|
|
1997
|
+
// reads anyway (see the NOTE on readRows).
|
|
1932
1998
|
const totalUses = knownTotalUses !== undefined
|
|
1933
1999
|
? knownTotalUses
|
|
1934
|
-
: (await this.queryFormationInvite(token))?.totalUses ?? null;
|
|
1935
|
-
if (totalUses
|
|
1936
|
-
|
|
2000
|
+
: (await this.queryFormationInvite(token, false))?.totalUses ?? null;
|
|
2001
|
+
if (totalUses == null) {
|
|
2002
|
+
return;
|
|
2003
|
+
}
|
|
2004
|
+
const used = await this.countFormationUsage(token, false);
|
|
2005
|
+
if (used >= totalUses) {
|
|
2006
|
+
throw new InvitationExhaustedError(token, used, totalUses);
|
|
1937
2007
|
}
|
|
1938
2008
|
}
|
|
1939
2009
|
/** Parameterised `FormationUsage` insert shared by redeem + record paths. */
|
|
@@ -1948,12 +2018,12 @@ export class ControlDatabase {
|
|
|
1948
2018
|
// not a fix for an observable mis-ordering.
|
|
1949
2019
|
const nowCanonical = await canonicalDatetime(this.db, opts.nowMs);
|
|
1950
2020
|
await this.db.exec(`
|
|
1951
|
-
insert into CadreControl.FormationUsage (Token,
|
|
2021
|
+
insert into CadreControl.FormationUsage (Token, UsageStampId, PeerKey, PeerSig, Disclosure, StrandId, StrandStampId)
|
|
1952
2022
|
with context Now = ?, ValidationKey = ?, ValidationSignature = ?
|
|
1953
|
-
values (?, ?, ?, ?, ?, ?,
|
|
2023
|
+
values (?, ?, ?, ?, ?, ?, ?)
|
|
1954
2024
|
`, [
|
|
1955
2025
|
nowCanonical, opts.validationKey, opts.validationSignature,
|
|
1956
|
-
opts.token, opts.
|
|
2026
|
+
opts.token, opts.usageStampId, opts.peerKey,
|
|
1957
2027
|
opts.peerSignature, opts.disclosure, opts.strandId, opts.strandStampId,
|
|
1958
2028
|
]);
|
|
1959
2029
|
}
|
|
@@ -1961,10 +2031,15 @@ export class ControlDatabase {
|
|
|
1961
2031
|
* Read a `FormationInvite` row by token, or null when absent. `expiresAtMs` is
|
|
1962
2032
|
* the parsed epoch-ms of the stored `datetime` (null when the invite never
|
|
1963
2033
|
* expires); the caller compares it against the wall clock for freshness.
|
|
2034
|
+
*
|
|
2035
|
+
* `retry: false` is passed only by {@link assertSeatRemains}, which runs INSIDE a
|
|
2036
|
+
* locked write body — same per-call opt-out, and for the same reason, as
|
|
2037
|
+
* {@link queryStampId}'s. Every unlocked caller keeps the default retried path.
|
|
1964
2038
|
*/
|
|
1965
|
-
async queryFormationInvite(token) {
|
|
2039
|
+
async queryFormationInvite(token, retry = true) {
|
|
1966
2040
|
this.ensureInitialized();
|
|
1967
|
-
|
|
2041
|
+
const sql = 'select Token, sAppId, ExpiresAt, TotalUses, ValidationUrl, StrandId from CadreControl.FormationInvite where Token = ?';
|
|
2042
|
+
for (const row of await this.readRows(sql, [token], 'formation-invite', retry)) {
|
|
1968
2043
|
return {
|
|
1969
2044
|
token: row.Token,
|
|
1970
2045
|
sAppId: row.sAppId,
|
|
@@ -1978,10 +2053,32 @@ export class ControlDatabase {
|
|
|
1978
2053
|
}
|
|
1979
2054
|
/**
|
|
1980
2055
|
* Count `FormationUsage` rows recorded against a token (uses consumed so far).
|
|
2056
|
+
*
|
|
2057
|
+
* This count IS the seat cap (`enforceFormationUseCap`), so an under-report admits a seat
|
|
2058
|
+
* the invitation never paid for. It is served by a table SCAN, deliberately: a bare
|
|
2059
|
+
* equality on a non-key column is a shape the optimystic vtab declines to claim, so the
|
|
2060
|
+
* engine applies the filter itself over a full read of the table.
|
|
2061
|
+
*
|
|
2062
|
+
* NOTE: it was briefly an index seek, and that is what the scan is protecting against.
|
|
2063
|
+
* `formation-unique-token-redesign` took `Token` out of the primary key and declared
|
|
2064
|
+
* `FormationUsageByToken` purely to spare this scan, recording as a tripwire that an
|
|
2065
|
+
* under-reporting descent would admit an unpaid-for seat — "a failure mode a scan could
|
|
2066
|
+
* not produce". The tripwire fired. Across machines a secondary-index descent returns only
|
|
2067
|
+
* the rows the READING machine wrote, so this count under-reported without bound on a
|
|
2068
|
+
* multi-machine party and the cap over-admitted. The index was removed 2026-08-25; the
|
|
2069
|
+
* measurement is recorded in the block comment at the `FormationUsage` declaration in
|
|
2070
|
+
* `schemas/control.qsql`, and the engine defect in
|
|
2071
|
+
* `blocked/secondary-index-seek-blind-to-sibling-rows`. Do NOT re-declare the index to
|
|
2072
|
+
* speed this up while that ticket is open.
|
|
2073
|
+
*
|
|
2074
|
+
* `retry: false` is passed only by {@link assertSeatRemains}, which runs INSIDE a
|
|
2075
|
+
* locked write body — same per-call opt-out, and for the same reason, as
|
|
2076
|
+
* {@link queryStampId}'s. Every unlocked caller keeps the default retried path.
|
|
1981
2077
|
*/
|
|
1982
|
-
async countFormationUsage(token) {
|
|
2078
|
+
async countFormationUsage(token, retry = true) {
|
|
1983
2079
|
this.ensureInitialized();
|
|
1984
|
-
|
|
2080
|
+
const sql = 'select count(1) as Count from CadreControl.FormationUsage where Token = ?';
|
|
2081
|
+
for (const row of await this.readRows(sql, [token], 'formation-usage-count', retry)) {
|
|
1985
2082
|
return row.Count ?? 0;
|
|
1986
2083
|
}
|
|
1987
2084
|
return 0;
|
|
@@ -2012,7 +2109,7 @@ export class ControlDatabase {
|
|
|
2012
2109
|
// prune redeemed/expired rows.
|
|
2013
2110
|
const metered = [];
|
|
2014
2111
|
let unlimitedOutstanding = false;
|
|
2015
|
-
for
|
|
2112
|
+
for (const row of await this.readRows('select Token, ExpiresAt, TotalUses from CadreControl.FormationInvite', undefined, 'outstanding-invites')) {
|
|
2016
2113
|
const expiresAtMs = parseNullableStoredDatetimeMs(row.ExpiresAt);
|
|
2017
2114
|
if (expiresAtMs !== null && expiresAtMs <= nowMs) {
|
|
2018
2115
|
continue;
|
|
@@ -2028,6 +2125,12 @@ export class ControlDatabase {
|
|
|
2028
2125
|
if (unlimitedOutstanding) {
|
|
2029
2126
|
return true;
|
|
2030
2127
|
}
|
|
2128
|
+
// NOTE: this loop issues one retried read per metered invite, each carrying its own
|
|
2129
|
+
// CONTROL_READ_RETRY_BUDGET_MS, on a path the inbound admission gate awaits under a
|
|
2130
|
+
// 2 s FAIL-OPEN deadline — so with N unexpired metered invites the worst case is
|
|
2131
|
+
// (1 + N) budgets, not one. Fine while a cadre holds a couple of live invites and
|
|
2132
|
+
// transient read failures are rare; if a party ever keeps many metered invites open,
|
|
2133
|
+
// give this whole method one shared deadline rather than raising the per-read budget.
|
|
2031
2134
|
for (const invite of metered) {
|
|
2032
2135
|
if (await this.countFormationUsage(invite.token) < invite.totalUses) {
|
|
2033
2136
|
return true;
|
|
@@ -2035,19 +2138,6 @@ export class ControlDatabase {
|
|
|
2035
2138
|
}
|
|
2036
2139
|
return false;
|
|
2037
2140
|
}
|
|
2038
|
-
/**
|
|
2039
|
-
* Next `UseNumber` for a token = max(existing)+1, per the `Monotonic` constraint.
|
|
2040
|
-
*
|
|
2041
|
-
* Called only from inside {@link withUseNumberRetry}'s locked body — a read takes no
|
|
2042
|
-
* transaction of its own, so it is safe there, and reading it under the lock is what stops
|
|
2043
|
-
* two local redemptions of one token from picking the same number.
|
|
2044
|
-
*/
|
|
2045
|
-
async nextUseNumber(token) {
|
|
2046
|
-
for await (const row of this.db.eval('select coalesce(max(UseNumber), 0) as MaxUse from CadreControl.FormationUsage where Token = ?', [token])) {
|
|
2047
|
-
return row.MaxUse + 1;
|
|
2048
|
-
}
|
|
2049
|
-
return 1;
|
|
2050
|
-
}
|
|
2051
2141
|
/**
|
|
2052
2142
|
* Close the database and cleanup resources.
|
|
2053
2143
|
*
|