@serfab/cadre-core 0.11.0 → 0.12.0

package/dist/control-database.d.ts

@@ -227,12 +227,111 @@ export declare class ControlDatabase {
      * having to restate the policy's attempts or classifier.
      */
     private controlWriteRetryPacing;
+    /**
+     * Pacing seams for {@link readRows}'s retry — the read twin of
+     * {@link controlWriteRetryPacing}, with the same contract: production leaves this
+     * empty, specs inject a recorded `sleep` / fake `now` so no test waits out a real
+     * backoff. Applied OVER the read policy's defaults but UNDER the call site's `label`.
+     */
+    private controlReadRetryPacing;
     constructor(config: ControlDatabaseConfig);
     /**
      * Initialize the database - load schema and register plugins
      */
     initialize(): Promise<void>;
     private loadSchema;
+    /**
+     * One UNRETRIED drain of a control read to rows — every read in this class bottoms out
+     * here (almost all via {@link readRows}, which adds the transient-failure retry), and
+     * this is the ONE place the committed-read opt-in is spelled.
+     *
+     * Reached only through {@link readRows}, which drops straight here for the call sites
+     * that pass `retry: false` — reads issued from INSIDE a locked write body, where a
+     * backoff sleep would hold the write lock and the write funnel re-runs the read anyway
+     * (see the NOTE on {@link readRows}).
+     *
+     * A control read normally queues on the database's exec mutex, so it does not answer
+     * until whatever statement holds that mutex finishes. When a control WRITE is parked
+     * against an unresponsive cohort member that is minutes, and a read that only wants
+     * the last committed state waits the whole time for no reason
+     * (`complete/control-reads-blocked-by-stalled-write`).
+     *
+     * Quereus's remedy is `readConcurrency: 'committed'` — a PER-CALL opt-in
+     * (`Statement.tryRouteConcurrent` refuses anything that did not ask for it by name)
+     * that runs an eligible read off the mutex against each table's last committed state.
+     * Routing is best-effort and never an error: an ineligible statement falls back to
+     * the serialized path silently — any side-effecting node, any table-valued function
+     * call (pure ones included), any table whose module does not declare
+     * `readCommittedSnapshot`, or an EXPLICIT transaction being open on this database.
+     *
+     * **It is asked for only while a write is in flight, and that condition is the whole
+     * point.** The opt-in does not merely move the read off the mutex: it connects the
+     * table with `_readCommitted`, and the optimystic vtab serves such a connection from
+     * a pinned pre-transaction snapshot that NEVER refreshes from the network, where an
+     * ordinary read calls `update()` on the collection first. Taking that path
+     * unconditionally therefore trades "reads block behind a stalled write" for "reads
+     * stop seeing what other machines wrote" — measured, not theorised: with every read
+     * opted in, `control-write-degraded-cohort-member` could no longer observe a sibling
+     * publishing its own `CadrePeer` row and failed at suite setup on two runs out of two.
+     *
+     * `getAutocommit()` reports the whole `Database` rather than this call (the same
+     * property {@link assertCommitBoundary} leans on), which is exactly what is wanted:
+     * false means some writer's transaction is open, which is the only situation where a
+     * read would have blocked. The check races that writer harmlessly in both directions
+     * — losing it either reads normally (today's behaviour) or reads committed state.
+     *
+     * What a committed read gives up is observing a write that has not finished
+     * committing, and that is safe for every caller here: an AWAITED write is committed by
+     * the time it returns, and a read inside {@link inTransaction} is disqualified by the
+     * open explicit transaction and stays serialized, so it still sees its own
+     * transaction's rows. A read after an UNAWAITED write on this database would be wrong
+     * — no such caller exists, and {@link withWriteLock} is what keeps it that way.
+     */
+    private readRowsOnce;
+    /**
+     * {@link readRowsOnce} plus the bounded transient-failure retry
+     * ({@link retryControlRead}) — the funnel every UNLOCKED control read goes through, so a
+     * read that failed because the cluster could not be asked properly (a stream reset
+     * mid-scan, a partially unreachable cohort) is re-presented a moment later instead of
+     * surfacing to every reader.
+     *
+     * MATERIALIZES the read rather than returning a lazy iterator, and that shape is what
+     * makes the retry possible at all: the failure happens during ITERATION, not at call
+     * time, and a half-consumed iterator cannot be retried without re-yielding rows the
+     * caller already saw. Materializing is safe here — every caller either drains to an
+     * array anyway or takes the first row of a statement that yields at most one (a
+     * primary-key lookup or a `count(1)`); nothing streams.
+     *
+     * Each ATTEMPT is a fresh {@link readRowsOnce} call, so the committed-read opt-in
+     * (`getAutocommit()`) is re-evaluated per attempt — a write can finish between attempts
+     * and change the right answer.
+     *
+     * The retry's budget ({@link CONTROL_READ_RETRY_BUDGET_MS}, 1.5 s) is deliberately far
+     * under the write funnel's: the tightest caller deadline over a control read is the
+     * inbound admission gate's 2 s FAIL-OPEN timeout, and a read that outlives it spends its
+     * retries after the gate has already admitted (see `control-read-retry.ts`).
+     *
+     * `label` names the read in the retry loop's `Control read [<label>] …` debug lines and
+     * nothing else; several reads are in flight concurrently in a real party, so an
+     * unlabelled line cannot be attributed — keep new call sites labelled.
+     *
+     * `retry: false` drops straight to {@link readRowsOnce}: reads issued from INSIDE a
+     * locked write body must not retry on their own. `retryControlWrite`'s contract is that
+     * backoff sleeps happen with NO lock held; a read retrying its own backoff inside a
+     * locked body would sleep holding the write lock (stalling every other local writer),
+     * and the write funnel already re-runs the body's reads when it re-runs the body. See
+     * {@link queryStampId} / {@link assertSeatRemains} / {@link queryCadrePeers} for the
+     * per-call opt-outs.
+     *
+     * NOTE: nothing structural enforces that opt-out — it is per-call because an unlocked
+     * read runs CONCURRENTLY with a locked body, so `this`-state cannot tell the two apart,
+     * and there is no async-context primitive available on every target platform (browser,
+     * React Native) to carry the answer. Reads reached through a CALLBACK invoked under the
+     * lock are the easy miss: `notifyMembershipChanged` runs its listener with the lock
+     * held, and its membership read is opted out at {@link queryCadrePeers}. If another
+     * under-lock callback seam is ever added, audit its read graph the same way.
+     */
+    private readRows;
     /**
      * Query all strands from the control database
      */
@@ -304,8 +403,17 @@ export declare class ControlDatabase {
      * carried into both, or those suites keep passing against a contract the database no
      * longer honours. Fine while the filter is one `Set.has` on `StampId`; if it grows a
      * second dimension, give the fakes a shared fixture or a contract test instead.
+     *
+     * `retry: false` is passed by exactly one caller —
+     * `CadreNode.refreshAuthorizedControlPeers`, the membership-gate snapshot refresh —
+     * because {@link notifyMembershipChanged} invokes that listener with the write lock
+     * HELD, and a retrying read there would sleep its backoff holding the lock (see the
+     * NOTE on {@link readRows}). It costs nothing: that refresh is best-effort, keeps the
+     * previous snapshot on failure, and is re-driven by the next write and by the timed
+     * reconcile. BOTH of this method's reads take the flag — retrying either one under the
+     * lock is the same defect.
      */
-    queryCadrePeers(): Promise<CadrePeerRow[]>;
+    queryCadrePeers(retry?: boolean): Promise<CadrePeerRow[]>;
     /**
      * Every approver public key currently enrolled in `CadreControl.ValidationKey` — the
      * set a `ValidationUrl` redemption's approval signature must be signed by (see
@@ -330,6 +438,16 @@ export declare class ControlDatabase {
      * `table` and its {@link GUARDED_KEY_COLUMN} column are interpolated into the SQL; both
      * come from closed literal unions — no caller-supplied string can reach the statement
      * (same injection-surface discipline as {@link countRows}'s `CONTROL_TABLE_SET` guard).
+     *
+     * NOTE: `retry: false` is passed by exactly the call sites that run INSIDE a locked
+     * write body — {@link deleteGuardedRow} and {@link insertCadrePeer}'s insert-if-absent
+     * guard. A read retrying there would sleep its backoff HOLDING the write lock
+     * (`retryControlWrite`'s contract is that backoff sleeps happen with no lock held), and
+     * the write funnel already re-runs the body's reads when it re-runs the body. The
+     * unlocked callers — the public stamp readers, {@link reauthorizeCadrePeer} and
+     * {@link reapRevokedRow}, whose stamp reads run BEFORE their locks are taken — keep the
+     * default retried path. Explicit per call rather than an ambient flag: unlocked reads
+     * run concurrently with a locked body, so `this`-state cannot tell the two apart.
      */
     private queryStampId;
     /**
@@ -364,8 +482,11 @@ export declare class ControlDatabase {
      * NOTE: re-reads the whole retired set on every call, and the caller runs per inbound
      * gate request while the table only ever grows. Cheap today (a cadre removes peers
      * rarely); if removals ever become routine, cache the set and invalidate it on write.
+     *
+     * `retry: false` is forwarded by {@link queryCadrePeers} for the one caller that reads
+     * under the write lock — see that method's note.
      */
-    queryRevokedStamps(tableName: RevocableTable): Promise<Set<string>>;
+    queryRevokedStamps(tableName: RevocableTable, retry?: boolean): Promise<Set<string>>;
     /**
      * Every locally-held `CadreControl.Revocation` tombstone — identity triple plus its
      * `ReissuedAt` counter. Consumed by the cohort-growth re-issue sweep, which
@@ -386,7 +507,7 @@ export declare class ControlDatabase {
      * {@link queryRevokedStamps} rather than inlining the SQL, so tests can interpose on
      * that seam (see {@link queryCadrePeers}).
      *
-     * Missing/legacy column values are coalesced
+     * Missing/null column values are coalesced
      * to their empty form (`''` key/sig, `[]` addrs, `0` stamp) so the caller's
      * verify/freshness gates uniformly reject an unpublished or malformed row.
      * The split `addrs` re-join to the exact stored `Multiaddr` (split-on-`,` is
@@ -421,7 +542,7 @@ export declare class ControlDatabase {
     /**
      * Read a single peer's device push token (the full `DeviceToken` row) by PeerId.
      *
-     * Returns null when no row exists. Missing/legacy column values are coalesced to
+     * Returns null when no row exists. Missing/null column values are coalesced to
      * their empty form (`''` token/sig, `0` stamp) so the caller's verify/freshness
      * gates uniformly reject an unpublished or malformed row. `platform` is returned
      * verbatim (the resolver validates it against {@link PushPlatform} and re-verifies
@@ -890,7 +1011,8 @@ export declare class ControlDatabase {
      * {@link assertCommitBoundary} (any `CadrePeer` path) or silently join each other's
      * open transaction — a torn commit if either side rolls back. This lock closes that
      * class: every public write method of this class runs its statement(s) under it, and
-     * `SeedBootstrapService` wraps its direct `db.exec` writes in it too. The race is
+     * `SeedBootstrapService` reaches it through {@link execWrite} rather than touching the
+     * `Database` directly, so it is covered too. The race is
      * real, not theoretical: the background self-record publish on a control connection
      * opening (`CadreNode.drainPendingControlReplication` → `registerSelf`) collided with
      * a foreground `authorizePeer` — both `CadrePeer` inserts — and tripped the boundary
@@ -1008,7 +1130,8 @@ export declare class ControlDatabase {
      * `StrandId` binds the invite to a pre-existing host strand (provision-then-record):
      * when set, a responder redeeming this token records a `FormationUsage` against that
      * strand and returns it (see {@link ControlFormationUsageRecorder.resolveStrand}); a
-     * null `StrandId` (the default) leaves the legacy responder-provisions path in place.
+     * null `StrandId` (the default) takes the unbound responder-provisions path: the
+     * responder provisions a fresh open strand and atomically records its one consent row.
      * Like ValidationUrl it is a nullable bound field, signed as `''` when absent.
      *
      * The ExpiresAt and TotalUses message fields must byte-match what the (auto-deferred,
@@ -1173,8 +1296,12 @@ export declare class ControlDatabase {
      * Read a `FormationInvite` row by token, or null when absent. `expiresAtMs` is
      * the parsed epoch-ms of the stored `datetime` (null when the invite never
      * expires); the caller compares it against the wall clock for freshness.
+     *
+     * `retry: false` is passed only by {@link assertSeatRemains}, which runs INSIDE a
+     * locked write body — same per-call opt-out, and for the same reason, as
+     * {@link queryStampId}'s. Every unlocked caller keeps the default retried path.
      */
-    queryFormationInvite(token: string): Promise<{
+    queryFormationInvite(token: string, retry?: boolean): Promise<{
         token: string;
         sAppId: string;
         expiresAtMs: number | null;
@@ -1185,23 +1312,28 @@ export declare class ControlDatabase {
     /**
      * Count `FormationUsage` rows recorded against a token (uses consumed so far).
      *
-     * NOTE: `Token` left the primary key in `formation-unique-token-redesign` and gained the
-     * `FormationUsageByToken` secondary index, so this filter is now served by an index SEEK
-     * (a descent through the index collection) where it used to be a leading-key partial match
-     * the optimystic vtab declined and the engine served by scan. That matters because the seat
-     * cap is this count: a descent that under-reports admits a seat the invite does not have,
-     * where a scan could not. Descent misses have been observed on a NETWORKED store (tracked
-     * by `debt-composite-pk-point-lookup-unreliable-untracked`, for full-primary-key lookups);
-     * no miss has been observed on this index on a SINGLE node. Across nodes the index is
-     * currently blind: a sibling never learns index entries for rows written elsewhere (the
-     * distributed commit drops the index update — measured 2026-08-12, tracked by
-     * `blocked/secondary-index-seek-blind-to-sibling-rows`), so on a multi-machine party this
-     * count under-reports and the cap over-admits until the upstream fix lands. If
-     * over-admission ever shows up on a converged single node — a token with more usage rows
-     * than its `TotalUses` and no concurrent redeemers to explain it — suspect this seek first
-     * and drop the index to fall back to the scan.
+     * This count IS the seat cap (`enforceFormationUseCap`), so an under-report admits a seat
+     * the invitation never paid for. It is served by a table SCAN, deliberately: a bare
+     * equality on a non-key column is a shape the optimystic vtab declines to claim, so the
+     * engine applies the filter itself over a full read of the table.
+     *
+     * NOTE: it was briefly an index seek, and that is what the scan is protecting against.
+     * `formation-unique-token-redesign` took `Token` out of the primary key and declared
+     * `FormationUsageByToken` purely to spare this scan, recording as a tripwire that an
+     * under-reporting descent would admit an unpaid-for seat — "a failure mode a scan could
+     * not produce". The tripwire fired. Across machines a secondary-index descent returns only
+     * the rows the READING machine wrote, so this count under-reported without bound on a
+     * multi-machine party and the cap over-admitted. The index was removed 2026-08-25; the
+     * measurement is recorded in the block comment at the `FormationUsage` declaration in
+     * `schemas/control.qsql`, and the engine defect in
+     * `blocked/secondary-index-seek-blind-to-sibling-rows`. Do NOT re-declare the index to
+     * speed this up while that ticket is open.
+     *
+     * `retry: false` is passed only by {@link assertSeatRemains}, which runs INSIDE a
+     * locked write body — same per-call opt-out, and for the same reason, as
+     * {@link queryStampId}'s. Every unlocked caller keeps the default retried path.
      */
-    countFormationUsage(token: string): Promise<number>;
+    countFormationUsage(token: string, retry?: boolean): Promise<number>;
     /**
      * Is any `FormationInvite` row still redeemable — unexpired AND with usage
      * below its `TotalUses`? A null `ExpiresAt` never expires and a null