@serfab/cadre-core 0.9.0 → 0.10.0

package/dist/control-write-retry.js

@@ -0,0 +1,411 @@
+import debug from 'debug';
+import { unwrapError } from '@quereus/quereus';
+const log = debug('sereus:cadre:control-db');
+/**
+ * Bounded retry for TRANSIENT control-write failures — the classifier and loop behind
+ * `ControlDatabase.lockedWithRetry`, the single funnel every local control write passes
+ * through.
+ *
+ * Why this exists: the control database replicates every block to the whole party, so a
+ * three-machine party commits control writes unanimously (`ceil(3 × 0.75) = 3` approvals) —
+ * and no `superMajorityThreshold` can lower that bar without violating Optimystic's
+ * partition-safety condition at the shipped admission fraction (the arithmetic lives in
+ * `docs/architecture.md` → "Replication cluster size"). Cadre therefore accepts unanimity
+ * and absorbs the transient half of its cost here: a write that failed because the cohort
+ * did not ANSWER (a stream reset mid-connection-formation, zero approvals) is re-presented
+ * a moment later, which the degraded-cohort-member scenario measured as safe (a failed
+ * write rolls back, nothing half-commits) and effective (the next write commits in ~1 s).
+ *
+ * A write somebody actually REJECTED is never retried — re-presenting it would re-present
+ * a spent signature against a cohort that already said no.
+ *
+ * TWO policies ship from here, and the split is deliberate: the default
+ * ({@link isRetriableControlWriteFailure}, {@link CONTROL_WRITE_ATTEMPTS}) sits under every
+ * control write, and {@link SCHEMA_INIT_RETRY_POLICY} is opted into by
+ * `ControlDatabase.loadSchema` alone. The widened one absorbs one extra failure class whose
+ * re-run safety holds for schema DDL and NOT for writes in general; anything added to the default
+ * list must be safe for all ~19 callers.
+ *
+ * Lives outside `control-database.ts` so a spec can drive the classifier against real engine
+ * errors without importing the whole database class.
+ */
+/**
+ * Attempts allowed per control write: the first, plus two retries of a transiently-failed
+ * cluster commit. Bounded so a genuinely unreachable cohort terminates in the transactor's
+ * own error rather than spinning; two retries covers the observed failure (a stream still
+ * forming when the write fired) with margin.
+ */
+export const CONTROL_WRITE_ATTEMPTS = 3;
+/**
+ * Attempts allowed for {@link SCHEMA_INIT_RETRY_POLICY}, the one call site that also absorbs a
+ * self-coordination refusal.
+ *
+ * Two more than {@link CONTROL_WRITE_ATTEMPTS} because what this retry buys is wall-clock for a
+ * cold-starting node's bootstrap dials to land, and because the refusal is cheap to re-provoke
+ * relative to a cluster round-trip: `findCoordinator` already spends its own bounded wait before
+ * raising (3 attempts, 500 ms apart, `db-p2p/src/libp2p-key-network.ts` ~422) and then decides
+ * LOCALLY that it may not elect itself, so no peer is contacted and no cohort deadline is
+ * consumed.
+ *
+ * Note that the ~1 s `findCoordinator` spends internally is real, on top of this policy's own
+ * backoff — which is exactly why {@link CONTROL_WRITE_RETRY_BUDGET_MS} and not this count is what
+ * actually terminates the loop in the worst case.
+ */
+export const SCHEMA_INIT_ATTEMPTS = 5;
+/**
+ * Elapsed-time ceiling on the whole retry loop, measured from the START of the first
+ * attempt and checked after a failed attempt BEFORE sleeping.
+ *
+ * This budget — not the attempt count — is what makes the policy safe to sit under every
+ * control write. A transient failure (stream reset while a connection is still forming)
+ * surfaces in well under a second, so it is retried and the loop adds at most ~2.2 s. A
+ * genuinely silent cohort member fails at ~20 s (two 10 s `ClusterClient` response-deadline
+ * attempts, measured in `control-write-degraded-cohort-member.integration.ts`), which
+ * already exceeds this budget when attempt 1 returns — so that case is surfaced immediately
+ * and retry adds ZERO latency to the case where it cannot help.
+ */
+export const CONTROL_WRITE_RETRY_BUDGET_MS = 10000;
+/**
+ * Backoff before retry N, jittered ±50% (so the first sleep is 125–375 ms — the jitter can
+ * never floor a delay to zero). Each single sleep is additionally capped at the LARGEST
+ * delay in the list, so a `stop()` racing a backoff is never held up by more than ~1 s.
+ * There is no `AbortSignal` at this seam to cancel a sleep outright; the cap is the
+ * substitute, deliberately not plumbing one through every write path.
+ */
+const CONTROL_WRITE_RETRY_DELAYS_MS = [250, 1000];
+/**
+ * Backoff for {@link SCHEMA_INIT_RETRY_POLICY}, jittered and capped the same way. Worst case
+ * ~4.6 s of sleep across four retries (375 + 750 + 1500 + 2000, each capped at the largest base),
+ * which stays inside {@link CONTROL_WRITE_RETRY_BUDGET_MS} — the budget still terminates the loop
+ * first if any single attempt is itself slow.
+ *
+ * Sized against measured facts, not guessed:
+ *
+ *  - the self-coordination guard clears the moment the node has ONE connection again (both
+ *    branches that raise `grace-period-not-elapsed` require `getConnections().length === 0`), not
+ *    when the 30 s grace period expires — so this does NOT need to cover the grace period, only
+ *    the gap until a bootstrap dial completes;
+ *  - the guard's 30 s `gracePeriodMs` cannot be tuned from this repo anyway: no caller in either
+ *    repo passes a `SelfCoordinationConfig`, so the default is always in force;
+ *  - a node that has NEVER connected is waved through as a bootstrap node (network high-water
+ *    mark of 1), so this retry only ever delays a node that HAS seen peers. One whose peers are
+ *    gone for good still fails, ~4.6 s later — which is why the budget stays at the shared
+ *    ceiling rather than stretching to cover the full 30 s window.
+ */
+const SCHEMA_INIT_RETRY_DELAYS_MS = [250, 500, 1000, 2000];
+/** The network transactor's aggregate, raised when some cohort peers gave no usable answer. */
+const TRANSACTOR_AGGREGATE = /Some peers did not complete:/;
+/**
+ * How the aggregate's per-batch details name a SINGLE-block batch — the shape both the block
+ * read (`get`) and phase 1 (`pend`) format, and the only shape safe to re-present.
+ */
+const SINGLE_BLOCK_BATCH_TOKEN = '[block:';
+/**
+ * How the aggregate's per-batch details name a MULTI-block batch — the shape phase 2
+ * (`commitBlocks`) formats, and the marker of an outcome too indeterminate to retry.
+ */
+const COMMIT_BATCH_TOKEN = '[blocks:';
+/**
+ * The cluster coordinator's super-majority shortfall (`db-p2p/src/repo/cluster-coordinator.ts`),
+ * matched ONLY with a ZERO rejection count. The same message with a non-zero count means a
+ * member actually voted no (that branch carries `membership-not-admitted` rejections), and
+ * retrying it would re-present a spent signature to a cohort that already refused it. This one
+ * is raised while collecting PROMISES, before any commit, so re-presenting it is safe. A
+ * decisive rejection is a different error entirely (`ValidatorRejectionError`,
+ * `Transaction rejected by validators`), which this classifier also never matches.
+ */
+const SUPER_MAJORITY_SHORTFALL_UNANSWERED = /Failed to get super-majority: \d+\/\d+ approvals \(needed \d+, 0 rejections\)/;
+/**
+ * Is this a transactor aggregate from a phase where nothing can have committed yet?
+ *
+ * The transactor raises `Some peers did not complete:` from three sites
+ * (`db-core/src/transactor/network-transactor.ts`), and only two of them are safe to
+ * re-present:
+ *
+ * - `get` (a block read, which a write body also performs) and `pend` (phase 1) fail before
+ *   anything commits, so the write is known not to have landed;
+ * - `commitBlocks` (phase 2) is REACHABLE here and NOT safe. `commitBlock` throws the
+ *   aggregate when a header/tail commit got NO response at all — precisely the transient
+ *   class this retry targets — and that escapes `commit()`, survives `TransactorSource`'s
+ *   cancel-and-rethrow (cancel reaches PENDING actions only; a peer that already committed
+ *   stays committed) and `Collection.syncInternal` (which retries StaleFailure RETURN values,
+ *   not throws), reaching this funnel QuereusError-wrapped. The tail commit is one batch to
+ *   one coordinator running consensus internally, so a no-response there is INDETERMINATE:
+ *   the commit may have completed with only the response lost. Re-running the write body over
+ *   a write that landed turns a success into a constraint failure (e.g.
+ *   `UNIQUE constraint failed: CadrePeer.PeerId`). Surfacing the transient error instead lets
+ *   the caller re-read committed state and decide. Commit-phase messages are vetoed by
+ *   {@link reportsIndeterminateCommit}, not here.
+ *
+ * The `cause` chain does not discriminate the phases — a commit-phase aggregate carries the
+ * same stream-reset/dial error a pend-phase one does. The per-batch DETAIL text does: `get`
+ * and `pend` format `<peerId>[block:<id>](<status>)`, `commitBlocks` formats
+ * `<peerId>[blocks:<count>](<status>)`. `[block:` cannot occur inside `[blocks:`, so the two
+ * tokens are disjoint and separate the phases.
+ *
+ * NOTE: the discriminator is a formatting detail of another repo. If Optimystic ever reformats
+ * those per-batch details, this fails CLOSED — the aggregate stops matching and control writes
+ * simply stop being retried, silently losing the absorption rather than doing anything unsafe.
+ * The guard against that is a scenario asserting the classifier against a live cluster failure
+ * (`control-write-retry-scenario-coverage`); if that lands and later reddens here, an upstream
+ * reformat is the first thing to check.
+ *
+ * An aggregate whose details came out EMPTY (possible when `formatBatchStatuses` has no
+ * batches to format) carries neither token, matches nothing here, and is not retried — an
+ * unattributable failure is not a proven non-commit.
+ */
+function isUncommittedTransactorAggregate(message) {
+    return TRANSACTOR_AGGREGATE.test(message) && message.includes(SINGLE_BLOCK_BATCH_TOKEN);
+}
+/**
+ * Does ANY message in the failure's `cause` chain describe a commit-phase batch — i.e. an
+ * outcome nobody can call committed or not?
+ *
+ * Vetoes the whole chain rather than the one message carrying the token, and vetoes it
+ * regardless of which matcher would otherwise have claimed it. The narrower per-message rule
+ * would already catch every shape the transactor is known to build (a commit-phase aggregate
+ * emits `[blocks:` for every batch it formats, and embeds its cause's message inline as
+ * `root: …`, so both tokens land in one string) — but "known to build" is an argument about
+ * another repo's error assembly, and this way an indeterminate commit anywhere in the chain
+ * costs a retry we could have had instead of risking a re-run over a write that landed.
+ */
+function reportsIndeterminateCommit(messages) {
+    return messages.some(message => message.includes(COMMIT_BATCH_TOKEN));
+}
+/** The coordinator's shortfall, raised pre-commit, with nobody having voted no. */
+function isUnansweredSuperMajorityShortfall(message) {
+    return SUPER_MAJORITY_SHORTFALL_UNANSWERED.test(message);
+}
+/**
+ * Optimystic refusing to let a node elect ITSELF coordinator because its last connection dropped
+ * less than the guard's grace period ago (`Libp2pKeyPeerNetwork.shouldAllowSelfCoordination` →
+ * `findCoordinator`, `db-p2p/src/libp2p-key-network.ts`).
+ *
+ * Matched ONLY with the `grace-period-not-elapsed` reason. The same sentence is raised with three
+ * other reasons and none of them is worth a retry: `disabled` is static configuration, and
+ * `partition-detected` / `suspicious-shrinkage` describe a network condition that will not resolve
+ * inside a few seconds of backoff. The grace-period one WILL — it clears as soon as the node holds
+ * one connection again, which a cold-starting node dialling its bootstrap peers is actively
+ * working on.
+ *
+ * NOTE: matched by TEXT, not by type, and that is forced rather than chosen.
+ * `FindCoordinatorError` (`code: 'SELF_COORDINATION_BLOCKED'`) is exported from `@optimystic/db-p2p`
+ * and this package depends on it, but the error OBJECT does not survive the trip:
+ * `OptimysticVirtualTable.initialize` catches and rethrows as `new Error(message)` with no `cause`
+ * (`quereus-plugin-optimystic/src/optimystic-module.ts`), so only the text reaches this repo. Like
+ * every matcher here this fails CLOSED — a rewording upstream stops the retry engaging, it never
+ * makes it unsafe.
+ */
+const SELF_COORDINATION_GRACE_REFUSAL = /Self-coordination blocked: grace-period-not-elapsed\. No coordinator available for key\./;
+/** Self-coordination refused because the node's last connection dropped moments ago. */
+function isSelfCoordinationGraceRefusal(message) {
+    return SELF_COORDINATION_GRACE_REFUSAL.test(message);
+}
+/**
+ * Every message in the failure's `cause` chain.
+ *
+ * `unwrapError` declares its `message` as `string`, but it follows `.cause` without checking what
+ * that holds — a chain link that is not an `Error` (a stream rejected with a bare string reason,
+ * an `AbortSignal.reason` that is a plain object) yields `undefined` there, and calling
+ * `.includes` on it would throw a `TypeError` out of this classifier, INSIDE `retryControlWrite`'s
+ * catch, replacing the real control-write failure with a confusing one. Non-strings are dropped:
+ * a link nobody can read is a link that matches nothing, which is already the conservative answer.
+ *
+ * NOTE: a cause chain with a CYCLE would spin forever inside `unwrapError` itself. No error in
+ * this repo builds one; if a hang ever localises to a control-write failure path, look there.
+ */
+function chainMessages(error) {
+    return unwrapError(error)
+        .map(({ message }) => message)
+        .filter(message => typeof message === 'string');
+}
+/**
+ * The error messages that mean "the cohort did not answer AND nothing committed", and nothing
+ * else. The observed one-shot failure was `registerSelf()` racing a connection still forming:
+ * a read/pend-phase transactor aggregate with `cause=The stream has been reset`.
+ *
+ * Deliberately NOT matched: every constraint / authorization failure
+ * (`CHECK constraint failed:`, `UNIQUE constraint failed:`) — those reach the same funnel
+ * and must be retried zero times. In particular the lost-use-number messages
+ * (`isLostUseNumberRace` in `control-database.ts`) are constraint failures, so this
+ * classifier and that one are DISJOINT and the two retry loops can never multiply.
+ */
+const RETRIABLE_CONTROL_WRITE_MATCHERS = [
+    isUncommittedTransactorAggregate,
+    isUnansweredSuperMajorityShortfall,
+];
+/**
+ * The above PLUS the self-coordination grace refusal — the classifier for schema init ONLY
+ * ({@link SCHEMA_INIT_RETRY_POLICY}), deliberately not folded into the list above.
+ *
+ * Why this class is safe HERE and not everywhere: a write refused at coordinator SELECTION never
+ * reached a peer, so re-presenting it is a proven non-commit. But `findCoordinator` is also
+ * reached from `NetworkTransactor.commitBlock` during PHASE 2 (`resolveCoordinator`,
+ * `db-core/src/transactor/network-transactor.ts`), and `commit()` commits the header block before
+ * the rest — so a general control write refused there can be refused AFTER something already
+ * committed. Worse, that error carries no `[blocks:` batch token, so
+ * {@link reportsIndeterminateCommit} would not veto it, and re-running an insert body over a write
+ * that landed is exactly the `UNIQUE constraint failed: CadrePeer.PeerId` failure that veto exists
+ * to prevent.
+ *
+ * Schema init does not have that problem: `apply schema` is a diff rather than a replay and a
+ * failed `create table` leaves the catalog clean, so a re-run emits only the tables that did not
+ * land (the full argument is at the `loadSchema` call site in `control-database.ts`).
+ */
+const RETRIABLE_SCHEMA_INIT_MATCHERS = [
+    ...RETRIABLE_CONTROL_WRITE_MATCHERS,
+    isSelfCoordinationGraceRefusal,
+];
+/**
+ * Did this control write fail because the cluster cohort did not ANSWER — i.e. is
+ * re-presenting the SAME signed write a moment later the right response?
+ *
+ * Classifies by MESSAGE, walking the `cause` chain with Quereus' own {@link unwrapError} —
+ * the exact shape of `isLostUseNumberRace`, for the same reason: neither surface is
+ * recognisable by type. The transactor and the coordinator both throw bare `Error`s, and by
+ * the time one reaches `ControlDatabase` it is wrapped in a `QuereusError` (the real chain
+ * is `QuereusError` → `Error` → `Error`), so the match must work at any depth. Anything
+ * that is not an `Error` is never retried.
+ *
+ * {@link reportsIndeterminateCommit} vetoes the whole chain before any matcher runs, so a
+ * failure that reports a commit-phase batch anywhere is never retried on the strength of some
+ * other level looking transient.
+ *
+ * NOTE: this depends on engine/transactor error TEXT. The messages producible without a
+ * network are driven from the REAL engine in `control-formation-use-number-retry.spec.ts`, so
+ * a rewording reddens a spec rather than silently disabling the retry. The two RETRIABLE
+ * messages need a real multi-node cluster to produce and are still literals in
+ * `control-write-retry.spec.ts`; `control-write-retry-scenario-coverage` closes that gap.
+ */
+export function isRetriableControlWriteFailure(error) {
+    return matchesRetriableMessage(error, RETRIABLE_CONTROL_WRITE_MATCHERS);
+}
+/**
+ * {@link isRetriableControlWriteFailure} widened by exactly one class — the self-coordination
+ * grace refusal ({@link RETRIABLE_SCHEMA_INIT_MATCHERS}) — for `ControlDatabase.loadSchema`.
+ *
+ * Opt-in per call site, via {@link SCHEMA_INIT_RETRY_POLICY}. Every other control write keeps the
+ * default classifier untouched.
+ */
+export function isRetriableSchemaInitFailure(error) {
+    return matchesRetriableMessage(error, RETRIABLE_SCHEMA_INIT_MATCHERS);
+}
+/**
+ * Shared body of both classifiers: walk the `cause` chain, let
+ * {@link reportsIndeterminateCommit} veto it, then ask `matchers`. The veto runs BEFORE any
+ * matcher for every policy — a chain reporting a commit-phase batch is never retried, however
+ * transient some other level looks.
+ */
+function matchesRetriableMessage(error, matchers) {
+    if (!(error instanceof Error)) {
+        return false;
+    }
+    const messages = chainMessages(error);
+    if (reportsIndeterminateCommit(messages)) {
+        return false;
+    }
+    return messages.some(message => matchers.some(matches => matches(message)));
+}
+/**
+ * The one non-default retry policy: `ControlDatabase.loadSchema`'s distributed DDL, which
+ * additionally absorbs optimystic's self-coordination grace refusal and gets more attempts over a
+ * longer backoff to do it (see {@link SCHEMA_INIT_ATTEMPTS} and
+ * {@link SCHEMA_INIT_RETRY_DELAYS_MS}). Still bounded by the shared
+ * {@link CONTROL_WRITE_RETRY_BUDGET_MS}.
+ *
+ * NOTE: exactly one call site opts in today, and the re-run safety this policy assumes is that
+ * site's, not a general property — `apply schema` is a diff and a failed `create table` leaves the
+ * catalog clean. A second opt-in must re-derive that argument for its own write body first; if this
+ * policy ever grows a third consumer, rename it for what the callers share rather than widening it
+ * by default.
+ */
+export const SCHEMA_INIT_RETRY_POLICY = {
+    attempts: SCHEMA_INIT_ATTEMPTS,
+    delaysMs: SCHEMA_INIT_RETRY_DELAYS_MS,
+    isRetriable: isRetriableSchemaInitFailure,
+};
+/**
+ * Run `attempt` up to {@link CONTROL_WRITE_ATTEMPTS} times (or `options.attempts`), retrying only
+ * failures the policy's classifier — {@link isRetriableControlWriteFailure} by default —
+ * calls transient, and only while {@link CONTROL_WRITE_RETRY_BUDGET_MS} has not elapsed.
+ *
+ * `attempt` must be atomic and re-runnable: a failed cluster write rolls back (nothing is
+ * half-applied), and re-running the body re-runs its reads too — `ControlDatabase`'s
+ * locked write bodies all satisfy this (see the contract on
+ * `ControlDatabase.withWriteLock`). The backoff sleeps happen with NO lock held; the caller
+ * takes and releases its lock inside `attempt`.
+ *
+ * On exhaustion (attempts or budget) the LAST error is rethrown unchanged — never wrapped,
+ * so the exact messages downstream code and the degraded-cohort-member scenario assert on
+ * survive. A non-retriable failure propagates from the attempt that raised it.
+ */
+export async function retryControlWrite(attempt, options = {}) {
+    const attempts = Math.max(1, options.attempts ?? CONTROL_WRITE_ATTEMPTS);
+    const delays = options.delaysMs ?? CONTROL_WRITE_RETRY_DELAYS_MS;
+    const isRetriable = options.isRetriable ?? isRetriableControlWriteFailure;
+    const sleep = options.sleep ?? defaultSleep;
+    const now = options.now ?? Date.now;
+    const start = now();
+    let lastError;
+    let attemptsMade = 0;
+    for (let attemptNumber = 1; attemptNumber <= attempts; attemptNumber++) {
+        attemptsMade = attemptNumber;
+        try {
+            const result = await attempt();
+            if (attemptNumber > 1) {
+                log('Control write committed on attempt %d/%d', attemptNumber, attempts);
+            }
+            return result;
+        }
+        catch (error) {
+            if (!isRetriable(error)) {
+                // The ONLY trace that this funnel saw a failure and declined it. Without it the
+                // log is silent either way, so "the classifier vetoed this one" is
+                // indistinguishable from "the retry is not wired into this path at all".
+                // "here" is load-bearing: this fires for every non-transient failure, and some
+                // of them — the lost-use-number constraint failures — ARE retried by the OUTER
+                // loop (`ControlDatabase.withUseNumberRetry`) that this one nests inside.
+                log('Control write failed non-transiently on attempt %d/%d, not retried here: %s', attemptNumber, attempts, error);
+                throw error;
+            }
+            lastError = error;
+            if (attemptNumber === attempts) {
+                break;
+            }
+            const elapsed = now() - start;
+            if (elapsed >= CONTROL_WRITE_RETRY_BUDGET_MS) {
+                break;
+            }
+            const delay = jitteredDelay(delays, attemptNumber);
+            log('Control write failed transiently (attempt %d/%d), retrying in %d ms: %s', attemptNumber, attempts, delay, error);
+            await sleep(delay);
+        }
+    }
+    log('Control write failed after %d/%d attempt(s): %s', attemptsMade, attempts, lastError);
+    throw lastError;
+}
+/**
+ * Backoff for the retry that follows attempt `attemptNumber`: the matching base delay
+ * (last entry repeats), jittered ±50%, then capped at the list's largest base so no single
+ * sleep exceeds it (see {@link CONTROL_WRITE_RETRY_DELAYS_MS} for why). `Math.random` is
+ * fine here — the jitter only de-synchronizes concurrent retriers, nothing is derived
+ * from it.
+ *
+ * NOTE: the cap bites on the LAST delay, whose base IS the largest, so ~half of those sleeps
+ * land exactly on the cap rather than spread — de-synchronization is only partial there.
+ * Harmless while retriers are a handful of party nodes whose attempts already start seconds
+ * apart; if a party ever retries in a tight synchronized herd, raise the cap above the
+ * largest base instead of jittering about it.
+ */
+function jitteredDelay(delays, attemptNumber) {
+    if (delays.length === 0) {
+        return 0;
+    }
+    const base = delays[Math.min(attemptNumber - 1, delays.length - 1)];
+    const jittered = base * (0.5 + Math.random());
+    return Math.min(jittered, Math.max(...delays));
+}
+function defaultSleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+//# sourceMappingURL=control-write-retry.js.map

package/dist/control-write-retry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"control-write-retry.js","sourceRoot":"","sources":["../src/control-write-retry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAE/C,MAAM,GAAG,GAAG,KAAK,CAAC,yBAAyB,CAAC,CAAC;AAE7C;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC;AAExC;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAEtC;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,6BAA6B,GAAG,KAAM,CAAC;AAEpD;;;;;;GAMG;AACH,MAAM,6BAA6B,GAAsB,CAAC,GAAG,EAAE,IAAK,CAAC,CAAC;AAEtE;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,2BAA2B,GAAsB,CAAC,GAAG,EAAE,GAAG,EAAE,IAAK,EAAE,IAAK,CAAC,CAAC;AAEhF,+FAA+F;AAC/F,MAAM,oBAAoB,GAAG,8BAA8B,CAAC;AAE5D;;;GAGG;AACH,MAAM,wBAAwB,GAAG,SAAS,CAAC;AAE3C;;;GAGG;AACH,MAAM,kBAAkB,GAAG,UAAU,CAAC;AAEtC;;;;;;;;GAQG;AACH,MAAM,mCAAmC,GACxC,+EAA+E,CAAC;AAEjF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,SAAS,gCAAgC,CAAC,OAAe;IACxD,OAAO,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAC,CAAC;AACzF,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,0BAA0B,CAAC,QAA2B;IAC9D,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC;AACvE,CAAC;AAED,mFAAmF;AACnF,SAAS,kCAAkC,CAAC,OAAe;IAC1D,OAAO,mCAAmC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAC1D,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,+BAA+B,GACpC,0FAA0F,CAAC;AAE5F,wFAAwF;AACxF,SAAS,8BAA8B,CAAC,OAAe;IACtD,OAAO,+BAA+B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACtD,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAS,aAAa,CAAC,KAAY;IAClC,OAAO,WAAW,CAAC,KAAK,CAAC;SACvB,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC;SAC7B,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC;AAClD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,gCAAgC,GAA8C;IACnF,gCAAgC;IAChC,kCAAkC;CAClC,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,8BAA8B,GAA8C;IACjF,GAAG,gCAAgC;IACnC,8BAA8B;CAC9B,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,UAAU,8BAA8B,CAAC,KAAc;IAC5D,OAAO,uBAAuB,CAAC,KAAK,EAAE,gCAAgC,CAAC,CAAC;AACzE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,4BAA4B,CAAC,KAAc;IAC1D,OAAO,uBAAuB,CAAC,KAAK,EAAE,8BAA8B,CAAC,CAAC;AACvE,CAAC;AAED;;;;;GAKG;AACH,SAAS,uBAAuB,CAC/B,KAAc,EACd,QAAmD;IAEnD,IAAI,CAAC,CAAC,KAAK,YAAY,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC;IACd,CAAC;IACD,MAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACtC,IAAI,0BAA0B,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1C,OAAO,KAAK,CAAC;IACd,CAAC;IACD,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAC7E,CAAC;AA8BD;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAuC;IAC3E,QAAQ,EAAE,oBAAoB;IAC9B,QAAQ,EAAE,2BAA2B;IACrC,WAAW,EAAE,4BAA4B;CACzC,CAAC;AAEF;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACtC,OAAyB,EACzB,UAAoC,EAAE;IAEtC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,IAAI,sBAAsB,CAAC,CAAC;IACzE,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,IAAI,6BAA6B,CAAC;IACjE,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,8BAA8B,CAAC;IAC1E,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,YAAY,CAAC;IAC5C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACpC,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC;IACpB,IAAI,SAAkB,CAAC;IACvB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,IAAI,QAAQ,EAAE,aAAa,EAAE,EAAE,CAAC;QACxE,YAAY,GAAG,aAAa,CAAC;QAC7B,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,MAAM,OAAO,EAAE,CAAC;YAC/B,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;gBACvB,GAAG,CAAC,0CAA0C,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;YAC1E,CAAC;YACD,OAAO,MAAM,CAAC;QACf,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzB,gFAAgF;gBAChF,mEAAmE;gBACnE,yEAAyE;gBACzE,+EAA+E;gBAC/E,+EAA+E;gBAC/E,0EAA0E;gBAC1E,GAAG,CAAC,6EAA6E,EAChF,aAAa,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;gBACjC,MAAM,KAAK,CAAC;YACb,CAAC;YACD,SAAS,GAAG,KAAK,CAAC;YAClB,IAAI,aAAa,KAAK,QAAQ,EAAE,CAAC;gBAChC,MAAM;YACP,CAAC;YACD,MAAM,OAAO,GAAG,GAAG,EAAE,GAAG,KAAK,CAAC;YAC9B,IAAI,OAAO,IAAI,6BAA6B,EAAE,CAAC;gBAC9C,MAAM;YACP,CAAC;YACD,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;YACnD,GAAG,CAAC,yEAAyE,EAC5E,aAAa,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;YACxC,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;QACpB,CAAC;IACF,CAAC;IACD,GAAG,CAAC,iDAAiD,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC1F,MAAM,SAAS,CAAC;AACjB,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAS,aAAa,CAAC,MAAyB,EAAE,aAAqB;IACtE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,CAAC;IACV,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAE,CAAC;IACrE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,YAAY,CAAC,EAAU;IAC/B,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AACxD,CAAC"}

package/dist/delegate-admission.d.ts

@@ -0,0 +1,134 @@
+/**
+ * Delegate-peer admission grants for a party control node that also runs the
+ * circuit-relay server.
+ *
+ * A strand node runs as its own libp2p instance with its own transport peerId,
+ * derived from the cadre identity key + strandId (`strand-transport-key.ts`).
+ * That derivation uses the member's PRIVATE seed by design, so no third party —
+ * including a co-cadre relay — can recompute it. When a NAT'd member's strand
+ * node tries to reserve a circuit-relay slot on a sibling's control node, the
+ * relay's membership connection gater sees an unknown peerId and denies the
+ * connection, which kills the reservation and fails the strand's
+ * `libp2p.start()` outright.
+ *
+ * The resolution (docs/strands.md → "Relays and admission"): a party control
+ * node running a relay is party-private infrastructure — it relays for its own
+ * party's nodes, including the extra transport identities its members' strand
+ * nodes run as, and for nobody else. The mechanism is a **member-announced
+ * delegate grant**: before a member's control node starts a strand node, it
+ * tells its siblings (over the already-authenticated `/sereus/strand-addr/1.0.0`
+ * RPC) the derived peerId that strand node will use, and the receiver holds a
+ * short-lived, in-memory admission grant for exactly that peerId. The grant is
+ * consulted ONLY by the connection-level gate
+ * (`CadreNode.admitInboundControlConnection`); the fail-closed per-stream gate
+ * (`authorizeInboundControlStream`) never honors it, so a delegate gets the
+ * connection (all a circuit-relay `hop` reservation needs) and nothing more.
+ *
+ * A grant is a delegation of trust to an already-trusted member: the receiver
+ * cannot verify that the announced peerId really is the member's strand node,
+ * so an authorized member can hand CONNECTION-level access to a peerId of its
+ * choosing for up to {@link DELEGATE_GRANT_TTL_MS}. The caps below bound how
+ * far a buggy or compromised member can stretch that.
+ *
+ * The receiver side is {@link DelegateAdmissionStore}. The announcer side keeps
+ * its own throttle state (a "last announced at" map that `CadreNode` owns) and
+ * decides from it, per reconcile pass, which relays are due a re-announce —
+ * {@link dueRelayAnnounces} and {@link pruneStoppedStrandAnnounces} are that
+ * decision, pure and testable, kept here beside the TTL they are derived from.
+ */
+/**
+ * How long one announced delegate grant stays valid without a re-announce.
+ * Announcers refresh at half this (`CadreNode`'s reconcile pass), so a live
+ * strand's grant never lapses while its control node can still reach the relay.
+ */
+export declare const DELEGATE_GRANT_TTL_MS: number;
+/** Cap on live grants per announcing member (soonest-expiry evicted first). */
+export declare const MAX_DELEGATE_GRANTS_PER_MEMBER = 32;
+/** Cap on live grants in total (soonest-expiry evicted first). */
+export declare const MAX_DELEGATE_GRANTS = 256;
+/**
+ * Composite map key for the per-(peer, strand) state on both sides: the
+ * receiver's replace-per-(announcer, strand) grant, and the announcer's
+ * per-(target, strand) throttle timestamp.
+ */
+export declare function peerStrandKey(peerId: string, strandId: string): string;
+/**
+ * In-memory store of delegate admission grants, keyed by (announcer, strandId)
+ * so a re-announce REPLACES the previous delegate peerId rather than
+ * accumulating — a restarted strand cannot leak grants. Expired entries are
+ * pruned lazily on every read/write. Injectable `now` keeps expiry testable
+ * without fake timers.
+ */
+export declare class DelegateAdmissionStore {
+    private readonly grants;
+    private readonly ttlMs;
+    constructor(ttlMs?: number);
+    /** Number of live (unexpired at last prune) grants — test/diagnostic surface. */
+    get size(): number;
+    /**
+     * Record (or refresh) a grant: `delegatePeerId` acts on behalf of
+     * `announcerPeerId` for `strandId`, valid for the store's TTL from `now`.
+     * Enforces {@link MAX_DELEGATE_GRANTS_PER_MEMBER} and
+     * {@link MAX_DELEGATE_GRANTS}, evicting the soonest-expiry grant first —
+     * biased against the stalest delegation, never against the fresh announce.
+     */
+    grant(announcerPeerId: string, strandId: string, delegatePeerId: string, now?: number): void;
+    /**
+     * Drop every grant. Called when the owning node stops: grants are scoped to
+     * the session that recorded them, and outliving it would admit delegates
+     * announced against a node that no longer exists.
+     */
+    clear(): void;
+    /** Is `remotePeerId` covered by a live grant? Prunes expired entries as it goes. */
+    has(remotePeerId: string, now?: number): boolean;
+    /** Drop every grant whose expiry has passed. */
+    private prune;
+    /**
+     * If the grants matching `filter` are at `cap`, evict the soonest-expiry one
+     * among them to make room for the insert the caller is about to do.
+     */
+    private evictAtCap;
+}
+/** A circuit relay named by a multiaddr, as an announce target. */
+export interface CircuitRelayTarget {
+    /** The relay's peerId — the `<relay>` in `…/p2p/<relay>/p2p-circuit…`. */
+    relayPeerId: string;
+    /**
+     * The relay's DIRECT dial multiaddr (the prefix before `/p2p-circuit`),
+     * usable as a dial fallback when no connection to the relay is open yet.
+     */
+    relayAddr: string;
+}
+/**
+ * Extract the circuit relays named by a multiaddr list — for each
+ * `…/p2p/<relay>/p2p-circuit…` entry, the relay's peerId and its direct dial
+ * prefix. A bare `/p2p-circuit` (no relay named), a non-circuit addr, an
+ * unparsable addr, and an unparsable relay peerId are all skipped. A trailing
+ * `/p2p/<dst>` after the circuit component names the DESTINATION, never the
+ * relay, and is ignored. Deduplicated by relay peerId (first addr wins).
+ */
+export declare function extractCircuitRelayTargets(addrs: string[]): CircuitRelayTarget[];
+/**
+ * Of `relays`, the ones whose grant for `strandId` is due a re-announce: never
+ * announced to, or last announced at least half the TTL ago. Half, so a failed
+ * refresh has one more attempt before the grant lapses — a lapsed grant means
+ * the relay denies the strand node's reservation re-dial.
+ *
+ * `announceAt` is the announcer-side "last announced at" map, keyed by
+ * {@link peerStrandKey} on the peer announced TO.
+ */
+export declare function dueRelayAnnounces(announceAt: ReadonlyMap<string, number>, relays: readonly CircuitRelayTarget[], strandId: string, now: number, ttlMs?: number): CircuitRelayTarget[];
+/**
+ * Drop `announceAt` entries whose strand is no longer running — a stopped
+ * strand needs no refresh, and without this the map grows for the node's
+ * lifetime. Mutates in place.
+ */
+export declare function pruneStoppedStrandAnnounces(announceAt: Map<string, number>, runningStrandIds: ReadonlySet<string>): void;
+/**
+ * The circuit relay named by `addr`, throwing when `addr` is unparsable or names
+ * no relay. The throwing form exists for CONFIG-facing callers (`relay-addrs.ts`),
+ * where a typo must fail loudly rather than cost the node its reachability;
+ * {@link extractCircuitRelayTargets} reads runtime-discovered addrs instead and
+ * logs-and-skips.
+ */
+export declare function circuitRelayTargetOrThrow(addr: string): CircuitRelayTarget;