@serfab/cadre-core 1.4.0 → 1.6.0

package/dist/cadre-node.d.ts

@@ -308,8 +308,10 @@ export declare class CadreNode implements SAppIdLookup {
      * (`strand-membership-reconciler.ts`, wired via `StartStrandConfig.pendingMembershipInvite`
      * in {@link launchStrand}): it deletes the entry once the invitation is redeemed, burned
      * against an already-seated member, or found dead (expired / cancelled / consumed
-     * elsewhere / the strand sealed). Until the strand is actually launched here the entry
-     * just waits.
+     * elsewhere / the strand sealed) — naming the invitation it settled, so a re-formation
+     * that replaced the entry meanwhile keeps its fresh one
+     * ({@link unstageMembershipInvite}). Until the strand is actually launched here the entry
+     * just waits; once launched, staging notifies the manager, which re-arms a finished loop.
      *
      * NOTE: entries live for the node's lifetime (one small pair per formed closed
      * strand), the same unbounded-keys tripwire {@link crossPartyStrandAddrs} documents —
@@ -1357,9 +1359,14 @@ export declare class CadreNode implements SAppIdLookup {
      * {@link SeedBootstrapService} this node builds — `applySeed`'s owner dials and
      * `dialInvite`.
      *
-     * See {@link DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS} for why a peer's dial is
-     * bounded as a whole, and {@link DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS}
-     * for why each address is bounded as well.
+     * See `peer-dial.ts`'s `DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS` for why a peer's dial is
+     * bounded as a whole, and `DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS` for why each
+     * address is bounded as well.
+     *
+     * Both are DERIVED from `network.linkRoundTripMs` rather than fixed, because the slowest
+     * address either has to cover is a relayed dial and that costs a fixed number of exchanges —
+     * `link-budget.ts` has the counts. A host's explicit `controlCohort` values still win, so a
+     * test that drives dead addresses on purpose keeps the duration it chose.
      */
     private controlDialBudget;
     /**
@@ -2381,8 +2388,10 @@ export declare class CadreNode implements SAppIdLookup {
      * caller runs the drive regardless).
      *
      * NOTE: against a relay that is DOWN this hook costs up to two strand-addr
-     * timeouts (10 s each: dial by peer id, then by addr) before the 10 s drive even
-     * starts, so one failed re-drive can hold the supervisor `driving` for ~30 s.
+     * timeouts (10 s each: dial by peer id, then by addr) before the reservation drive even
+     * starts, so one failed re-drive holds the supervisor `driving` for those 20 s plus the
+     * drive's own deadline — 34 s at the default declared link round trip, and longer on a host
+     * that declared a slower one (`link-budget.ts`).
      * Bounded and harmless while the relay is unreachable anyway; if recovery
      * latency after a relay comes back ever matters, skip the announce when the
      * control node holds no connection to the relay (the drive's own dial fails
@@ -2602,13 +2611,15 @@ export declare class CadreNode implements SAppIdLookup {
      * local), and only then does it reach out. Everything above in {@link start}
      * has completed by the time this runs.
      *
-     * BUDGET: the supervisor's first attempt — `relay-reservation.ts`'s
-     * `DEFAULT_RELAY_RESERVE_TIMEOUT_MS`, 10 s — is what `start()` waits on — deliberately the module default rather
-     * than a boot-specific one. A healthy dial-plus-reserve is sub-second even over
-     * a WAN, so 10 s is slack for a slow link; going much longer would make a dead
-     * relay indistinguishable from a hung start, and much shorter would fail nodes
-     * on links that were merely slow. The retries carry on in the background after
-     * this resolves, exactly as they do for a {@link reserveRelays} caller.
+     * BUDGET: the supervisor's first attempt is what `start()` waits on — deliberately the drive's
+     * ordinary deadline rather than a boot-specific one. That deadline is now COUNTED, four link
+     * round trips at the declared `network.linkRoundTripMs` (14 s at its default, where it was a
+     * fixed 10 s): a healthy dial-plus-reserve is sub-second even over a WAN, so this is slack for
+     * a slow link, while going much longer would make a dead relay indistinguishable from a hung
+     * start and much shorter would fail nodes on links that were merely slow. A host that declares
+     * a slower link lengthens it without touching this path — `link-budget.ts`. The retries carry
+     * on in the background after this resolves, exactly as they do for a {@link reserveRelays}
+     * caller.
      *
      * `network.requireRelay === false` softens only the outcome below: a first
      * attempt that lands no `/p2p-circuit` address is logged instead of thrown, and
@@ -3057,7 +3068,11 @@ export declare class CadreNode implements SAppIdLookup {
      *    an intact control DB) and mints + persists a fresh one otherwise. The invitation
      *    will admit THIS key's public half as the `Strand.Member`.
      * 2. Stage the invitation in {@link pendingMembershipInvites} for the strand
-     *    bring-up flow (`strand-node-binds-member-peer`) to redeem via `consumeInvite`.
+     *    bring-up flow (`strand-node-binds-member-peer`) to redeem via `consumeInvite`,
+     *    and tell the instance manager, which re-arms an already-finished membership
+     *    reconciler so a RE-formation against a launched strand is attempted at once — the
+     *    removed-party case, where the loop finished long before the removal. A strand not
+     *    launched here yet has no loop to re-arm, and its bring-up finds the entry.
      *
      * Throws — failing the whole {@link formStrand} — when the identity cannot be
      * persisted: a joiner "joined" without a persistable identity could never become a
@@ -3067,6 +3082,14 @@ export declare class CadreNode implements SAppIdLookup {
      * rejected) and redeeming a FRESH invitation.
      */
     private adoptFormationMembershipInvite;
+    /**
+     * The reconciler's half of the {@link pendingMembershipInvites} seam: drop `settled`
+     * (spent, burned, or dead) only while it is still the staged entry. A re-formation
+     * replaces the entry between a pass's read and its settle, and the fresh invitation it
+     * staged is the one the re-armed loop is about to redeem — deleting it here would lose
+     * it silently, the very outcome the re-arm exists to prevent.
+     */
+    private unstageMembershipInvite;
     /**
      * The pending single-use membership invitation a closed-strand formation carried back
      * for `strandId`, or `undefined` when none is staged — the seam the strand bring-up

package/dist/cadre-node.js

@@ -27,7 +27,8 @@ import { StrandInstanceManager, liveStrandStatus } from './strand-instance-manag
 import { PeerJoinBackfill } from './peer-join-backfill.js';
 import { deriveCohortMembers } from './strand-cohort.js';
 import { selectControlCohortDials, DEFAULT_CONTROL_COHORT_RECONCILE_MS, DEFAULT_CONTROL_COHORT_TARGET_DEGREE } from './control-cohort.js';
-import { dialPeerAddrs, DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS, DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS } from './peer-dial.js';
+import { dialPeerAddrs, CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS } from './peer-dial.js';
+import { connectionManagerTimeouts, peerJoinPushBudget, relayReservationBudgetMs, relayedDialBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
 import { EnrollmentService } from './enrollment.js';
 import { HibernationManager } from './hibernation-manager.js';
 import { ControlDatabase, isStrandIdConflict } from './control-database.js';
@@ -423,8 +424,10 @@ export class CadreNode {
          * (`strand-membership-reconciler.ts`, wired via `StartStrandConfig.pendingMembershipInvite`
          * in {@link launchStrand}): it deletes the entry once the invitation is redeemed, burned
          * against an already-seated member, or found dead (expired / cancelled / consumed
-         * elsewhere / the strand sealed). Until the strand is actually launched here the entry
-         * just waits.
+         * elsewhere / the strand sealed) — naming the invitation it settled, so a re-formation
+         * that replaced the entry meanwhile keeps its fresh one
+         * ({@link unstageMembershipInvite}). Until the strand is actually launched here the entry
+         * just waits; once launched, staging notifies the manager, which re-arms a finished loop.
          *
          * NOTE: entries live for the node's lifetime (one small pair per formed closed
          * strand), the same unbounded-keys tripwire {@link crossPartyStrandAddrs} documents —
@@ -762,6 +765,11 @@ export class CadreNode {
             // throws on a malformed `relayAddrs` entry, and that failure belongs on the same
             // logged-and-cleaned-up path as every other one in start().
             this.warnIfAnnounceAddrsDiscardRelay();
+            // Refuse a bad `linkRoundTripMs` HERE rather than wherever a budget is first derived
+            // from it: every consumer of it is conditional (no control storage, no relays, no dial
+            // yet), so a node with a zero or NaN declaration would otherwise boot and fail later
+            // inside a best-effort path that logs and carries on. `link-budget.ts`.
+            resolveLinkRoundTripMs(this.config.network?.linkRoundTripMs);
             // Install the WebRTC TURN-relay tracker BEFORE any libp2p bring-up, so it
             // wraps globalThis.RTCPeerConnection before the control node can create one.
             // Inert on Node.js (no RTCPeerConnection); disposed in cleanup().
@@ -1267,6 +1275,11 @@ export class CadreNode {
             // before looking anywhere else; if catch-up pushes start showing up as
             // connection-churn noise, raise it.
             debounceMs: 250,
+            // Dial and response deadlines counted in link round trips, not fixed milliseconds — a
+            // relayed dial that cannot finish inside the budget fails identically on every retry.
+            // See `link-budget.ts`. Spread BEFORE the host's own config so an explicit
+            // `controlBackfill.dialTimeoutMs` still wins.
+            ...peerJoinPushBudget(this.config.network?.linkRoundTripMs),
             ...this.config.controlBackfill
         });
         this.controlBackfill.start();
@@ -1396,7 +1409,13 @@ export class CadreNode {
             // itself, so the unknown case is provably byte-for-byte the old behaviour.
             // The yardstick moves ALONE: `assumedClusterSize` stays pinned at 2, because a
             // party of phones cannot promise three quarters of its machines are awake.
-            clusterPolicy: controlClusterPolicy(this.declaredEnrolledMachines),
+            // With neither declared this is still the frozen constant by identity, which is the
+            // production path. A degenerate deadline is Optimystic's to refuse, in
+            // createControlNode below.
+            clusterPolicy: controlClusterPolicy({
+                enrolledMachines: this.declaredEnrolledMachines,
+                cohortQueryTimeoutMs: network?.cohortQueryTimeoutMs
+            }),
             arachnode: { enableRingZulu: profile === 'storage' },
             ...(identityKey && { privateKey: identityKey }),
             ...(network?.transports && { transports: network.transports }),
@@ -1406,6 +1425,10 @@ export class CadreNode {
             // whose event loop is saturated by pure-JS Noise crypto. A configured value
             // replaces the default whole (see DEFAULT_CONNECTION_MONITOR).
             connectionMonitor: network?.connectionMonitor ?? DEFAULT_CONNECTION_MONITOR,
+            // libp2p's own dial and inbound-upgrade limits, from the same declared link as every
+            // cadre dial budget. Left to db-p2p, both stay at 10 s, which cannot open a relayed
+            // connection above a 2.5 s round trip — and the listener's side of that fails silently.
+            connectionManager: connectionManagerTimeouts(network?.linkRoundTripMs),
             // `{ wsPort }` when a listen entry names WebSocket, otherwise `{}` — and always
             // `{}` when `network.transports` is set, since the embedder owns transport policy
             // then. Spread NEXT to `transports` because the two answer the same question.
@@ -2753,15 +2776,22 @@ export class CadreNode {
      * {@link SeedBootstrapService} this node builds — `applySeed`'s owner dials and
      * `dialInvite`.
      *
-     * See {@link DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS} for why a peer's dial is
-     * bounded as a whole, and {@link DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS}
-     * for why each address is bounded as well.
+     * See `peer-dial.ts`'s `DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS` for why a peer's dial is
+     * bounded as a whole, and `DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS` for why each
+     * address is bounded as well.
+     *
+     * Both are DERIVED from `network.linkRoundTripMs` rather than fixed, because the slowest
+     * address either has to cover is a relayed dial and that costs a fixed number of exchanges —
+     * `link-budget.ts` has the counts. A host's explicit `controlCohort` values still win, so a
+     * test that drives dead addresses on purpose keeps the duration it chose.
      */
     controlDialBudget() {
         const cohort = this.config.network?.controlCohort;
+        const perAddressMs = cohort?.perAddressDialTimeoutMs
+            ?? relayedDialBudgetMs(this.config.network?.linkRoundTripMs);
         return {
-            perAddressMs: cohort?.perAddressDialTimeoutMs ?? DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS,
-            totalMs: cohort?.dialTimeoutMs ?? DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS,
+            perAddressMs,
+            totalMs: cohort?.dialTimeoutMs ?? CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS * perAddressMs,
         };
     }
     /**
@@ -4906,6 +4936,7 @@ export class CadreNode {
             revocationEnforcement: this.config.strandRevocationEnforcement,
             membershipReconciliation: this.config.strandMembershipReconciliation,
             onSelfRevoked: (revokedStrandId) => this.emit('strand:revoked', { strandId: revokedStrandId }),
+            onRejoinBlocked: (blockedStrandId) => this.emit('strand:rejoin-blocked', { strandId: blockedStrandId }),
             // The joiner's first-sync write gate (strand-first-sync-gate.ts): a launch that
             // comes up `'syncing'` announces the moment its database is published.
             firstSync: this.config.strandFirstSync,
@@ -4919,7 +4950,7 @@ export class CadreNode {
             // cleared once spent, burned, or dead — see pendingMembershipInvites.
             pendingMembershipInvite: {
                 get: () => this.pendingMembershipInvites.get(strand.Id),
-                clear: () => { this.pendingMembershipInvites.delete(strand.Id); }
+                clear: (settled) => this.unstageMembershipInvite(strand.Id, settled)
             },
             // The RESOLVED flag, never the raw argument — see the doc comment above.
             founder: resolvedFounder,
@@ -5147,8 +5178,10 @@ export class CadreNode {
      * caller runs the drive regardless).
      *
      * NOTE: against a relay that is DOWN this hook costs up to two strand-addr
-     * timeouts (10 s each: dial by peer id, then by addr) before the 10 s drive even
-     * starts, so one failed re-drive can hold the supervisor `driving` for ~30 s.
+     * timeouts (10 s each: dial by peer id, then by addr) before the reservation drive even
+     * starts, so one failed re-drive holds the supervisor `driving` for those 20 s plus the
+     * drive's own deadline — 34 s at the default declared link round trip, and longer on a host
+     * that declared a slower one (`link-budget.ts`).
      * Bounded and harmless while the relay is unreachable anyway; if recovery
      * latency after a relay comes back ever matters, skip the announce when the
      * control node holds no connection to the relay (the drive's own dial fails
@@ -5646,13 +5679,15 @@ export class CadreNode {
      * local), and only then does it reach out. Everything above in {@link start}
      * has completed by the time this runs.
      *
-     * BUDGET: the supervisor's first attempt — `relay-reservation.ts`'s
-     * `DEFAULT_RELAY_RESERVE_TIMEOUT_MS`, 10 s — is what `start()` waits on — deliberately the module default rather
-     * than a boot-specific one. A healthy dial-plus-reserve is sub-second even over
-     * a WAN, so 10 s is slack for a slow link; going much longer would make a dead
-     * relay indistinguishable from a hung start, and much shorter would fail nodes
-     * on links that were merely slow. The retries carry on in the background after
-     * this resolves, exactly as they do for a {@link reserveRelays} caller.
+     * BUDGET: the supervisor's first attempt is what `start()` waits on — deliberately the drive's
+     * ordinary deadline rather than a boot-specific one. That deadline is now COUNTED, four link
+     * round trips at the declared `network.linkRoundTripMs` (14 s at its default, where it was a
+     * fixed 10 s): a healthy dial-plus-reserve is sub-second even over a WAN, so this is slack for
+     * a slow link, while going much longer would make a dead relay indistinguishable from a hung
+     * start and much shorter would fail nodes on links that were merely slow. A host that declares
+     * a slower link lengthens it without touching this path — `link-budget.ts`. The retries carry
+     * on in the background after this resolves, exactly as they do for a {@link reserveRelays}
+     * caller.
      *
      * `network.requireRelay === false` softens only the outcome below: a first
      * attempt that lands no `/p2p-circuit` address is logged instead of thrown, and
@@ -5714,7 +5749,13 @@ export class CadreNode {
             this.relayReserveError = 'control node unavailable';
             return this.getRelayReservationState();
         }
-        const supervisor = superviseRelayReservation(this.controlNode, addrs, opts);
+        // The drive's deadline is counted in link round trips (`link-budget.ts`), so a host that
+        // declared a slower link gets a longer drive without naming a number here. A caller's own
+        // `timeoutMs` still wins.
+        const supervisor = superviseRelayReservation(this.controlNode, addrs, {
+            timeoutMs: relayReservationBudgetMs(this.config.network?.linkRoundTripMs),
+            ...opts
+        });
         this.relayReserveSupervisor = supervisor;
         await supervisor.firstAttempt;
         // Read through `this`, not `supervisor`: an overlapping call may have replaced
@@ -6558,7 +6599,11 @@ export class CadreNode {
      *    an intact control DB) and mints + persists a fresh one otherwise. The invitation
      *    will admit THIS key's public half as the `Strand.Member`.
      * 2. Stage the invitation in {@link pendingMembershipInvites} for the strand
-     *    bring-up flow (`strand-node-binds-member-peer`) to redeem via `consumeInvite`.
+     *    bring-up flow (`strand-node-binds-member-peer`) to redeem via `consumeInvite`,
+     *    and tell the instance manager, which re-arms an already-finished membership
+     *    reconciler so a RE-formation against a launched strand is attempted at once — the
+     *    removed-party case, where the loop finished long before the removal. A strand not
+     *    launched here yet has no loop to re-arm, and its bring-up finds the entry.
      *
      * Throws — failing the whole {@link formStrand} — when the identity cannot be
      * persisted: a joiner "joined" without a persistable identity could never become a
@@ -6577,16 +6622,23 @@ export class CadreNode {
                 'cannot become a member without it. Fix the underlying cause, then redeem a fresh ' +
                 'invitation (the delivered one dies with this error).', { cause: error });
         }
-        // NOTE: staging alone is not enough on a node whose reconciler for this strand has
-        // already finished. `StrandMembershipReconciler` latches a terminal stopped state on
-        // its `done` path and is rebuilt only by a strand relaunch, so a RE-formation against
-        // an already-launched strand stages an invitation no loop will ever look at. Harmless
-        // for a party that is still a member (the invitation would only be burned), and the
-        // live defect for one that has been REMOVED — which is exactly when an app re-forms.
-        // Tracked as `backlog/bug-removed-party-cannot-redeem-its-way-back`; pinned by
-        // `strand-party-removal-via-formation-e2e.integration.ts` (test 2).
         this.pendingMembershipInvites.set(strandId, invite);
         log('formStrand: staged membership invitation for strand %s (party key persisted)', strandId);
+        this.strandManager.notifyMembershipInviteStaged(strandId);
+    }
+    /**
+     * The reconciler's half of the {@link pendingMembershipInvites} seam: drop `settled`
+     * (spent, burned, or dead) only while it is still the staged entry. A re-formation
+     * replaces the entry between a pass's read and its settle, and the fresh invitation it
+     * staged is the one the re-armed loop is about to redeem — deleting it here would lose
+     * it silently, the very outcome the re-arm exists to prevent.
+     */
+    unstageMembershipInvite(strandId, settled) {
+        if (this.pendingMembershipInvites.get(strandId)?.inviteKey !== settled.inviteKey) {
+            log('strand %s: a fresh membership invitation replaced the one just settled — keeping it staged', strandId);
+            return;
+        }
+        this.pendingMembershipInvites.delete(strandId);
     }
     /**
      * The pending single-use membership invitation a closed-strand formation carried back