@serfab/cadre-core 1.4.0 → 1.5.0

package/src/link-budget.ts

@@ -0,0 +1,217 @@
+/**
+ * Dial and reservation budgets derived from ONE declared link round trip, instead of fixed
+ * millisecond numbers chosen on a fast local network.
+ *
+ * Reaching another machine through a relay costs a FIXED NUMBER OF EXCHANGES, not a fixed
+ * number of milliseconds. So every budget written as milliseconds has a link speed above which
+ * it can never open a connection at all — and the slower the link, the more of them fall over,
+ * silently, because an abandoned dial looks the same as a peer that is not there. Counting round
+ * trips instead is what stops that class of defect coming back: the next person to widen a
+ * budget changes ONE declaration, and the next person to add a dial writes down its round-trip
+ * count beside the arithmetic.
+ *
+ * ── The instrument ──
+ *
+ * Every count and every number below comes from
+ * `packages/integration-tests/src/scenarios/relayed-dial-cost-by-latency.integration.ts`, which
+ * is committed and opt-in (`RELAY_DIAL_COST=1`). **Its doc comment is the single home of the
+ * measurement**; this module holds only what the derivation needs. Re-run it before changing
+ * anything here.
+ *
+ * ── The unit ──
+ *
+ * ONE LINK ROUND TRIP is a message from one machine reaching the other and a reply coming
+ * back, over the path those two machines actually use — through the relay, when that is the
+ * only way they can reach each other. In the instrument, which injects a constant ONE-WAY
+ * frame delay `d`, one link round trip is `2d`.
+ *
+ * A leg between a node and the RELAY is half a link round trip, because it crosses one of the
+ * two hops rather than both. That is why the reservation drive's cost reads as "four
+ * node-to-relay round trips" in one telling and "two link round trips" in another: same
+ * quantity, different unit. This module uses the LINK round trip throughout, so there is one
+ * unit to reason in.
+ *
+ * ── The counts, and what they cost ──
+ *
+ * | operation                                            | link round trips | at the declared default |
+ * | ---------------------------------------------------- | ---------------- | ----------------------- |
+ * | open a relayed connection to another machine          | 4                | 8 000 ms                |
+ * | dial the relay itself                                 | 1                | 2 000 ms                |
+ * | request a reservation on an open relay connection     | 1                | 2 000 ms                |
+ * | negotiate a protocol over an established circuit      | 1                | 2 000 ms                |
+ *
+ * **Measured** 2026-09-26, one Windows machine, loopback dedicated relay, re-run for this
+ * change: a relayed dial took 20-25 ms at no delay, **7 255-7 279 ms at 900 ms one-way**, and
+ * **12 066-12 094 ms at 1 500 ms one-way**. Four link round trips of pure delay would be 7 200
+ * and 12 000, so the real cost sits about 60-95 ms ABOVE the arithmetic — the handshakes and a
+ * phone's pure-JS crypto, which no delay figure contains. That residue is small but it is why
+ * {@link DECLARED_LINK_ROUND_TRIP_MS} carries headroom over the band sereus supports rather
+ * than matching it exactly: a declaration equal to the measured round trip would derive a
+ * budget marginally BELOW the dial it has to contain.
+ *
+ * ── The ceiling this does NOT lift ──
+ *
+ * Above roughly 1 250 ms one-way — a 2.5-second link round trip — no relayed connection can be
+ * established by this stack whatever cadre declares here. Two budgets of 10 000 ms each inside
+ * libp2p bound the same dial and cannot be reached from sereus at all:
+ * `connectionManager.dialTimeout` (libp2p's own default, which `@optimystic/db-p2p`'s
+ * `libp2p-node-base.ts` neither sets nor exposes) and `connectionManager.inboundUpgradeTimeout`
+ * (which it sets to 10 000). Both appear in the same re-run: at 1 500 ms one-way the dial under
+ * libp2p's default failed with `The operation was aborted due to timeout`, and with the shipped
+ * inbound-upgrade budget the DIALER's own 12 094 ms dial resolved while the LISTENER had
+ * already thrown the half-built connection away — so the first stream over it died with
+ * `Unexpected EOF - stream closed while reading 0/1 bytes` and the listener reported no peer at
+ * all. That is why a too-slow link produces silence rather than an error.
+ *
+ * Two consequences for a reader choosing a declaration. Below that ceiling, the DEFAULT
+ * declaration of 2 000 ms does not reach it: four round trips of 2 000 ms is 8 000 ms, which
+ * covers a one-way delay up to 1 000 ms, so a deployment somewhere between 1 000 and 1 250 ms
+ * one-way has to declare its own (about 2 200-2 500) to use the band libp2p still allows. And
+ * declaring much ABOVE 2 500 buys nothing: the budgets here grow, and
+ * the connection still fails inside libp2p. Lifting that needs an upstream change and a
+ * decision about how slow a link sereus intends to carry —
+ * `tickets/blocked/how-slow-a-relayed-link-does-sereus-carry`. A reader who raised these
+ * numbers and still cannot connect at 1 500 ms one-way has met that ceiling, not this module.
+ */
+
+/**
+ * The link round trip cadre assumes when a host declares none, in milliseconds. A deployment
+ * that knows its own link states it with `NetworkConfig.linkRoundTripMs` instead.
+ *
+ * **Why 2 000.** It states the slowest link sereus already claims to support, and two other
+ * budgets fix that claim independently: `COHORT_READ_DEADLINE_MS`
+ * (`quereus-plugin-sereus/src/cluster-size.ts`, 5 000 ms) and
+ * `DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS` (`strand-first-sync-gate.ts`, 120 000 ms) were both
+ * sized against two parties reaching each other only through a public relay at a round trip
+ * near 1.8 s. Declaring 2 000 rounds that band up and leaves about 720 ms over the worst
+ * relayed dial measured at it (7 279 ms against a derived 8 000 ms).
+ *
+ * Raising it is how a deployment that KNOWS its link is slower moves every derived budget at
+ * once. The cost is the ordinary cost of a longer deadline: a peer that is genuinely gone holds
+ * the operation for that much longer before it is abandoned and retried. Above about 2 500 it
+ * buys nothing — see the module doc's ceiling paragraph.
+ */
+export const DECLARED_LINK_ROUND_TRIP_MS = 2000;
+
+/**
+ * Link round trips one relayed connection setup costs: transport, encryption and multiplexer
+ * handshakes to the relay, the circuit open, then both handshakes again end to end through it.
+ * Measured at 8 one-way delays, which is 4 link round trips.
+ */
+export const RELAYED_DIAL_ROUND_TRIPS = 4;
+
+/**
+ * Link round trips one whole reservation drive is budgeted for
+ * (`relay-reservation.ts`'s `driveRelayReservation`, whose ONE deadline covers the dial, the
+ * reservation request and the wait together).
+ *
+ * The protocol work measured is 2 — dial the relay (1) plus request the reservation (1). It is
+ * budgeted at 4, the relayed-dial count, because the wait afterwards may instead be satisfied
+ * by libp2p's own relay discovery, which repeats both legs after identify has run; 4 is
+ * therefore the largest thing this one deadline can be asked to contain, and the poll interval
+ * (`DEFAULT_RELAY_RESERVE_POLL_MS`) rounds up on top of it.
+ */
+export const RELAY_RESERVATION_ROUND_TRIPS = 4;
+
+/**
+ * Link round trips one request-and-answer over an ALREADY-OPEN circuit costs: the protocol
+ * negotiation (1, measured at 2 one-way delays) plus the request and its response (1). It does
+ * not include a dial — a caller that may have to open the connection budgets
+ * {@link RELAYED_DIAL_ROUND_TRIPS} separately, which is exactly what the two-field shape of
+ * Optimystic's `dialTimeoutMs` / `responseTimeoutMs` pair is for.
+ */
+export const CIRCUIT_REQUEST_ROUND_TRIPS = 2;
+
+/**
+ * What a block-transfer push message's BYTES get to cross, on top of the latency
+ * {@link CIRCUIT_REQUEST_ROUND_TRIPS} covers, in milliseconds.
+ *
+ * This part is bandwidth, not latency, so it does not scale with the declared round trip: a
+ * chunk is capped at `PeerJoinBackfillConfig.maxChunkBytes` (1 MiB by default) and crossing it
+ * costs whatever the link's throughput costs.
+ *
+ * NOTE: 6 000 ms is not a measurement of any throughput — it is the residue of the 10 000 ms
+ * this deadline shipped as before it was split into a latency part and a transfer part, chosen
+ * so that at the default declaration the derived value is the same 10 000 and no fast link got
+ * slower. Nobody has measured how long 1 MiB takes to cross a relayed mobile link. If pushes
+ * ever time out with the transfer only part done, measure that and raise THIS, not the declared
+ * round trip.
+ */
+export const PUSH_TRANSFER_ALLOWANCE_MS = 6000;
+
+/**
+ * Resolve a host's declared link round trip, falling back to
+ * {@link DECLARED_LINK_ROUND_TRIP_MS}.
+ *
+ * Validated here rather than downstream because nothing downstream would: every consumer
+ * multiplies this value into a `setTimeout` deadline, where a zero silently turns a budget into
+ * "give up immediately" and a `NaN` turns it into "never time out".
+ *
+ * Both node bring-up paths call this EAGERLY — `CadreNode.start()`, and
+ * `StrandInstanceManager.buildStrandRuntime` behind `addStrand`/`resumeStrand` — so a bad
+ * declaration fails the same start that Optimystic's own check on `cohortQueryTimeoutMs` fails.
+ * The eager call is what makes that true: every budget derived here is behind a condition (a
+ * node with no control storage builds no catch-up, a node with no relay addrs drives no
+ * reservation), so waiting for a first consumer would let a broken declaration boot and then
+ * throw inside a best-effort path that logs and carries on.
+ */
+export function resolveLinkRoundTripMs(linkRoundTripMs?: number): number {
+	if (linkRoundTripMs === undefined) {
+		return DECLARED_LINK_ROUND_TRIP_MS;
+	}
+	if (!Number.isFinite(linkRoundTripMs) || linkRoundTripMs <= 0) {
+		throw new Error(
+			`network.linkRoundTripMs must be a finite number of milliseconds above zero, not ${String(linkRoundTripMs)}`
+		);
+	}
+	return linkRoundTripMs;
+}
+
+/**
+ * Deadline for OPENING a connection to another machine that may only be reachable through a
+ * relay: {@link RELAYED_DIAL_ROUND_TRIPS} at the declared link round trip.
+ */
+export function relayedDialBudgetMs(linkRoundTripMs?: number): number {
+	return RELAYED_DIAL_ROUND_TRIPS * resolveLinkRoundTripMs(linkRoundTripMs);
+}
+
+/**
+ * Deadline for one whole relay reservation drive: {@link RELAY_RESERVATION_ROUND_TRIPS} at the
+ * declared link round trip.
+ */
+export function relayReservationBudgetMs(linkRoundTripMs?: number): number {
+	return RELAY_RESERVATION_ROUND_TRIPS * resolveLinkRoundTripMs(linkRoundTripMs);
+}
+
+/**
+ * Deadline for one request and its answer over a circuit that is already open —
+ * {@link CIRCUIT_REQUEST_ROUND_TRIPS} at the declared link round trip, plus `transferAllowanceMs`
+ * for the payload's own bytes.
+ */
+export function circuitRequestBudgetMs(
+	linkRoundTripMs?: number,
+	transferAllowanceMs = PUSH_TRANSFER_ALLOWANCE_MS
+): number {
+	return CIRCUIT_REQUEST_ROUND_TRIPS * resolveLinkRoundTripMs(linkRoundTripMs) + transferAllowanceMs;
+}
+
+/** The two deadlines one peer-join catch-up push needs, both derived from the declared link. */
+export interface PeerJoinPushBudget {
+	/** Opening the connection to the peer being caught up — a relayed dial, in the worst case. */
+	dialTimeoutMs: number;
+	/** The push request and its answer over that connection, including the chunk's own bytes. */
+	responseTimeoutMs: number;
+}
+
+/**
+ * The dial and response deadlines a peer-join catch-up push needs at a declared link. One
+ * helper rather than two call-site expressions, because the two construction sites
+ * (`CadreNode`'s control catch-up and `StrandInstanceManager`'s per-strand one) and
+ * `DEFAULT_PEER_JOIN_BACKFILL` must not drift apart.
+ */
+export function peerJoinPushBudget(linkRoundTripMs?: number): PeerJoinPushBudget {
+	return {
+		dialTimeoutMs: relayedDialBudgetMs(linkRoundTripMs),
+		responseTimeoutMs: circuitRequestBudgetMs(linkRoundTripMs)
+	};
+}

package/src/peer-dial.ts

@@ -27,32 +27,47 @@ import debug from 'debug';
 import type { Connection } from '@libp2p/interface';
 import type { Multiaddr } from '@multiformats/multiaddr';
 import { withDeadline } from './control-stream.js';
+import { relayedDialBudgetMs } from './link-budget.js';
 
 const log = debug('sereus:cadre:peer-dial');
 
 /**
- * Default limit on ONE address's dial attempt, in ms.
+ * Default limit on ONE address's dial attempt, in ms, at the DEFAULT declared link round trip:
+ * `RELAYED_DIAL_ROUND_TRIPS` (4) x `DECLARED_LINK_ROUND_TRIP_MS` (2000 ms) = 8000 ms, which is
+ * what this constant was already set to as a fixed number.
  *
  * It has to cover the slowest address that should succeed, because an attempt
  * that needs longer fails the same way on every retry. That is a relayed dial on
  * a mobile link: connect to the relay (transport, encryption and multiplexer
  * handshakes, when no relay connection is open yet), open the circuit, then run
- * both handshakes again end to end through the relay at roughly twice the
- * round-trip time. At a few hundred ms per round trip that is several seconds.
- * A direct dial that works is far quicker — a phone's WebSocket dial to a node
- * forwarded over `adb reverse` measured 1.6 s end to end. 8 s covers the relayed
- * case with some margin while still abandoning a silently dropped address
- * sooner than libp2p's own 10 s dial timeout would.
+ * both handshakes again end to end through the relay. `link-budget.ts` is where that
+ * reasoning is now a measured count instead of an estimate — four link round trips, measured at
+ * 7 279 ms on a link whose round trip is 1.8 s — so the value MOVES with a host's
+ * `NetworkConfig.linkRoundTripMs` rather than pinning the band this node can reach to whatever
+ * number was typed here. A direct dial that works is far quicker — a phone's WebSocket dial to
+ * a node forwarded over `adb reverse` measured 1.6 s end to end.
+ *
+ * Deriving it rather than leaving it fixed is deliberate: at 8000 ms fixed, the per-address
+ * limit — not libp2p's own 10 s dial timeout — was the first thing to cut a relayed dial off,
+ * from about 1000 ms of one-way link delay upward. Nothing said so, and a reader raising
+ * `DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS` below would not have found it.
  *
  * NOTE: a `/p2p-circuit/webrtc` address also negotiates a WebRTC session after
  * the relay leg. If such dials are ever seen failing at exactly this limit, raise
- * it (or `network.controlCohort.perAddressDialTimeoutMs` on the affected nodes)
- * rather than the per-peer limit, which cannot help an address that is itself
+ * the declared link round trip (or `network.controlCohort.perAddressDialTimeoutMs` on the
+ * affected nodes) rather than the per-peer limit, which cannot help an address that is itself
  * cut off.
  *
  * Override per node with `network.controlCohort.perAddressDialTimeoutMs`.
  */
-export const DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS = 8_000;
+export const DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS = relayedDialBudgetMs();
+
+/**
+ * How many whole per-address dial attempts {@link DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS}
+ * budgets for: the address that works plus the three dead ones a lent cadre-host node is
+ * observed to report ahead of it (loopback and LAN WebSocket addresses, plus virtual adapters).
+ */
+export const CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS = 4;
 
 /**
  * Default limit on dialing ONE peer — every candidate address together — in ms.
@@ -66,18 +81,25 @@ export const DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS = 8_000;
  *
  * Sized to fit a typical list with dead addresses ahead of the one that works:
  * a lent cadre-host node reports two to four addresses a phone can dial
- * (loopback and LAN WebSocket addresses, plus any virtual adapters). Three
- * silently dropped addresses cost 3 × 8 s = 24 s, leaving 6 s for the working
- * address after them — well over the 1.6 s the phone measured. A peer whose
- * working address sits behind four or more dropped ones does not fit, and fails
- * each pass the same way; if that shows up, the remedy is ordering (try the
- * address that last connected first), not a larger number here.
+ * (loopback and LAN WebSocket addresses, plus any virtual adapters). So it is
+ * {@link CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS} whole per-address budgets — three silently
+ * dropped addresses, then the one that works, each getting the FULL per-address limit. A peer
+ * whose working address sits behind four or more dropped ones does not fit, and fails each pass
+ * the same way; if that shows up, the remedy is ordering (try the address that last connected
+ * first), not a larger number here.
+ *
+ * Counted as whole per-address budgets rather than the flat 30_000 it was before, because the
+ * old number left only 6 s after three dropped addresses — enough for the 1.6 s direct dial the
+ * phone measured, but NOT for the relayed dial the per-address limit is sized to cover. A peer
+ * whose only working address was a relayed one behind two dead ones therefore could not be
+ * dialled at all, and the arithmetic in this comment was what said otherwise.
  *
  * Override per node with `network.controlCohort.dialTimeoutMs` — tests that
  * drive dead addresses on purpose set it low so a pass's duration is a chosen
  * number rather than a transitive libp2p default stretched by machine load.
  */
-export const DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS = 30_000;
+export const DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS =
+	CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS * DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS;
 
 /** Time limits for dialing one peer from a list of candidate addresses. */
 export interface PeerDialBudget {

package/src/peer-join-backfill.ts

@@ -2,6 +2,7 @@ import debug from 'debug';
 import type { Connection, Libp2p, PeerId } from '@libp2p/interface';
 import type { ActionId, IPeerNetwork } from '@optimystic/db-core';
 import { BlockTransferClient, type BlockCommitProof, type IRawStorage } from '@optimystic/db-p2p';
+import { peerJoinPushBudget } from './link-budget.js';
 
 // Peer-join whole-store block catch-up, shared by the STRAND networks and the CONTROL
 // network. On `connection:open` (debounced) it pushes every committed, materialized block
@@ -43,19 +44,74 @@ import { BlockTransferClient, type BlockCommitProof, type IRawStorage } from '@o
 //   `scheduleConnectedPeers()` on a membership change (the production join order is
 //   connect-then-authorize, so the denial at dial time is the expected first pass).
 //
-// NOTE: a failing dial to a non-speaking peer costs one dial timeout per connection:open
-// from such a peer, and is never memoized (only clean runs are). Bounded today by the
-// debounce plus the unreachable-peer bail in `runCatchUp` — one dial per event, not one
-// per chunk. If a node ever holds many such connections, or the enumeration ahead of the
-// first chunk gets expensive, pre-check `libp2p.peerStore` for this network's
-// block-transfer protocol before enumerating.
+// Both per-push deadlines are DERIVED from the declared link round trip
+// (`link-budget.ts`), not fixed milliseconds. That is not tuning — it is what makes this
+// module work at all over a relay. Opening a relayed connection costs a fixed number of
+// exchanges, so the 3000 ms this catch-up used to allow its dial could never finish one
+// above 375 ms of one-way link delay: the catch-up existed so that a machine which joined
+// after blocks were committed physically ends up holding them, and through a relay on any
+// link slow enough to matter it had never once managed to. The reproduction is
+// `packages/integration-tests/src/scenarios/relayed-dial-cost-by-latency.integration.ts`.
 //
-// NOTE: a run that fails is retried only when that peer next opens a connection (or, with
-// an `authorizePeer` gate wired, when `scheduleConnectedPeers()` is driven). Over a stable
-// connection a transient push failure therefore leaves the peer partially copied until the
-// next reconnect (read repair still covers reads meanwhile). If these meshes ever hold
-// long-lived connections where that matters, re-arm the debounce timer with a backoff on a
-// non-clean run instead of waiting for connection:open.
+// It does NOT work at every speed. Above roughly 1250 ms one-way, two libp2p budgets that
+// sereus cannot reach abandon the connection before any deadline here is consulted, and the
+// listener's one makes that failure look like an absent peer rather than a timeout — see
+// `link-budget.ts` ("The ceiling this does NOT lift") and
+// `tickets/blocked/how-slow-a-relayed-link-does-sereus-carry`.
+//
+// RETRY, and why it backs off. A run whose PUSH FAILED — the transport threw, which is what
+// a dial or response deadline expiring looks like here — re-arms on a doubling backoff
+// (`retryBackoffMs` to `maxRetryBackoffMs`), and a `connection:open` arriving while that wait
+// is outstanding is DROPPED rather than collapsing it back to the debounce. Both halves are
+// load-bearing:
+//
+// - Without the re-arm, a transient push failure over a stable connection left the peer
+//   partially copied until its next reconnect (read repair still covered reads meanwhile).
+// - Without dropping churn inside the wait, a peer that cannot be reached at all is
+//   re-dialled on every `connection:open` forever. That is not hypothetical: the 2026-09-26
+//   relayed reproduction shows one peer re-opening a connection about every 14.5 s for a
+//   200-second run, each event starting a catch-up whose dial could not possibly finish.
+//   The connection kept re-appearing because Optimystic's own block-transfer push path
+//   budgets its dial at `transferTimeoutMs ?? 30000` — the one dial budget in the stack
+//   above the measured relayed setup cost — so it succeeded where this one could not.
+//
+// A NON-CLEAN run is NOT on its own enough to re-arm, and the distinction is what keeps the
+// backoff from becoming a worse problem than the one it fixes. `clean` also goes false for
+// outcomes no amount of retrying can change: the membership gate DENIED the peer; this
+// network's raw storage implements no `listBlockIds`, so the catch-up is inert; or the
+// receiver reported blocks in `missing`, which it does per block for a payload it cannot
+// parse and for a revision whose retained commit proof this node does not hold (a push
+// carrying no proof is refused outright by a receiver running the default
+// `requirePushCertificate: true`, and an unretained proof is the ordinary case — see
+// {@link Chunk.proofs}). Re-arming on those would re-push the WHOLE store to that peer every
+// `maxRetryBackoffMs` for as long as the node runs, and report the failure below as a link
+// budget problem when nothing about the link is wrong. All three still leave the peer
+// un-memoized, so its next `connection:open` retries — the behaviour that predates the
+// backoff, and the right one for a verdict rather than a timeout. Denial in particular is
+// re-driven on purpose by `scheduleConnectedPeers()` the moment the membership commit lands,
+// because the control network's join order is connect-then-authorize.
+//
+// A re-arm is also skipped when the peer is no longer CONNECTED. This module's trigger is
+// `connection:open`, so a peer that went away already has one: re-arming instead would leave
+// a machine dialing a peer it cannot see once a minute for the rest of its uptime.
+//
+// After `PEER_JOIN_BACKFILL_WARN_AFTER_FAILURES` consecutive failures one `console.warn`
+// names the peer and the budget, because otherwise a machine on a too-slow link says
+// nothing at all: the connection simply never appears and only a DEBUG log mentions a dial
+// timeout.
+//
+// NOTE: a failing dial to a non-speaking peer costs one dial timeout per attempt, and is
+// never memoized (only clean runs are). Bounded by the backoff above plus the
+// unreachable-peer bail in `runCatchUp` — one dial per attempt, not one per chunk. If a node
+// ever holds many such connections, or the enumeration ahead of the first chunk gets
+// expensive, pre-check `libp2p.peerStore` for this network's block-transfer protocol before
+// enumerating.
+//
+// NOTE: a peer that disconnects INSIDE a backoff wait still costs the one attempt that wait
+// was already armed for — the connectivity check is made when the re-arm is decided, not when
+// the timer fires, and this module subscribes to `connection:open` only. One dial, not a
+// recurring one; if a node ever holds many transient peers, subscribe to `connection:close`
+// and clear the pending timer there.
 
 const log = debug('sereus:cadre:peer-join-backfill');
 
@@ -67,6 +123,14 @@ const log = debug('sereus:cadre:peer-join-backfill');
  */
 export const MAX_BLOCK_MESSAGE_BYTES = 8 * 1024 * 1024;
 
+/**
+ * Consecutive failed catch-up runs against ONE peer before the module says so on
+ * `console.warn` rather than only in its DEBUG log. At the default backoff that is reached
+ * about 35 seconds in (5 s + 10 s + 20 s), which is late enough to have ruled out a blip and
+ * early enough to be the first thing an operator sees about a machine that is not catching up.
+ */
+export const PEER_JOIN_BACKFILL_WARN_AFTER_FAILURES = 3;
+
 /** Tuning for the per-peer catch-up. Every field optional; defaults in {@link DEFAULT_PEER_JOIN_BACKFILL}. */
 export interface PeerJoinBackfillConfig {
   /** Default true. False disables the catch-up entirely (the pre-existing behaviour). */
@@ -79,10 +143,28 @@ export interface PeerJoinBackfillConfig {
   maxChunkBytes?: number;
   /** Max blocks per push message. Default 64. */
   maxChunkBlocks?: number;
-  /** Per-push dial deadline, ms. Default 3000 (matches SpreadOnChurnMonitor). */
+  /**
+   * Per-push dial deadline, ms. Default {@link peerJoinPushBudget}'s `dialTimeoutMs` — four link
+   * round trips at the declared link, 8000 ms as shipped. NOT a fixed number: a relayed dial
+   * costs a fixed number of exchanges, so a host on a slower link moves this (and every other
+   * cadre dial budget) by declaring `NetworkConfig.linkRoundTripMs`. Naming it here still wins
+   * over the derived value — `link-budget.ts` has the counts and the measurement.
+   */
   dialTimeoutMs?: number;
-  /** Per-push response deadline, ms. Default 10_000 (matches SpreadOnChurnMonitor). */
+  /**
+   * Per-push response deadline, ms. Default {@link peerJoinPushBudget}'s `responseTimeoutMs` —
+   * two link round trips at the declared link plus a transfer allowance for the chunk's own
+   * bytes, 10_000 ms as shipped. Derived differently from {@link dialTimeoutMs} because it
+   * bounds a data transfer over a connection that is already open, not a dial.
+   */
   responseTimeoutMs?: number;
+  /**
+   * First wait before re-running a catch-up whose last run did not complete cleanly, ms.
+   * Default 5000. Doubles per consecutive failure up to {@link maxRetryBackoffMs}.
+   */
+  retryBackoffMs?: number;
+  /** Ceiling on that doubling wait, ms. Default 60_000. */
+  maxRetryBackoffMs?: number;
 }
 
 /** The resolved defaults every {@link PeerJoinBackfill} starts from. */
@@ -92,8 +174,13 @@ export const DEFAULT_PEER_JOIN_BACKFILL: Required<PeerJoinBackfillConfig> = {
   maxBlocks: 10_000,
   maxChunkBytes: 1024 * 1024,
   maxChunkBlocks: 64,
-  dialTimeoutMs: 3000,
-  responseTimeoutMs: 10_000
+  // Derived at the DEFAULT declared link, for a `PeerJoinBackfill` built without a host's
+  // declaration (this module's own tests, an embedder driving it directly). The two production
+  // construction sites pass `peerJoinPushBudget(network?.linkRoundTripMs)` so a host that
+  // declares a slower link moves both — see `link-budget.ts`.
+  ...peerJoinPushBudget(),
+  retryBackoffMs: 5000,
+  maxRetryBackoffMs: 60_000
 };
 
 /**
@@ -209,8 +296,18 @@ export class PeerJoinBackfill {
    * reconnect, which is exactly what the re-arm exists to avoid.
    */
   private readonly rearmAfterFlight = new Set<string>();
-  /** Pending per-peer debounce timers, cleared on stop. */
+  /** Pending per-peer timers — the debounce, or a backoff re-arm. Cleared on stop. */
   private readonly timers = new Map<string, ReturnType<typeof setTimeout>>();
+  /** Consecutive non-clean runs per peer, cleared when one finishes cleanly. */
+  private readonly failures = new Map<string, number>();
+  /**
+   * Epoch ms before which a peer's catch-up must not be re-run, set alongside a backoff timer
+   * that will run it. Its presence is what makes `connection:open` churn free: a schedule
+   * arriving inside the wait is dropped, rather than collapsing the backoff to the debounce.
+   */
+  private readonly retryAfter = new Map<string, number>();
+  /** Peers already reported on `console.warn`; reset when a run finally lands cleanly. */
+  private readonly warned = new Set<string>();
   private readonly onConnectionOpen: (evt: CustomEvent<Connection>) => void;
   private loggedNoListBlockIds = false;
 
@@ -244,6 +341,7 @@ export class PeerJoinBackfill {
     }
     this.timers.clear();
     this.rearmAfterFlight.clear();
+    this.retryAfter.clear();
     log('[%s] stopped', this.deps.label);
   }
 
@@ -279,6 +377,12 @@ export class PeerJoinBackfill {
       this.rearmAfterFlight.add(key);
       return;
     }
+    // A peer whose last run failed is already re-armed on a backoff timer that will run it, so
+    // DROP this schedule instead of shortening the wait — otherwise a peer that cannot be
+    // reached is re-dialled on every connection:open forever (see the module comment's retry
+    // paragraph). A denied run never gets here: it does not set a backoff.
+    const retryAt = this.retryAfter.get(key);
+    if (retryAt !== undefined && Date.now() < retryAt) return;
     const existing = this.timers.get(key);
     if (existing) clearTimeout(existing);
     this.timers.set(key, setTimeout(() => {
@@ -299,7 +403,7 @@ export class PeerJoinBackfill {
     }
     this.inFlight.add(key);
     try {
-      const { result, clean } = await this.runCatchUp(peerId);
+      const { result, clean, pushFailed } = await this.runCatchUp(peerId);
       // NOTE: a run that hit `maxBlocks` is still "clean" and still memoizes the peer, so
       // the tail past the ceiling never reaches it. Deliberate: enumeration is not
       // resumable, so not memoizing would re-push the same prefix on every reconnect
@@ -307,6 +411,18 @@ export class PeerJoinBackfill {
       // realistically exceed maxBlocks, the fix is a resumable cursor, not either policy.
       if (clean && !this.stopped) {
         this.done.add(key);
+        this.failures.delete(key);
+        this.retryAfter.delete(key);
+        this.warned.delete(key);
+      } else if (pushFailed && !this.stopped && this.started && this.isConnected(peerId)) {
+        // `pushFailed`, not `!clean`: a denial, an inert store and a receiver's per-block
+        // rejection are all verdicts a retry cannot change, and re-arming on them would
+        // re-push the whole store to that peer forever. See the module comment.
+        //
+        // Gated on `started` too: the re-arm exists to REPLACE a connection:open-driven retry,
+        // so a caller driving `catchUpPeer` by hand against a backfill that was never started
+        // owns its own retry policy and must not be left holding a background timer.
+        this.scheduleRetryWithBackoff(peerId);
       }
       log('[%s] catch-up peer=%s offered=%d accepted=%d rejected=%d uncommitted=%d unmaterialized=%d capped=%d oversized=%d denied=%s done=%s',
         this.deps.label, key, result.offered, result.accepted, result.rejected.length,
@@ -327,8 +443,61 @@ export class PeerJoinBackfill {
     }
   }
 
-  /** The actual copy. `clean` = every chunk pushed and the remote persisted every block. */
-  private async runCatchUp(peerId: PeerId): Promise<{ result: PeerJoinBackfillResult; clean: boolean }> {
+  /** Whether this network's libp2p node still holds a connection to that peer. */
+  private isConnected(peerId: PeerId): boolean {
+    return this.deps.libp2p.getConnections(peerId).length > 0;
+  }
+
+  /**
+   * Re-arm one peer's catch-up after a run whose push failed, on a wait that doubles
+   * per consecutive failure up to `maxRetryBackoffMs`. Sets {@link retryAfter} alongside the
+   * timer, which is what makes `connection:open` churn inside the wait free.
+   */
+  private scheduleRetryWithBackoff(peerId: PeerId): void {
+    const key = peerId.toString();
+    const failures = (this.failures.get(key) ?? 0) + 1;
+    this.failures.set(key, failures);
+    const delayMs = Math.min(
+      this.config.retryBackoffMs * 2 ** (failures - 1),
+      this.config.maxRetryBackoffMs
+    );
+    this.retryAfter.set(key, Date.now() + delayMs);
+    const existing = this.timers.get(key);
+    if (existing) clearTimeout(existing);
+    this.timers.set(key, setTimeout(() => {
+      this.timers.delete(key);
+      this.retryAfter.delete(key);
+      void this.catchUpPeer(peerId);
+    }, delayMs));
+    log('[%s] catch-up peer=%s failed %d time(s) in a row; retrying in %dms',
+      this.deps.label, key, failures, delayMs);
+    this.warnPersistentFailure(key, failures);
+  }
+
+  /**
+   * Say ONCE per peer, outside the DEBUG log, that its catch-up is not landing. Without this a
+   * machine on a link too slow for a relayed dial produces no statement that anything is wrong:
+   * the peer never appears to hold the blocks, and the only trace is a DEBUG line naming a dial
+   * timeout. Reset when a run finally lands cleanly, so a peer that recovers can report again.
+   */
+  private warnPersistentFailure(key: string, failures: number): void {
+    if (failures < PEER_JOIN_BACKFILL_WARN_AFTER_FAILURES || this.warned.has(key)) return;
+    this.warned.add(key);
+    console.warn(
+      `[cadre:${this.deps.label}] peer-join block catch-up to peer ${key} has failed ${failures} times in a row `
+      + `(dial budget ${this.config.dialTimeoutMs}ms, response budget ${this.config.responseTimeoutMs}ms). `
+      + 'That peer may not be holding blocks committed before it joined. If it is reachable only through a relay, '
+      + 'these budgets are derived from network.linkRoundTripMs (see link-budget.ts); above about a 2.5-second '
+      + 'round trip no relayed connection can be established at all, whatever they are set to.'
+    );
+  }
+
+  /**
+   * The actual copy. `clean` = every chunk pushed and the remote persisted every block.
+   * `pushFailed` = at least one push THREW, which is the only non-clean outcome a retry can
+   * change; the others are verdicts (denied, inert store, blocks the receiver refused).
+   */
+  private async runCatchUp(peerId: PeerId): Promise<{ result: PeerJoinBackfillResult; clean: boolean; pushFailed: boolean }> {
     const result = emptyResult();
     const { storage } = this.deps;
 
@@ -345,7 +514,7 @@ export class PeerJoinBackfill {
       }
       if (!authorized) {
         result.denied = true;
-        return { result, clean: false };
+        return { result, clean: false, pushFailed: false };
       }
     }
 
@@ -354,7 +523,7 @@ export class PeerJoinBackfill {
         this.loggedNoListBlockIds = true;
         log('[%s] raw storage does not implement listBlockIds(); backfill is inert', this.deps.label);
       }
-      return { result, clean: false };
+      return { result, clean: false, pushFailed: false };
     }
 
     const client = this.createPushClient(peerId);
@@ -473,6 +642,6 @@ export class PeerJoinBackfill {
     }
 
     const clean = !chunkFailed && result.rejected.length === 0;
-    return { result, clean };
+    return { result, clean, pushFailed: chunkFailed };
   }
 }