@serfab/cadre-core 1.17.0 → 1.19.0

package/dist/peer-join-backfill.js

@@ -1,15 +1,22 @@
 import debug from 'debug';
+import { jsonEncodedBytes, MAX_BLOCK_BYTES } from '@optimystic/db-core';
 import { BlockTransferClient, buildBlockTransferProtocol } from '@optimystic/db-p2p';
 import { peerJoinPushBudget } from './link-budget.js';
 // Peer-join whole-store block catch-up, shared by the STRAND networks and the CONTROL
 // network. On a new peer's `peer:identify` result — libp2p's identify protocol runs once per
 // connection and reports the protocols the remote supports — naming this network's own
-// block-transfer protocol, it schedules a debounced push of every committed, materialized
-// block in the local raw store to that peer, so a machine that joined after blocks were
-// committed still ends up physically holding them. Optimystic has no cohort-join catch-up
-// of its own; a block committed while its writer was alone has exactly one holder forever
-// without this (named collection-header blocks are written once, at collection creation,
-// and their revision never moves again — so no later commit ever carries them anywhere).
+// block-transfer protocol, it schedules a push of every committed, materialized block in the
+// local raw store that the peer does not already hold (see "What is pushed" below), so a
+// machine that joined after blocks were committed still ends up physically holding them.
+// Optimystic has no cohort-join catch-up of its own; a block committed while its writer was
+// alone has exactly one holder forever without this (named collection-header blocks are
+// written once, at collection creation, and their revision never moves again — so no later
+// commit ever carries them anywhere).
+//
+// A peer this instance has never started a run for is run at once; later schedules for it wait
+// out `debounceMs`. The joining machine's first-sync gate waits for this push rather than
+// fetching the same blocks one read at a time, so on first contact a settle window only delays
+// the joiner. Identify churn during that first run is replayed once the run ends.
 //
 // A peer whose identify does not name the protocol — a bare circuit relay, a bootstrap node,
 // or (on the control network) a stranger the inbound gate admits without membership — is
@@ -20,7 +27,7 @@ import { peerJoinPushBudget } from './link-budget.js';
 // printing exactly that warning about their own relay, forever, on every reconnect. See
 // {@link speaksBlockTransfer}.
 //
-// NOTE: this copies the WHOLE local store to every newly connected peer, which is right
+// NOTE: this offers the WHOLE local store to every newly connected peer, which is right
 // while a network is one party's handful of machines (see docs/architecture.md →
 // Replication cluster size — the control network replicates to the whole party by intent,
 // and "On a strand, the machine count is not a party count"). If these meshes ever get
@@ -32,10 +39,11 @@ import { peerJoinPushBudget } from './link-budget.js';
 // than running two.
 //
 // NOTE: strand nodes pass db-p2p no `kvStore`, so Optimystic's under-replication ledger (the
-// copies a node owes peers) dies at every restart. This module's whole-store push on connect is
-// what delivers a write a machine made alone after it restarts (`joiner-first-commit-over-slow-relay`,
-// restart arm). If this module is narrowed (the cohort filter above) or deleted, give strand
-// nodes a persistent kvStore first.
+// copies a node owes peers) dies at every restart. This module's push on connect is what
+// delivers a write a machine made alone after it restarts (`joiner-first-commit-over-slow-relay`,
+// restart arm): the peer did not sign that commit, so the block is still offered to it. If this
+// module is narrowed (the cohort filter above) or deleted, give strand nodes a persistent
+// kvStore first.
 //
 // Membership — the one place the two networks differ, expressed as the optional
 // `authorizePeer` dep:
@@ -50,11 +58,12 @@ import { peerJoinPushBudget } from './link-budget.js';
 //   invitation, configured bootstrap/relay peers — see docs/architecture.md, the
 //   control-network inbound connection gate). Pushing the whole control store to such a
 //   peer would hand a stranger the party's entire membership, addresses and strand list.
-//   `CadreNode` passes `isAuthorizedMember`; the gate is consulted at PUSH time, not at
-//   schedule time, and fails closed on a thrown check. A denied run is not memoized, so
-//   the peer is retried on its next `peer:identify` (a reconnect re-runs identify) — or
-//   sooner, via `scheduleConnectedPeers()` on a membership change (the production join order is
-//   connect-then-authorize, so the denial at dial time is the expected first pass).
+//   `CadreNode` passes a lookup in its in-memory authorized-member snapshot; the gate is
+//   consulted at PUSH time, not at schedule time, and fails closed on a thrown check. A denied
+//   run is not memoized, so the peer is retried on its next `peer:identify` (a reconnect
+//   re-runs identify) — or sooner, via `scheduleConnectedPeers()` on a membership change (the
+//   production join order is connect-then-authorize, so the denial at dial time is the
+//   expected first pass).
 //
 // Both per-push deadlines are DERIVED from the declared link round trip
 // (`link-budget.ts`), not fixed milliseconds. That is not tuning — it is what makes this
@@ -89,21 +98,17 @@ import { peerJoinPushBudget } from './link-budget.js';
 //   budgets its dial at `transferTimeoutMs ?? 30000` — the one dial budget in the stack
 //   above the measured relayed setup cost — so it succeeded where this one could not.
 //
-// A NON-CLEAN run is NOT on its own enough to re-arm, and the distinction is what keeps the
-// backoff from becoming a worse problem than the one it fixes. `clean` also goes false for
-// outcomes no amount of retrying can change: the membership gate DENIED the peer; this
-// network's raw storage implements no `listBlockIds`, so the catch-up is inert; or the
-// receiver reported blocks in `missing`, which it does per block for a payload it cannot
-// parse and for a revision whose retained commit proof this node does not hold (a push
-// carrying no proof is refused outright by a receiver running the default
-// `requirePushCertificate: true`, and an unretained proof is the ordinary case — see
-// {@link Chunk.proofs}). Re-arming on those would re-push the WHOLE store to that peer every
-// `maxRetryBackoffMs` for as long as the node runs, and report the failure below as a link
-// budget problem when nothing about the link is wrong. All three still leave the peer
-// un-memoized, so its next `peer:identify` retries — the behaviour that predates the
-// backoff, and the right one for a verdict rather than a timeout. Denial in particular is
-// re-driven on purpose by `scheduleConnectedPeers()` the moment the membership commit lands,
-// because the control network's join order is connect-then-authorize.
+// A retry re-sends only the blocks the peer has not yet accepted (see "What is pushed"), so a
+// run that died part-way through the store resumes where it stopped.
+//
+// Only a failed push re-arms. Two other outcomes also leave a run short of catching the peer
+// up, and no amount of retrying changes them: the membership gate DENIED the peer, or this
+// network's raw storage implements no `listBlockIds`, so the catch-up is inert. Re-arming on
+// those would retry every `maxRetryBackoffMs` for as long as the node runs, and report the
+// failure below as a link budget problem when nothing about the link is wrong. Neither marks
+// the peer caught up, so its next `peer:identify` retries. Denial in particular is re-driven on
+// purpose by `scheduleConnectedPeers()` the moment the membership commit lands, because the
+// control network's join order is connect-then-authorize.
 //
 // A re-arm is also skipped when the peer is no longer CONNECTED. This module's trigger is
 // `peer:identify`, so a peer that went away already has one: re-arming instead would leave
@@ -114,6 +119,43 @@ import { peerJoinPushBudget } from './link-budget.js';
 // nothing at all: the connection simply never appears and only a DEBUG log mentions a dial
 // timeout.
 //
+// WHAT IS PUSHED: each block once per peer. Per peer, this module remembers for the runtime the
+// revision of each block the peer accepted and the revision it refused ({@link PeerMemo}). A
+// refusal is the receiver's `missing` list: a payload it cannot parse, a block it failed to
+// persist, or a revision whose retained commit proof this node does not hold (a receiver running the default
+// `requirePushCertificate: true` refuses a push carrying no proof, and an unretained proof is
+// the ordinary case — see {@link Chunk.proofs}). A refusal is a per-block verdict: logged once
+// when it happens, and the block is offered to that peer again only once its local revision
+// moves past the refused one. A run skips a block, counting it in `alreadyHeld`, when:
+//
+// - the peer accepted that revision or a later one from this node;
+// - the peer refused that revision;
+// - the commit proof retained for that revision carries the peer's own commit-round approval.
+//   A block's proof lists the cohort members that signed the commit round for exactly that
+//   revision, so a founder's solo commit is never pushed back to it by a joiner now holding it,
+//   and a revision two machines committed together is not pushed between them.
+//
+// The proof rule stands in for "do not push a block back to the peer it came from": Optimystic
+// reports a push's sender only to its own rebalance monitor and exposes it to no embedder.
+//
+// NOTE: accepted tradeoff — a signer that crashed between signing and applying, or later lost its
+// store, is not re-sent that revision by this path; read repair covers its reads, as for any block
+// a cohort member lacks. Revisit if Optimystic tells embedders who holds what (it takes reports,
+// `reportBlockHeld`, but gives none) or a measurement shows pushes this rule misses.
+//
+// NOTE: the reply does not say why a block is missing, so a receiver's transient persist failure
+// is remembered as a refusal too and not retried this runtime; read repair covers that block's
+// reads. If receivers log `persist:fail` in practice, ask Optimystic for a per-block reason in
+// the push reply and remember only the verdicts a retry cannot change.
+//
+// A peer is marked caught up (`done`) once a run finishes with no push that threw, the gate
+// admitted it and the store is listable. A refusal does not hold that back, since it is
+// remembered per block. It stays caught up until its last connection closes
+// (`peer:disconnect`): `peer:identify` churn for a connected, caught-up peer is dropped before
+// any work, and a reconnect re-runs past the debounce, pushing only what the memo and the
+// proofs do not cover. {@link PeerJoinBackfill.forceCatchUpPeer} runs past `done` too, with the
+// same skips.
+//
 // NOTE: the protocol check happens once — either when a connection's `peer:identify` fires,
 // or, for a peer already connected when this instance starts, when `scheduleConnectedPeers()`
 // reads its protocols back from the peer store — and is never repeated once a peer is
@@ -132,17 +174,10 @@ import { peerJoinPushBudget } from './link-budget.js';
 //
 // NOTE: a peer that disconnects INSIDE a backoff wait still costs the one attempt that wait
 // was already armed for — the connectivity check is made when the re-arm is decided, not when
-// the timer fires, and this module subscribes to `peer:identify` only. One dial, not a
-// recurring one; if a node ever holds many transient peers, subscribe to `connection:close`
-// and clear the pending timer there.
+// the timer fires, and the `peer:disconnect` handler leaves backoff state alone (see
+// `forgetCaughtUp`). One dial, not a recurring one; if a node ever holds many transient peers,
+// clear the pending timer in that handler.
 const log = debug('sereus:cadre:peer-join-backfill');
-/**
- * The block-transfer protocol's hard cap on one length-prefixed message. Mirrors
- * `MAX_BLOCK_MESSAGE_BYTES` in `@optimystic/db-p2p`'s `protocol-limits.ts` (8 MiB), which
- * that package does not re-export from its index — keep the two in sync if upstream ever
- * changes it or starts exporting it.
- */
-export const MAX_BLOCK_MESSAGE_BYTES = 8 * 1024 * 1024;
 /**
  * Whether an identified peer's protocol list names this network's own block-transfer
  * protocol — the one gate that decides whether {@link PeerJoinBackfill} schedules a peer at
@@ -176,14 +211,27 @@ export const DEFAULT_PEER_JOIN_BACKFILL = {
     maxRetryBackoffMs: 60000
 };
 function emptyResult() {
-    return { offered: 0, accepted: 0, rejected: [], uncommitted: 0, unmaterialized: 0, capped: 0, oversized: [], denied: false };
+    return { offered: 0, alreadyHeld: 0, accepted: 0, rejected: [], uncommitted: 0, unmaterialized: 0, capped: 0, oversized: [], denied: false };
 }
-/**
- * Base64 wire size of a raw buffer — `pushBlocks` base64-encodes each block into the JSON
- * request, so the protocol cap must be judged against the encoded size, not the raw bytes.
- */
-function base64WireBytes(rawBytes) {
-    return Math.ceil(rawBytes / 3) * 4;
+/** Whether the peer's replies already settled revision `rev` of `blockId`, either way. */
+function memoSettled(memo, blockId, rev) {
+    return (memo.accepted.get(blockId) ?? -Infinity) >= rev
+        || (memo.rejected.get(blockId) ?? -Infinity) >= rev;
+}
+/** Whether `peerId` approved the commit round that `proof` certifies. */
+function signedBy(proof, peerId) {
+    return proof?.commits[peerId]?.type === 'approve';
+}
+/** Record one push reply in the memo: each block as accepted or refused at the revision pushed. */
+function recordReply(memo, revs, ids, missing) {
+    for (const id of ids) {
+        const rev = revs[id].rev;
+        const verdicts = missing.has(id) ? memo.rejected : memo.accepted;
+        verdicts.set(id, Math.max(verdicts.get(id) ?? -Infinity, rev));
+    }
+}
+function emptyChunk() {
+    return { blocks: {}, meta: {}, proofs: {}, bytes: 0 };
 }
 /**
  * Copies every block in one network's own raw store to each peer its libp2p node connects
@@ -193,8 +241,8 @@ function base64WireBytes(rawBytes) {
  *
  * Best-effort throughout: nothing here throws into a libp2p event handler or into a
  * runtime bring-up; a failed chunk is logged and the run continues. A peer is marked
- * fully caught up ONLY after a run with no thrown chunk and an empty `missing` list, so
- * the next `peer:identify` from a peer whose catch-up failed retries it.
+ * caught up only after a run with no thrown chunk, so a peer whose catch-up failed is
+ * retried; each block is pushed to each peer once (see "What is pushed" in the module comment).
  */
 export class PeerJoinBackfill {
     constructor(deps, config) {
@@ -202,10 +250,22 @@ export class PeerJoinBackfill {
         this.started = false;
         this.stopped = false;
         /**
-         * Peers fully caught up this runtime (never rescheduled until the runtime is rebuilt; only
-         * {@link forceCatchUpPeer} pushes to one again).
+         * Peers caught up since their current connection opened. Cleared for a peer when its last
+         * connection closes, so a reconnect re-runs; {@link forceCatchUpPeer} runs past it.
          */
         this.done = new Set();
+        /**
+         * Peers that disconnected while their run was in flight, so that run does not mark them
+         * {@link done} after the disconnect cleared it. Cleared when the run ends.
+         */
+        this.disconnectedInFlight = new Set();
+        /** Per-peer, per-block verdicts from push replies; see {@link PeerMemo}. */
+        this.memos = new Map();
+        /**
+         * Peers a run has been started for. A peer not in it is run at once on its first schedule;
+         * later schedules wait out `debounceMs`.
+         */
+        this.contacted = new Set();
         /**
          * Peers with a catch-up currently running, each with that run (suppresses concurrent
          * duplicates; {@link forceCatchUpPeer} waits on the run).
@@ -213,9 +273,9 @@ export class PeerJoinBackfill {
         this.inFlight = new Map();
         /**
          * Peers whose (re-)schedule arrived while their OWN run was in flight, replayed once
-         * that run finishes. Load-bearing for the gated path: the gate's authorization check is
-         * a control-database read, so the window between it and the run's end is wide enough for
-         * the very membership commit that would have authorized the peer to land inside it —
+         * that run finishes. Load-bearing for the gated path: the gate is judged once, at the start
+         * of a run, and the run's pushes take long enough for the very membership commit that would
+         * have authorized the peer to land after that check —
          * dropping that schedule (rather than deferring it) leaves the denied peer waiting for a
          * reconnect, which is exactly what the re-arm exists to avoid.
          */
@@ -242,18 +302,23 @@ export class PeerJoinBackfill {
                 this.schedulePeer(peerId);
             }
         };
+        this.onPeerDisconnect = (evt) => {
+            this.forgetCaughtUp(evt.detail.toString());
+        };
     }
     /**
      * Subscribe to `peer:identify` AND schedule a catch-up for already-connected peers whose
      * stored protocols already name this network's block-transfer protocol — a peer that does
-     * not speak it (a bare circuit relay, a bootstrap node) is never scheduled at all. See
-     * {@link speaksBlockTransfer} and the module comment.
+     * not speak it (a bare circuit relay, a bootstrap node) is never scheduled at all. Also
+     * subscribes to `peer:disconnect`, so a peer caught up before it went away is run again on
+     * its return. See {@link speaksBlockTransfer} and the module comment.
      */
     start() {
         if (this.started || this.stopped || !this.config.enabled)
             return;
         this.started = true;
         this.deps.libp2p.addEventListener('peer:identify', this.onPeerIdentify);
+        this.deps.libp2p.addEventListener('peer:disconnect', this.onPeerDisconnect);
         // A runtime rebuilt over live connections (resumeStrand) never sees their peer:identify —
         // it already fired before this instance existed, but its result is persisted in the peer
         // store — so walk what is already connected once.
@@ -268,6 +333,7 @@ export class PeerJoinBackfill {
         this.stopped = true;
         if (this.started) {
             this.deps.libp2p.removeEventListener('peer:identify', this.onPeerIdentify);
+            this.deps.libp2p.removeEventListener('peer:disconnect', this.onPeerDisconnect);
         }
         for (const timer of this.timers.values()) {
             clearTimeout(timer);
@@ -277,6 +343,24 @@ export class PeerJoinBackfill {
         this.retryAfter.clear();
         log('[%s] stopped', this.deps.label);
     }
+    /**
+     * A peer's last connection closed: it is no longer caught up, so its next `peer:identify`
+     * re-runs. Only `done` is cleared. The memo stays (it keeps the re-run to what moved),
+     * `contacted` stays (a link flapping inside one settle window costs one run), and the
+     * backoff state stays (clearing it re-dials an unreachable peer on every reconnect; see the
+     * module comment's retry paragraph).
+     *
+     * When the returning peer wrote while away with a cohort that named this node, its own
+     * re-run pushes those blocks here and reports them through `reportBlockHeld`, so its
+     * under-replication drain does not push them again. Revisit if a measurement shows reconnect
+     * re-runs dominating push traffic.
+     */
+    forgetCaughtUp(key) {
+        this.done.delete(key);
+        if (this.inFlight.has(key)) {
+            this.disconnectedInFlight.add(key);
+        }
+    }
     /**
      * (Re-)schedule a debounced catch-up for every currently-connected peer whose protocols, as
      * already stored in the libp2p peer store, name this network's block-transfer protocol. A
@@ -318,7 +402,11 @@ export class PeerJoinBackfill {
             this.schedulePeer(peerId);
         }
     }
-    /** Debounced entry point for connection churn: one run per peer per settle window. */
+    /**
+     * Entry point for connection churn: one run per peer per settle window, except that a peer
+     * never run for is run at once (on a zero-delay timer, so a burst of identify events in one
+     * turn still collapses, and no work happens inside the libp2p event dispatch).
+     */
     schedulePeer(peerId) {
         const key = peerId.toString();
         if (this.stopped || this.done.has(key))
@@ -339,10 +427,11 @@ export class PeerJoinBackfill {
         const existing = this.timers.get(key);
         if (existing)
             clearTimeout(existing);
+        const delayMs = this.contacted.has(key) ? this.config.debounceMs : 0;
         this.timers.set(key, setTimeout(() => {
             this.timers.delete(key);
             void this.catchUpPeer(peerId);
-        }, this.config.debounceMs));
+        }, delayMs));
     }
     /**
      * Run one peer's catch-up now, bypassing the debounce. Never rejects. Returns an
@@ -357,14 +446,16 @@ export class PeerJoinBackfill {
         return await this.trackRun(peerId);
     }
     /**
-     * Run one peer's catch-up now and wait for it, whatever this instance remembers about the
-     * peer: wait out a run already in flight for it, then push the whole store again even when
-     * the peer is memoized as caught up. Never rejects; an all-zero result only when stopped.
+     * Run one peer's catch-up now and wait for it, even when the peer is memoized as caught up:
+     * wait out a run already in flight for it, then run again. The run still skips every block the
+     * peer is known to hold (it accepted or refused that revision, or signed it), so for a peer that
+     * holds the whole store it pushes nothing and returns `alreadyHeld` equal to the store size.
+     * Never rejects; an all-zero result only when stopped.
      *
      * For a caller that must know the peer holds this store before it proceeds, where
      * {@link catchUpPeer}'s empty answer would hide a run that was denied (a `peer:identify`
-     * run can pass the gate a moment before the caller authorized the peer) or a memo older
-     * than the commits since.
+     * run can pass the gate a moment before the caller authorized the peer) or commits made
+     * since the peer was caught up.
      */
     async forceCatchUpPeer(peerId) {
         const key = peerId.toString();
@@ -382,6 +473,7 @@ export class PeerJoinBackfill {
      */
     async trackRun(peerId) {
         const key = peerId.toString();
+        this.contacted.add(key);
         const run = this.settleRun(peerId);
         this.inFlight.set(key, run);
         try {
@@ -389,6 +481,7 @@ export class PeerJoinBackfill {
         }
         finally {
             this.inFlight.delete(key);
+            this.disconnectedInFlight.delete(key);
             // Replay a schedule that arrived mid-run. `schedulePeer` re-checks `done`, so a run
             // that finished clean re-arms nothing; a denied or failed one gets its retry.
             if (this.rearmAfterFlight.delete(key)) {
@@ -401,28 +494,32 @@ export class PeerJoinBackfill {
         const key = peerId.toString();
         try {
             const { result, clean, pushFailed } = await this.runCatchUp(peerId);
-            // NOTE: a run that hit `maxBlocks` is still "clean" and still memoizes the peer, so
-            // the tail past the ceiling never reaches it. Deliberate: enumeration is not
-            // resumable, so not memoizing would re-push the same prefix on every reconnect
-            // without ever advancing. Loud in the log below (capped > 0). If a store can
-            // realistically exceed maxBlocks, the fix is a resumable cursor, not either policy.
+            // NOTE: a run that hit `maxBlocks` is still "clean" and still marks the peer caught up,
+            // so no `peer:identify` run reaches the tail past the ceiling. Loud in the log below
+            // (capped > 0). Blocks the peer is known to hold are not offered and do not count toward
+            // the ceiling, so a forced run resumes past the prefix already sent. If a store can
+            // realistically exceed maxBlocks, leave a capped run out of `done` so each identify run
+            // sends the next `maxBlocks`.
             if (clean && !this.stopped) {
-                this.done.add(key);
+                // A disconnect mid-run cleared `done` already; re-adding it would drop the reconnect.
+                // Not judged by `isConnected`: hand-driven runs target peers with no connection.
+                if (!this.disconnectedInFlight.has(key)) {
+                    this.done.add(key);
+                }
                 this.failures.delete(key);
                 this.retryAfter.delete(key);
                 this.warned.delete(key);
             }
             else if (pushFailed && !this.stopped && this.started && this.isConnected(peerId)) {
-                // `pushFailed`, not `!clean`: a denial, an inert store and a receiver's per-block
-                // rejection are all verdicts a retry cannot change, and re-arming on them would
-                // re-push the whole store to that peer forever. See the module comment.
+                // `pushFailed`, not `!clean`: a denial and an inert store are verdicts a retry cannot
+                // change, and re-arming on them would retry forever. See the module comment.
                 //
                 // Gated on `started` too: the re-arm exists to REPLACE a peer:identify-driven retry,
                 // so a caller driving `catchUpPeer` by hand against a backfill that was never started
                 // owns its own retry policy and must not be left holding a background timer.
                 this.scheduleRetryWithBackoff(peerId);
             }
-            log('[%s] catch-up peer=%s offered=%d accepted=%d rejected=%d uncommitted=%d unmaterialized=%d capped=%d oversized=%d denied=%s done=%s', this.deps.label, key, result.offered, result.accepted, result.rejected.length, result.uncommitted, result.unmaterialized, result.capped, result.oversized.length, result.denied, clean);
+            log('[%s] catch-up peer=%s offered=%d alreadyHeld=%d accepted=%d rejected=%d uncommitted=%d unmaterialized=%d capped=%d oversized=%d denied=%s done=%s', this.deps.label, key, result.offered, result.alreadyHeld, result.accepted, result.rejected.length, result.uncommitted, result.unmaterialized, result.capped, result.oversized.length, result.denied, clean);
             return result;
         }
         catch (error) {
@@ -474,14 +571,44 @@ export class PeerJoinBackfill {
             + 'these budgets and libp2p\'s own connection limits are derived from network.linkRoundTripMs (see link-budget.ts); '
             + 'a link slower than that declaration, or a peer that declared a faster one, cannot open a relayed connection at all.');
     }
+    /** This peer's memo, created empty on first use. */
+    memoFor(key) {
+        let memo = this.memos.get(key);
+        if (!memo) {
+            memo = { accepted: new Map(), rejected: new Map() };
+            this.memos.set(key, memo);
+        }
+        return memo;
+    }
     /**
-     * The actual copy. `clean` = every chunk pushed and the remote persisted every block.
+     * Tell the node which blocks of one push reply the peer accepted, at the revision pushed.
+     * Best-effort: a failed report is logged and costs only the drain's duplicate push.
+     */
+    async reportAccepted(key, revs, ids, missing) {
+        const report = this.deps.reportBlockHeld;
+        if (!report)
+            return;
+        for (const id of ids) {
+            if (missing.has(id))
+                continue;
+            try {
+                await report(id, revs[id].rev, key);
+            }
+            catch (error) {
+                log('[%s] reportBlockHeld for peer=%s block=%s failed: %s', this.deps.label, key, id, error.message);
+            }
+        }
+    }
+    /**
+     * The actual copy. `clean` = the gate admitted the peer, the store is listable and no push
+     * threw (a block the receiver refused is remembered per block instead, see {@link PeerMemo}).
      * `pushFailed` = at least one push THREW, which is the only non-clean outcome a retry can
-     * change; the others are verdicts (denied, inert store, blocks the receiver refused).
+     * change; the others are verdicts (denied, inert store).
      */
     async runCatchUp(peerId) {
         const result = emptyResult();
         const { storage } = this.deps;
+        const key = peerId.toString();
         // The membership gate, judged at push time (never at schedule time — authorization can
         // change during the debounce window in either direction). Fails CLOSED on a throw: a
         // gate that cannot answer must not leak the store. A denied run is not clean, so the
@@ -507,8 +634,8 @@ export class PeerJoinBackfill {
             return { result, clean: false, pushFailed: false };
         }
         const client = this.createPushClient(peerId);
-        const encoder = new TextEncoder();
-        let chunk = { ids: [], buffers: [], meta: {}, proofs: {}, bytes: 0 };
+        const memo = this.memoFor(key);
+        let chunk = emptyChunk();
         let chunkFailed = false;
         let anyChunkDelivered = false;
         /**
@@ -519,19 +646,22 @@ export class PeerJoinBackfill {
          */
         const peerUnreachable = () => chunkFailed && !anyChunkDelivered;
         const flush = async () => {
-            if (chunk.ids.length === 0)
+            const { blocks, meta, proofs } = chunk;
+            const ids = Object.keys(blocks);
+            if (ids.length === 0)
                 return;
-            const { ids, buffers, meta, proofs } = chunk;
-            chunk = { ids: [], buffers: [], meta: {}, proofs: {}, bytes: 0 };
+            chunk = emptyChunk();
             try {
                 // ONE certification value rather than two arguments, so a caller cannot pair a proof
                 // for one revision with meta for another — both were written from the same `latest`.
-                const response = await client.pushBlocks(ids, buffers, 'replication', { blockMeta: meta, blockProofs: proofs }, {
+                const response = await client.pushBlocks(blocks, 'replication', { blockMeta: meta, blockProofs: proofs }, {
                     dialTimeoutMs: this.config.dialTimeoutMs,
                     responseTimeoutMs: this.config.responseTimeoutMs
                 });
                 anyChunkDelivered = true;
                 const missing = new Set(response.missing);
+                recordReply(memo, meta, ids, missing);
+                await this.reportAccepted(key, meta, ids, missing);
                 for (const id of ids) {
                     if (missing.has(id)) {
                         result.rejected.push(id);
@@ -541,12 +671,12 @@ export class PeerJoinBackfill {
                     }
                 }
                 if (response.missing.length > 0) {
-                    log('[%s] peer=%s rejected %d block(s): %o', this.deps.label, peerId.toString(), response.missing.length, response.missing);
+                    log('[%s] peer=%s rejected %d block(s), not offered to it again at these revisions: %o', this.deps.label, key, response.missing.length, response.missing);
                 }
             }
             catch (error) {
                 chunkFailed = true;
-                log('[%s] push to peer=%s failed for %d block(s): %s', this.deps.label, peerId.toString(), ids.length, error.message);
+                log('[%s] push to peer=%s failed for %d block(s): %s', this.deps.label, key, ids.length, error.message);
             }
         };
         for await (const blockId of storage.listBlockIds()) {
@@ -558,65 +688,79 @@ export class PeerJoinBackfill {
                 result.capped += 1;
                 continue;
             }
-            const metadata = await storage.getMetadata(blockId);
-            const latest = metadata?.latest;
-            if (!latest) {
-                result.uncommitted += 1;
+            const offer = await this.readOffer(blockId, key, memo, result);
+            if (!offer)
                 continue;
-            }
-            // NOTE: a block whose latest revision is a DELETE materializes to nothing (a
-            // tombstone), so it is skipped here — a peer that joins after such a delete never
-            // receives the tombstone from this path and relies on read repair for it. Fine
-            // while nothing deletes whole blocks pre-join; if the whole-store coverage gate in
-            // strand-membership-closed-strand-e2e ever reports a tombstone residue, teach this
-            // to push the promoted delete transform instead of skipping.
-            const block = await storage.getMaterializedBlock(blockId, latest.actionId);
-            if (!block) {
-                result.unmaterialized += 1;
-                log('[%s] block=%s has latest rev %d but no materialized content; skipped', this.deps.label, blockId, latest.rev);
-                continue;
-            }
-            const buffer = encoder.encode(JSON.stringify(block));
-            // NOTE: MAX_BLOCK_MESSAGE_BYTES caps the whole framed request, not one block, and
-            // this test ignores the request envelope (ids array, blockMeta, JSON punctuation).
-            // A lone block within a few KiB of the cap therefore still ships and is rejected by
-            // the receiver's length-prefix decoder — a permanent chunk failure for that peer. No
-            // block comes close today; if one ever can, subtract a measured envelope allowance
-            // here rather than raising the cap.
-            if (base64WireBytes(buffer.length) >= MAX_BLOCK_MESSAGE_BYTES) {
-                result.oversized.push(blockId);
-                log('[%s] block=%s is %d bytes (%d on the wire) — exceeds the %d-byte protocol cap alone; skipped', this.deps.label, blockId, buffer.length, base64WireBytes(buffer.length), MAX_BLOCK_MESSAGE_BYTES);
-                continue;
-            }
             // Flush before adding when this block would overflow either budget. A single block
             // larger than maxChunkBytes still ships — alone, in its own chunk.
-            if (chunk.ids.length > 0
-                && (chunk.ids.length + 1 > this.config.maxChunkBlocks || chunk.bytes + buffer.length > this.config.maxChunkBytes)) {
+            const inChunk = Object.keys(chunk.blocks).length;
+            if (inChunk > 0
+                && (inChunk + 1 > this.config.maxChunkBlocks || chunk.bytes + offer.bytes > this.config.maxChunkBytes)) {
                 await flush();
                 if (this.stopped || peerUnreachable())
                     break;
             }
-            chunk.ids.push(blockId);
-            chunk.buffers.push(buffer);
-            chunk.meta[blockId] = { rev: latest.rev, actionId: latest.actionId };
-            // Read the proof for EXACTLY the revision being pushed, from the same `latest` the meta
-            // above was written from. Absent is normal (nothing retained a proof for that revision);
-            // the block still ships with its meta and the receiver decides.
-            const proof = await storage.getBlockProof(blockId, latest.rev);
-            if (proof) {
-                chunk.proofs[blockId] = proof;
+            chunk.blocks[blockId] = offer.block;
+            chunk.meta[blockId] = { rev: offer.latest.rev, actionId: offer.latest.actionId };
+            if (offer.proof) {
+                chunk.proofs[blockId] = offer.proof;
             }
-            chunk.bytes += buffer.length;
+            chunk.bytes += offer.bytes;
             result.offered += 1;
         }
         if (!this.stopped && !peerUnreachable()) {
             await flush();
         }
         if (result.capped > 0) {
-            log('[%s] peer=%s catch-up CAPPED at %d blocks; %d block id(s) not attempted', this.deps.label, peerId.toString(), this.config.maxBlocks, result.capped);
+            log('[%s] peer=%s catch-up CAPPED at %d blocks; %d block id(s) not attempted', this.deps.label, key, this.config.maxBlocks, result.capped);
+        }
+        return { result, clean: !chunkFailed, pushFailed: chunkFailed };
+    }
+    /**
+     * Read one block for the push to `key`, or count why it is skipped in `result` and return
+     * nothing: not committed, already held by the peer (memo or its own commit signature), not
+     * materialized, or over the per-block size limit. The proof is read for EXACTLY the revision the
+     * meta will name, from the same `latest`; absent is normal (nothing retained a proof for that
+     * revision), and the block still ships with its meta for the receiver to decide.
+     */
+    async readOffer(blockId, key, memo, result) {
+        const { storage } = this.deps;
+        const latest = (await storage.getMetadata(blockId))?.latest;
+        if (!latest) {
+            result.uncommitted += 1;
+            return undefined;
+        }
+        if (memoSettled(memo, blockId, latest.rev)) {
+            result.alreadyHeld += 1;
+            return undefined;
+        }
+        const proof = await storage.getBlockProof(blockId, latest.rev);
+        if (signedBy(proof, key)) {
+            result.alreadyHeld += 1;
+            return undefined;
+        }
+        // NOTE: a block whose latest revision is a DELETE materializes to nothing (a
+        // tombstone), so it is skipped here — a peer that joins after such a delete never
+        // receives the tombstone from this path and relies on read repair for it. Fine
+        // while nothing deletes whole blocks pre-join; if the whole-store coverage gate in
+        // strand-membership-closed-strand-e2e ever reports a tombstone residue, teach this
+        // to push the promoted delete transform instead of skipping.
+        const block = await storage.getMaterializedBlock(blockId, latest.actionId);
+        if (!block) {
+            result.unmaterialized += 1;
+            log('[%s] block=%s has latest rev %d but no materialized content; skipped', this.deps.label, blockId, latest.rev);
+            return undefined;
+        }
+        // A writer refuses a block over MAX_BLOCK_BYTES, and Optimystic sizes the push frame to carry
+        // one block at that limit with its meta and proof. A block past the limit grew some other way
+        // and may not fit the frame, and a push that does not fit fails on every run.
+        const bytes = jsonEncodedBytes(block);
+        if (bytes > MAX_BLOCK_BYTES) {
+            result.oversized.push(blockId);
+            log('[%s] block=%s encodes to %d bytes, over the %d-byte per-block limit; skipped', this.deps.label, blockId, bytes, MAX_BLOCK_BYTES);
+            return undefined;
         }
-        const clean = !chunkFailed && result.rejected.length === 0;
-        return { result, clean, pushFailed: chunkFailed };
+        return { latest, block, bytes, proof };
     }
 }
 //# sourceMappingURL=peer-join-backfill.js.map