@optimystic/db-p2p 1.0.0-beta.2 → 1.0.0

package/dist/src/repo/under-replication-drain.d.ts

@@ -0,0 +1,208 @@
+import type { Libp2p, Startable } from '@libp2p/interface';
+import type { BlockDurabilityReachedEvent, BlockId, IKeyNetwork } from '@optimystic/db-core';
+import type { PartitionDetector } from '../cluster/partition-detector.js';
+import type { PushBlockOutcome } from '../cluster/block-transfer-service.js';
+import type { IUnderReplicationLedger } from './i-under-replication-ledger.js';
+export interface UnderReplicationDrainConfig {
+    /** Debounce after a connection opens or a peer is identified, before a pass runs. Default: 5000 */
+    debounceMs: number;
+    /**
+     * Minimum interval between passes (ms). A trigger inside the window is DEFERRED to the window's
+     * end, never dropped, so a flapping peer costs at most one pass per interval and a peer that
+     * arrives mid-window is still drained to. Default: 15000
+     */
+    minIntervalMs: number;
+    /**
+     * Self-arming re-check timer while any ledger entry is outstanding (ms). Passes otherwise fire
+     * only on connection events, so a failed push on a then-quiet network — or a shortfall recorded
+     * while its missing member was already connected — would wait for the next arrival. Cheap while
+     * nothing can be pushed: a tick with no pushable peer connected costs one connection scan and
+     * one in-memory count. `0` disables. Default: 30000
+     */
+    recheckIntervalMs: number;
+    /**
+     * Entries examined per pass. Passes rotate through the ledger, oldest-recorded first, so a large
+     * backlog drains one budget per pass and no entry starves behind a stuck one. A deferred entry is
+     * not touched at all; it is re-examined on a later pass. Default: 64
+     */
+    blockBudget: number;
+    /**
+     * Consecutive rounds a reachable peer may fail to confirm one block before it is ABANDONED for
+     * that block — no longer pushed to — until it reconnects, which retries it from scratch. The
+     * entry itself stays in the ledger, visible, whatever happens here. Default: 5
+     */
+    maxAttempts: number;
+}
+export declare const DEFAULT_UNDER_REPLICATION_DRAIN_CONFIG: UnderReplicationDrainConfig;
+/**
+ * The one method the drain may call on the node's storage repo: emit a full-replication event.
+ * Handed in as a function, exactly as the cluster side receives a `CommitCertificateSink`, so the
+ * drain has no way to reach anything else on the repo.
+ */
+export type BlockDurabilitySink = (event: BlockDurabilityReachedEvent) => void;
+/**
+ * The push primitive, already bound to this node's own store, dialer and protocol prefix — the
+ * node binds `pushBlockToPeers` (`cluster/block-transfer-service.ts`); a unit test binds a double.
+ */
+export type BoundBlockPusher = (blockId: BlockId, peerIds: readonly string[]) => Promise<PushBlockOutcome>;
+/** The slice of libp2p the drain uses: identity, the connection events, and who is connected. */
+export type DrainLibp2p = Pick<Libp2p, 'peerId' | 'addEventListener' | 'removeEventListener' | 'getConnections'> & {
+    peerStore: Pick<Libp2p['peerStore'], 'get'>;
+};
+export interface UnderReplicationDrainDeps {
+    libp2p: DrainLibp2p;
+    ledger: IUnderReplicationLedger;
+    /** Re-resolves a block's cohort for an entry that could not name its missing members. */
+    keyNetwork: Pick<IKeyNetwork, 'findCluster'>;
+    partitionDetector: Pick<PartitionDetector, 'detectPartition'>;
+    pushBlock: BoundBlockPusher;
+    emit: BlockDurabilitySink;
+    /** The `/optimystic/<networkName>` prefix; a connected peer counts as pushable only once identify
+     *  reports it serving this network's block-transfer protocol. Default: none (any connected peer). */
+    protocolPrefix?: string;
+}
+/** What woke a pass. */
+export type DrainTrigger = 'start' | 'connection' | 'identify' | 'recheck' | 'manual';
+/** What one pass did. Counts are per ledger ENTRY (one per block). */
+export interface DrainPassResult {
+    trigger: DrainTrigger;
+    /** Set when the pass did no work at all, and why. */
+    skipped?: 'partition' | 'no-pushable-peer';
+    /** Entries in the ledger when the pass began. */
+    entries: number;
+    /** Entries examined this pass, up to the block budget. */
+    examined: number;
+    /** Entries the budget left for a later pass. */
+    deferred: number;
+    /** Entries whose cohort resolves to this node alone: nothing owed, nothing counted. */
+    solo: number;
+    /** Entries owing a copy to nobody currently pushable (not connected, not yet identified, or
+     *  abandoned) — or whose cohort could not be resolved this pass. */
+    waiting: number;
+    /** Entries pushed to at least one peer. */
+    pushed: number;
+    /** Entries whose last shortfall cleared — one full-replication event fired for each. */
+    cleared: number;
+    /** Entries deleted because the block is gone from local storage. */
+    dropped: number;
+    /** Peers newly abandoned for a block this pass. */
+    abandoned: number;
+}
+/**
+ * Sends the copies the under-replication ledger says this node still owes, and says when a block
+ * finally has all of them.
+ *
+ * `CoordinatorRepo.commit` writes an entry per block it acknowledged below full replication, naming
+ * the cohort members that had not confirmed it (or an EMPTY set when no cohort could be named — a
+ * `local` or `unrouted` write). This service is the other half: on start, whenever a peer connects or
+ * is identified, and on a re-check timer while anything is outstanding, it walks the ledger and pushes
+ * each block to the owed peers that are reachable right now. A peer that confirms is removed from the
+ * entry; when the entry empties it is deleted and ONE {@link BlockDurabilityReachedEvent} fires through
+ * the sink — after the deletion, never before.
+ *
+ * What it deliberately does not do:
+ * - **Dial the world.** A copy goes only to a peer that is connected and identified as serving this
+ *   network. An entry whose owed peers are all away simply waits; the next arrival wakes the drain.
+ * - **Re-resolve a NAMED peer.** A `majority` entry names members that were in the cohort when the
+ *   write was acknowledged; a copy there is never harmful even if the cohort has since moved on.
+ * - **Count an attempt against a node that is alone.** An entry with no named members whose cohort
+ *   still resolves to this node alone is left exactly as it is: nobody is owed anything yet.
+ * - **Delete an entry to tidy up.** Only a confirmed copy (or the block leaving local storage) removes
+ *   one. A peer that never returns leaves its entry in place; the ledger's size cap is the backstop.
+ * - **Coordinate with the rebalance growth arm or spread-on-churn.** All three may push the same
+ *   block to the same peer; the receiver is idempotent, so the overlap costs duplicate work only.
+ *
+ * NOTE: a pass that finds work SCANS the ledger (`IUnderReplicationLedger.list`, one store read per
+ * entry) and examines up to `blockBudget` entries, oldest first. Bounded by the node's owned-block
+ * count today. If a node ever holds so many under-replicated blocks that the scan shows up — a long
+ * solo run with a persistent store — index the ledger by missing peer instead of scanning, and
+ * consider draining newest-first so a host's most recent pending writes settle before old ones.
+ */
+export declare class UnderReplicationDrain implements Startable {
+    private readonly deps;
+    private running;
+    private readonly config;
+    private readonly blockTransferProtocol;
+    private debounceTimer;
+    private timer;
+    private timerDueAt;
+    private lastPassAt;
+    private passInFlight;
+    private pendingTrigger;
+    /** Rotation cursor into the oldest-first entry list, so a backlog wider than the budget is
+     *  swept in full over successive passes rather than the same oldest slice every time. */
+    private cursor;
+    /** Consecutive unconfirmed rounds per (block, peer) — the give-up counter. */
+    private readonly failures;
+    /** Peers given up on per block; excluded from pushes until they reconnect. */
+    private readonly abandoned;
+    private readonly onConnectionOpen;
+    private readonly onPeerIdentify;
+    constructor(deps: UnderReplicationDrainDeps, config?: Partial<UnderReplicationDrainConfig>);
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    /** Run one pass now, bypassing the debounce and the interval throttle. Waits for an in-flight pass first. */
+    checkNow(): Promise<DrainPassResult>;
+    /** Observability: given-up (block, peer) pairs, whether a timer is armed, and the last pass. */
+    getDiagnostics(): {
+        abandonedPairs: number;
+        timerArmed: boolean;
+        lastPassAt: number;
+    };
+    private handleConnectionOpen;
+    private schedule;
+    /** Run now, or defer to the end of the minimum interval — a trigger is never dropped. */
+    private maybeRun;
+    /** Arm the one timer, pulling an already-armed one EARLIER when `delayMs` asks for sooner. */
+    private armTimer;
+    private clearTimer;
+    private clearDebounce;
+    /** Arm the re-check while anything is outstanding; otherwise let the timer lapse. */
+    private rearmRecheck;
+    /** One pass, never overlapping another: a trigger that lands mid-pass runs one more afterwards. */
+    private run;
+    private performPass;
+    /** The next `blockBudget` entries from the rotation cursor, wrapping; the cursor advances past them. */
+    private takeBudget;
+    private drainEntry;
+    /**
+     * Take the confirmed peers off the entry. An entry that could not name its members is named now,
+     * at its own revision, with the cohort resolved for this push — so `satisfy` has a set to shrink.
+     * Both writes carry the revision guard: a newer shortfall recorded while the push was in flight
+     * is left alone (`name` touches only its own revision; `satisfy` keeps a higher one), and neither
+     * recreates an entry a concurrent `settle` removed mid-push.
+     *
+     * NOTE: an entry settled mid-push by a `full` commit at a newer revision is gone by the time
+     * `satisfy` runs, which reports that as "gone", so the event fires here too. It is true — the block
+     * IS fully replicated — but under the older `rev`/`actionId`, and a listener may also have heard
+     * `full` on that commit's own result. If a host ever needs the two told apart, have `satisfy`
+     * distinguish "deleted now" from "absent".
+     *
+     * @returns whether the entry is gone — its last missing member confirmed.
+     */
+    private recordConfirmed;
+    /** Log each refusal so "cannot place" reads differently from "cannot reach", and count the round. */
+    private noteRefusals;
+    /** Count one unconfirmed round for a peer; abandon it at the bound. @returns whether it was abandoned now. */
+    private countFailure;
+    /**
+     * Who the entry still owes. Named members are taken as recorded. An unknown set re-resolves the
+     * cohort NOW; a cohort of this node alone is `solo` — not a failure, not an attempt.
+     */
+    private owedPeers;
+    /**
+     * Connected peers identified as serving this network's block-transfer protocol — the only peers a
+     * push can land on. Identify fills the peer store's protocol list, so a peer that just connected
+     * is not yet pushable; the `peer:identify` trigger brings it in. With no prefix configured every
+     * connected peer counts.
+     */
+    private pushablePeers;
+    private servesBlockTransfer;
+    private forgetPeer;
+    /** An abandoned peer that has gone is forgotten: when it returns it is retried from scratch. */
+    private forgetDisconnected;
+    private forgetBlock;
+    /** Drop state for blocks the ledger no longer lists (settled by a `full` commit, evicted, dropped). */
+    private pruneState;
+}
+//# sourceMappingURL=under-replication-drain.d.ts.map

package/dist/src/repo/under-replication-drain.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"under-replication-drain.d.ts","sourceRoot":"","sources":["../../../src/repo/under-replication-drain.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAc,MAAM,EAAU,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC/E,OAAO,KAAK,EAAE,2BAA2B,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAE7F,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AAC1E,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,sCAAsC,CAAC;AAE7E,OAAO,KAAK,EAAE,uBAAuB,EAAwB,MAAM,iCAAiC,CAAC;AAKrG,MAAM,WAAW,2BAA2B;IAC3C,mGAAmG;IACnG,UAAU,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,aAAa,EAAE,MAAM,CAAC;IACtB;;;;;;OAMG;IACH,iBAAiB,EAAE,MAAM,CAAC;IAC1B;;;;OAIG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,WAAW,EAAE,MAAM,CAAC;CACpB;AAED,eAAO,MAAM,sCAAsC,EAAE,2BAMpD,CAAC;AAEF;;;;GAIG;AACH,MAAM,MAAM,mBAAmB,GAAG,CAAC,KAAK,EAAE,2BAA2B,KAAK,IAAI,CAAC;AAE/E;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,MAAM,EAAE,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE3G,iGAAiG;AACjG,MAAM,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,QAAQ,GAAG,kBAAkB,GAAG,qBAAqB,GAAG,gBAAgB,CAAC,GAAG;IAClH,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,KAAK,CAAC,CAAC;CAC5C,CAAC;AAEF,MAAM,WAAW,yBAAyB;IACzC,MAAM,EAAE,WAAW,CAAC;IACpB,MAAM,EAAE,uBAAuB,CAAC;IAChC,yFAAyF;IACzF,UAAU,EAAE,IAAI,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;IAC7C,iBAAiB,EAAE,IAAI,CAAC,iBAAiB,EAAE,iBAAiB,CAAC,CAAC;IAC9D,SAAS,EAAE,gBAAgB,CAAC;IAC5B,IAAI,EAAE,mBAAmB,CAAC;IAC1B;yGACqG;IACrG,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,wBAAwB;AACxB,MAAM,MAAM,YAAY,GAAG,OAAO,GAAG,YAAY,GAAG,UAAU,GAAG,SAAS,GAAG,QAAQ,CAAC;AAEtF,sEAAsE;AACtE,MAAM,WAAW,eAAe;IAC/B,OAAO,EAAE,YAAY,CAAC;IACtB,qDAAqD;IACrD,OAAO,CAAC,EAAE,WAAW,GAAG,kBAAkB,CAAC;IAC3C,iDAAiD;IACjD,OAAO,EAAE,MAAM,CAAC;IAChB,0DAA0D;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,gDAAgD;IAChD,QAAQ,EAAE,MAAM,CAAC;IACjB,uFAAuF;IACvF,IAAI,EAAE,MAAM,CAAC;IACb;wEACoE;IACpE,OAAO,EAAE,MAAM,CAAC;IAChB,2CAA2C;IAC3C,MAAM,EAAE,MAAM,CAAC;IACf,wFAAwF;IACxF,OAAO,EAAE,MAAM,CAAC;IAChB,oEAAoE;IACpE,OAAO,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,SAAS,EAAE,MAAM,CAAC;CAClB;AAYD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,qBAAsB,YAAW,SAAS;IAsBrD,OAAO,CAAC,QAAQ,CAAC,IAAI;IArBtB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA8B;IACrD,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAS;IAC/C,OAAO,CAAC,aAAa,CAA8C;IACnE,OAAO,CAAC,KAAK,CAA8C;IAC3D,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,YAAY,CAAyC;IAC7D,OAAO,CAAC,cAAc,CAA6B;IACnD;6FACyF;IACzF,OAAO,CAAC,MAAM,CAAK;IACnB,8EAA8E;IAC9E,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA2C;IACpE,8EAA8E;IAC9E,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmC;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAA2C;IAC5E,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAa;gBAG1B,IAAI,EAAE,yBAAyB,EAChD,MAAM,GAAE,OAAO,CAAC,2BAA2B,CAAM;IAU5C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAWtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAe3B,6GAA6G;IACvG,QAAQ,IAAI,OAAO,CAAC,eAAe,CAAC;IAK1C,gGAAgG;IAChG,cAAc,IAAI;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,OAAO,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE;IAQrF,OAAO,CAAC,oBAAoB;IAmB5B,OAAO,CAAC,QAAQ;IAUhB,yFAAyF;IACzF,OAAO,CAAC,QAAQ;IAWhB,8FAA8F;IAC9F,OAAO,CAAC,QAAQ;IAehB,OAAO,CAAC,UAAU;IAOlB,OAAO,CAAC,aAAa;IAOrB,qFAAqF;YACvE,YAAY;IAQ1B,mGAAmG;IACnG,OAAO,CAAC,GAAG;YAeG,WAAW;IAsCzB,wGAAwG;IACxG,OAAO,CAAC,UAAU;YAYJ,UAAU;IAwDxB;;;;;;;;;;;;;;OAcG;YACW,eAAe;IAY7B,qGAAqG;IACrG,OAAO,CAAC,YAAY;IAgBpB,8GAA8G;IAC9G,OAAO,CAAC,YAAY;IAsBpB;;;OAGG;YACW,SAAS;IAgBvB;;;;;OAKG;YACW,aAAa;YAeb,mBAAmB;IAYjC,OAAO,CAAC,UAAU;IAOlB,gGAAgG;IAChG,OAAO,CAAC,kBAAkB;IAS1B,OAAO,CAAC,WAAW;IAKnB,uGAAuG;IACvG,OAAO,CAAC,UAAU;CAMlB"}

package/dist/src/repo/under-replication-drain.js

@@ -0,0 +1,461 @@
+import { routingKeyForBlock } from '@optimystic/db-core';
+import { buildBlockTransferProtocol } from '../cluster/block-transfer-service.js';
+import { createLogger } from '../logger.js';
+const log = createLogger('under-replication-drain');
+export const DEFAULT_UNDER_REPLICATION_DRAIN_CONFIG = {
+    debounceMs: 5000,
+    minIntervalMs: 15_000,
+    recheckIntervalMs: 30_000,
+    blockBudget: 64,
+    maxAttempts: 5
+};
+const zeroCounts = (trigger) => ({
+    trigger, entries: 0, examined: 0, deferred: 0, solo: 0, waiting: 0, pushed: 0, cleared: 0, dropped: 0, abandoned: 0
+});
+/**
+ * Sends the copies the under-replication ledger says this node still owes, and says when a block
+ * finally has all of them.
+ *
+ * `CoordinatorRepo.commit` writes an entry per block it acknowledged below full replication, naming
+ * the cohort members that had not confirmed it (or an EMPTY set when no cohort could be named — a
+ * `local` or `unrouted` write). This service is the other half: on start, whenever a peer connects or
+ * is identified, and on a re-check timer while anything is outstanding, it walks the ledger and pushes
+ * each block to the owed peers that are reachable right now. A peer that confirms is removed from the
+ * entry; when the entry empties it is deleted and ONE {@link BlockDurabilityReachedEvent} fires through
+ * the sink — after the deletion, never before.
+ *
+ * What it deliberately does not do:
+ * - **Dial the world.** A copy goes only to a peer that is connected and identified as serving this
+ *   network. An entry whose owed peers are all away simply waits; the next arrival wakes the drain.
+ * - **Re-resolve a NAMED peer.** A `majority` entry names members that were in the cohort when the
+ *   write was acknowledged; a copy there is never harmful even if the cohort has since moved on.
+ * - **Count an attempt against a node that is alone.** An entry with no named members whose cohort
+ *   still resolves to this node alone is left exactly as it is: nobody is owed anything yet.
+ * - **Delete an entry to tidy up.** Only a confirmed copy (or the block leaving local storage) removes
+ *   one. A peer that never returns leaves its entry in place; the ledger's size cap is the backstop.
+ * - **Coordinate with the rebalance growth arm or spread-on-churn.** All three may push the same
+ *   block to the same peer; the receiver is idempotent, so the overlap costs duplicate work only.
+ *
+ * NOTE: a pass that finds work SCANS the ledger (`IUnderReplicationLedger.list`, one store read per
+ * entry) and examines up to `blockBudget` entries, oldest first. Bounded by the node's owned-block
+ * count today. If a node ever holds so many under-replicated blocks that the scan shows up — a long
+ * solo run with a persistent store — index the ledger by missing peer instead of scanning, and
+ * consider draining newest-first so a host's most recent pending writes settle before old ones.
+ */
+export class UnderReplicationDrain {
+    deps;
+    running = false;
+    config;
+    blockTransferProtocol;
+    debounceTimer = null;
+    timer = null;
+    timerDueAt = 0;
+    lastPassAt = 0;
+    passInFlight = null;
+    pendingTrigger = null;
+    /** Rotation cursor into the oldest-first entry list, so a backlog wider than the budget is
+     *  swept in full over successive passes rather than the same oldest slice every time. */
+    cursor = 0;
+    /** Consecutive unconfirmed rounds per (block, peer) — the give-up counter. */
+    failures = new Map();
+    /** Peers given up on per block; excluded from pushes until they reconnect. */
+    abandoned = new Map();
+    onConnectionOpen;
+    onPeerIdentify;
+    constructor(deps, config = {}) {
+        this.deps = deps;
+        this.config = { ...DEFAULT_UNDER_REPLICATION_DRAIN_CONFIG, ...config };
+        this.blockTransferProtocol = buildBlockTransferProtocol(deps.protocolPrefix ?? '');
+        this.onConnectionOpen = (event) => this.handleConnectionOpen(event);
+        this.onPeerIdentify = () => this.schedule('identify');
+    }
+    // ── Startable ────────────────────────────────────────────────────
+    async start() {
+        if (this.running)
+            return;
+        this.running = true;
+        this.deps.libp2p.addEventListener('connection:open', this.onConnectionOpen);
+        this.deps.libp2p.addEventListener('peer:identify', this.onPeerIdentify);
+        log('started');
+        // The one case no in-memory mechanism ever covered: entries the ledger carried across a
+        // restart. Off the start path so a slow scan never blocks the node coming up.
+        void this.run('start').catch(err => { log('start pass error: %o', err); });
+    }
+    async stop() {
+        if (!this.running)
+            return;
+        this.running = false;
+        this.deps.libp2p.removeEventListener('connection:open', this.onConnectionOpen);
+        this.deps.libp2p.removeEventListener('peer:identify', this.onPeerIdentify);
+        this.clearDebounce();
+        this.clearTimer();
+        this.pendingTrigger = null;
+        // An in-flight pass checks `running` between entries and bails; it is not awaited here so a
+        // stop never waits out a push deadline.
+        log('stopped');
+    }
+    // ── Public API ───────────────────────────────────────────────────
+    /** Run one pass now, bypassing the debounce and the interval throttle. Waits for an in-flight pass first. */
+    async checkNow() {
+        if (this.passInFlight)
+            await this.passInFlight.catch(() => undefined);
+        return this.run('manual');
+    }
+    /** Observability: given-up (block, peer) pairs, whether a timer is armed, and the last pass. */
+    getDiagnostics() {
+        let abandonedPairs = 0;
+        for (const peers of this.abandoned.values())
+            abandonedPairs += peers.size;
+        return { abandonedPairs, timerArmed: this.timer !== null, lastPassAt: this.lastPassAt };
+    }
+    // ── Scheduling ───────────────────────────────────────────────────
+    handleConnectionOpen(event) {
+        if (!this.running)
+            return;
+        // A peer that comes back is retried from scratch: its abandonment and give-up counts go.
+        // Only a FRESH connection counts — libp2p opens parallel connections to one peer, and a second
+        // one must not wipe the record of a peer that never went away.
+        // NOTE: "fresh" is read as "this is the peer's only connection", which relies on libp2p's
+        // connection manager having registered the new connection before this listener runs. True on
+        // libp2p 3.x: the manager subscribes to `connection:open` at its own start (before any user
+        // listener) and stores the connection synchronously, before its first await. If a libp2p
+        // upgrade ever defers that store, every parallel connection would look fresh and an abandoned
+        // peer would be retried on each one — cheap, not wrong — and the check would want the
+        // `peer:connect` event instead, which fires only for a peer's first connection.
+        const remotePeer = event?.detail?.remotePeer;
+        if (remotePeer !== undefined && this.deps.libp2p.getConnections(remotePeer).length <= 1) {
+            this.forgetPeer(remotePeer.toString());
+        }
+        this.schedule('connection');
+    }
+    schedule(trigger) {
+        if (!this.running)
+            return;
+        this.clearDebounce();
+        this.debounceTimer = setTimeout(() => {
+            this.debounceTimer = null;
+            this.maybeRun(trigger);
+        }, this.config.debounceMs);
+        unref(this.debounceTimer);
+    }
+    /** Run now, or defer to the end of the minimum interval — a trigger is never dropped. */
+    maybeRun(trigger) {
+        if (!this.running)
+            return;
+        const wait = this.lastPassAt + this.config.minIntervalMs - Date.now();
+        if (wait > 0) {
+            log('throttled trigger=%s wait=%dms', trigger, wait);
+            this.armTimer(wait, trigger);
+            return;
+        }
+        void this.run(trigger).catch(err => { log('pass error trigger=%s: %o', trigger, err); });
+    }
+    /** Arm the one timer, pulling an already-armed one EARLIER when `delayMs` asks for sooner. */
+    armTimer(delayMs, trigger) {
+        if (!this.running)
+            return;
+        const dueAt = Date.now() + delayMs;
+        if (this.timer !== null) {
+            if (this.timerDueAt <= dueAt)
+                return;
+            this.clearTimer();
+        }
+        this.timerDueAt = dueAt;
+        this.timer = setTimeout(() => {
+            this.timer = null;
+            this.maybeRun(trigger);
+        }, delayMs);
+        unref(this.timer);
+    }
+    clearTimer() {
+        if (this.timer !== null) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+    }
+    clearDebounce() {
+        if (this.debounceTimer !== null) {
+            clearTimeout(this.debounceTimer);
+            this.debounceTimer = null;
+        }
+    }
+    /** Arm the re-check while anything is outstanding; otherwise let the timer lapse. */
+    async rearmRecheck() {
+        if (!this.running || this.config.recheckIntervalMs <= 0)
+            return;
+        const outstanding = await this.deps.ledger.size();
+        if (outstanding > 0)
+            this.armTimer(this.config.recheckIntervalMs, 'recheck');
+    }
+    // ── The pass ─────────────────────────────────────────────────────
+    /** One pass, never overlapping another: a trigger that lands mid-pass runs one more afterwards. */
+    run(trigger) {
+        if (this.passInFlight) {
+            this.pendingTrigger = trigger;
+            return this.passInFlight;
+        }
+        const pass = this.performPass(trigger).finally(() => {
+            this.passInFlight = null;
+            const pending = this.pendingTrigger;
+            this.pendingTrigger = null;
+            if (pending !== null)
+                this.maybeRun(pending);
+        });
+        this.passInFlight = pass;
+        return pass;
+    }
+    async performPass(trigger) {
+        const counts = zeroCounts(trigger);
+        // A pass that scanned counts for the throttle however it ends — a ledger fault mid-pass must
+        // not make the next trigger an unthrottled retry. A skipped pass is free and does not count.
+        let scanned = false;
+        try {
+            if (this.deps.partitionDetector.detectPartition()) {
+                log('pass:skip trigger=%s reason=partition', trigger);
+                return { ...counts, skipped: 'partition' };
+            }
+            const pushable = await this.pushablePeers();
+            this.forgetDisconnected(pushable);
+            if (pushable.size === 0) {
+                return { ...counts, skipped: 'no-pushable-peer' };
+            }
+            const entries = await this.deps.ledger.list();
+            scanned = true;
+            this.pruneState(entries);
+            counts.entries = entries.length;
+            const examined = this.takeBudget(entries);
+            counts.examined = examined.length;
+            counts.deferred = entries.length - examined.length;
+            for (const entry of examined) {
+                if (!this.running)
+                    break;
+                await this.drainEntry(entry, pushable, counts);
+            }
+            log('pass:done trigger=%s entries=%d examined=%d deferred=%d solo=%d waiting=%d pushed=%d cleared=%d dropped=%d abandoned=%d', trigger, counts.entries, counts.examined, counts.deferred, counts.solo, counts.waiting, counts.pushed, counts.cleared, counts.dropped, counts.abandoned);
+            return counts;
+        }
+        finally {
+            if (scanned)
+                this.lastPassAt = Date.now();
+            await this.rearmRecheck();
+        }
+    }
+    /** The next `blockBudget` entries from the rotation cursor, wrapping; the cursor advances past them. */
+    takeBudget(entries) {
+        const budget = Math.max(1, this.config.blockBudget);
+        if (entries.length <= budget) {
+            this.cursor = 0;
+            return entries;
+        }
+        const start = this.cursor % entries.length;
+        const taken = [...entries.slice(start, start + budget), ...entries.slice(0, Math.max(0, start + budget - entries.length))];
+        this.cursor = (start + budget) % entries.length;
+        return taken;
+    }
+    async drainEntry(entry, pushable, counts) {
+        const owed = await this.owedPeers(entry);
+        if (owed.kind === 'solo') {
+            counts.solo++;
+            return;
+        }
+        if (owed.kind === 'unresolved') {
+            counts.waiting++;
+            return;
+        }
+        const abandoned = this.abandoned.get(entry.blockId);
+        const reachable = owed.peerIds.filter(peerId => pushable.has(peerId) && !abandoned?.has(peerId));
+        if (reachable.length === 0) {
+            counts.waiting++;
+            return;
+        }
+        const outcome = await this.deps.pushBlock(entry.blockId, reachable);
+        if (outcome.status === 'no-local-data') {
+            // A block this node no longer holds is not one it can owe — the same call spread-on-churn
+            // makes when a tracked block has left local storage.
+            log('entry:dropped block=%s rev=%d (no local data)', entry.blockId, entry.rev);
+            await this.deps.ledger.delete(entry.blockId);
+            this.forgetBlock(entry.blockId);
+            counts.dropped++;
+            return;
+        }
+        if (outcome.status === 'unavailable') {
+            // Could not find out what this node holds — not "holds nothing". Keep the entry, count nothing.
+            log('entry:unavailable block=%s rev=%d reason=%s (keeping)', entry.blockId, entry.rev, outcome.reason);
+            counts.waiting++;
+            return;
+        }
+        counts.pushed++;
+        const heldRev = outcome.latest?.rev ?? entry.rev;
+        for (const peerId of outcome.confirmed) {
+            log('push:ok block=%s peer=%s rev=%d', entry.blockId, peerId, heldRev);
+            this.failures.get(entry.blockId)?.delete(peerId);
+        }
+        if (outcome.confirmed.length > 0) {
+            const cleared = await this.recordConfirmed(entry, owed, outcome.confirmed, heldRev);
+            if (cleared) {
+                counts.cleared++;
+                this.forgetBlock(entry.blockId);
+                // After the deletion, never before: a listener that re-reads the ledger sees it gone.
+                this.deps.emit({ blockIds: [entry.blockId], rev: entry.rev, actionId: entry.actionId, collectionId: outcome.collectionId });
+                return;
+            }
+        }
+        if (outcome.refusals.length > 0) {
+            await this.deps.ledger.noteAttempt(entry.blockId);
+            counts.abandoned += this.noteRefusals(entry, outcome);
+        }
+    }
+    /**
+     * Take the confirmed peers off the entry. An entry that could not name its members is named now,
+     * at its own revision, with the cohort resolved for this push — so `satisfy` has a set to shrink.
+     * Both writes carry the revision guard: a newer shortfall recorded while the push was in flight
+     * is left alone (`name` touches only its own revision; `satisfy` keeps a higher one), and neither
+     * recreates an entry a concurrent `settle` removed mid-push.
+     *
+     * NOTE: an entry settled mid-push by a `full` commit at a newer revision is gone by the time
+     * `satisfy` runs, which reports that as "gone", so the event fires here too. It is true — the block
+     * IS fully replicated — but under the older `rev`/`actionId`, and a listener may also have heard
+     * `full` on that commit's own result. If a host ever needs the two told apart, have `satisfy`
+     * distinguish "deleted now" from "absent".
+     *
+     * @returns whether the entry is gone — its last missing member confirmed.
+     */
+    async recordConfirmed(entry, owed, confirmed, heldRev) {
+        if (!owed.named) {
+            await this.deps.ledger.name(entry.blockId, entry.rev, owed.peerIds);
+        }
+        const remaining = await this.deps.ledger.satisfy(entry.blockId, confirmed, heldRev);
+        if (remaining === undefined) {
+            log('entry:cleared block=%s rev=%d action=%s', entry.blockId, entry.rev, entry.actionId);
+            return true;
+        }
+        return false;
+    }
+    /** Log each refusal so "cannot place" reads differently from "cannot reach", and count the round. */
+    noteRefusals(entry, outcome) {
+        let abandoned = 0;
+        for (const refusal of outcome.refusals) {
+            if (refusal.reason === 'rejected') {
+                // Uncertified means this node retained no proof for the revision: the receiver's
+                // default policy refuses it, and no retry changes that until the block is rewritten.
+                log('push:rejected block=%s peer=%s rev=%d certified=%s%s', entry.blockId, refusal.peerId, entry.rev, outcome.certified, outcome.certified ? '' : ' (no retained proof — cannot place)');
+            }
+            else {
+                log('push:unreachable block=%s peer=%s err=%s', entry.blockId, refusal.peerId, refusal.error);
+            }
+            if (this.countFailure(entry.blockId, refusal.peerId))
+                abandoned++;
+        }
+        return abandoned;
+    }
+    /** Count one unconfirmed round for a peer; abandon it at the bound. @returns whether it was abandoned now. */
+    countFailure(blockId, peerId) {
+        let perPeer = this.failures.get(blockId);
+        if (!perPeer) {
+            perPeer = new Map();
+            this.failures.set(blockId, perPeer);
+        }
+        const count = (perPeer.get(peerId) ?? 0) + 1;
+        if (count < this.config.maxAttempts) {
+            perPeer.set(peerId, count);
+            return false;
+        }
+        perPeer.delete(peerId);
+        let peers = this.abandoned.get(blockId);
+        if (!peers) {
+            peers = new Set();
+            this.abandoned.set(blockId, peers);
+        }
+        peers.add(peerId);
+        log('peer:abandoned block=%s peer=%s after=%d rounds (until it reconnects)', blockId, peerId, count);
+        return true;
+    }
+    /**
+     * Who the entry still owes. Named members are taken as recorded. An unknown set re-resolves the
+     * cohort NOW; a cohort of this node alone is `solo` — not a failure, not an attempt.
+     */
+    async owedPeers(entry) {
+        const selfId = this.deps.libp2p.peerId.toString();
+        if (entry.missingPeerIds.length > 0) {
+            return { kind: 'peers', peerIds: entry.missingPeerIds.filter(peerId => peerId !== selfId), named: true };
+        }
+        let cluster;
+        try {
+            cluster = await this.deps.keyNetwork.findCluster(routingKeyForBlock(entry.blockId));
+        }
+        catch (err) {
+            log('cohort:unresolved block=%s err=%s', entry.blockId, err instanceof Error ? err.message : String(err));
+            return { kind: 'unresolved' };
+        }
+        const members = Object.keys(cluster).filter(peerId => peerId !== selfId);
+        return members.length === 0 ? { kind: 'solo' } : { kind: 'peers', peerIds: members, named: false };
+    }
+    /**
+     * Connected peers identified as serving this network's block-transfer protocol — the only peers a
+     * push can land on. Identify fills the peer store's protocol list, so a peer that just connected
+     * is not yet pushable; the `peer:identify` trigger brings it in. With no prefix configured every
+     * connected peer counts.
+     */
+    async pushablePeers() {
+        const seen = new Set();
+        const pushable = new Set();
+        for (const connection of this.deps.libp2p.getConnections()) {
+            const peerId = connection.remotePeer;
+            const peerIdStr = peerId.toString();
+            if (seen.has(peerIdStr))
+                continue;
+            seen.add(peerIdStr);
+            if (this.deps.protocolPrefix === undefined || await this.servesBlockTransfer(peerId)) {
+                pushable.add(peerIdStr);
+            }
+        }
+        return pushable;
+    }
+    async servesBlockTransfer(peerId) {
+        try {
+            const peer = await this.deps.libp2p.peerStore.get(peerId);
+            return peer.protocols.includes(this.blockTransferProtocol);
+        }
+        catch {
+            // Not in the peer store (not yet identified) or a store fault: not pushable this pass.
+            return false;
+        }
+    }
+    // ── In-memory state upkeep ───────────────────────────────────────
+    forgetPeer(peerId) {
+        for (const peers of this.failures.values())
+            peers.delete(peerId);
+        for (const [blockId, peers] of this.abandoned) {
+            if (peers.delete(peerId) && peers.size === 0)
+                this.abandoned.delete(blockId);
+        }
+    }
+    /** An abandoned peer that has gone is forgotten: when it returns it is retried from scratch. */
+    forgetDisconnected(pushable) {
+        for (const [blockId, peers] of this.abandoned) {
+            for (const peerId of peers) {
+                if (!pushable.has(peerId))
+                    peers.delete(peerId);
+            }
+            if (peers.size === 0)
+                this.abandoned.delete(blockId);
+        }
+    }
+    forgetBlock(blockId) {
+        this.failures.delete(blockId);
+        this.abandoned.delete(blockId);
+    }
+    /** Drop state for blocks the ledger no longer lists (settled by a `full` commit, evicted, dropped). */
+    pruneState(entries) {
+        const listed = new Set(entries.map(entry => entry.blockId));
+        for (const blockId of [...this.failures.keys(), ...this.abandoned.keys()]) {
+            if (!listed.has(blockId))
+                this.forgetBlock(blockId);
+        }
+    }
+}
+/** Never hold the process open for a drain timer. */
+function unref(timer) {
+    timer.unref?.();
+}
+//# sourceMappingURL=under-replication-drain.js.map