mindoodb 0.0.85 → 0.0.88

package/dist/server/peer/PeerReplicator.js

@@ -0,0 +1,1072 @@
+"use strict";
+/**
+ * Server-to-server replication with one trusted peer.
+ *
+ * One replicator instance per entry in `trusted-servers.json` that carries a
+ * `url`. It owns everything about that link: the peer session, the set of
+ * tenants both sides hold, the mirror loop over every database, the live push
+ * out of {@link SyncEventBus}, and the SSE subscription that pulls the other
+ * direction.
+ *
+ * ## What it replicates
+ *
+ * Everything both servers already have in common, and nothing else. The tenant
+ * set is intersected via a Bloom summary (see {@link refreshIntersection}), so
+ * a server never learns which tenants it does *not* host on the other side. For
+ * each shared tenant it mirrors **all** databases — the union of the local and
+ * remote database lists — in both stores (`docs` and `attachments`).
+ *
+ * ## Convergence invariants
+ *
+ * 1. **Directory first.** Databases are ordered by
+ *    {@link orderDatabasesForReplication}: `directory`, then `userdirectory`,
+ *    then the rest. An application entry signed by a key the peer does not
+ *    trust yet is rejected per entry, so the grants have to land first.
+ * 2. **Cursor holds on rejection.** Unlike a client, a peer must not skip past
+ *    a rejected entry — that is permanent divergence between two servers that
+ *    are supposed to be identical. The peer path passes
+ *    `rejectionPolicy: "hold"`, which clamps the cursor so the next run
+ *    re-offers the entry once the missing trust has replicated.
+ * 3. **Receipts are preserved, not re-stamped.** Enforced on the receiving side
+ *    in `ServerNetworkContentAddressedStore`; this side just transports.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PeerReplicator = void 0;
+exports.rollUpHealth = rollUpHealth;
+const HttpTransport_1 = require("../../appendonlystores/network/HttpTransport");
+const ClientNetworkContentAddressedStore_1 = require("../../appendonlystores/network/ClientNetworkContentAddressedStore");
+const IrohNetworkTransport_1 = require("../../core/appendonlystores/network/IrohNetworkTransport");
+const irohLocator_1 = require("../../core/appendonlystores/network/irohLocator");
+const types_1 = require("../../core/types");
+const types_2 = require("../../core/appendonlystores/network/types");
+const AuthChallenge_1 = require("../../core/crypto/AuthChallenge");
+const logging_1 = require("../../core/logging");
+const bloom_1 = require("../../core/appendonlystores/bloom");
+const base64_1 = require("../../core/utils/base64");
+const types_3 = require("../../core/appendonlystores/types");
+const purgedDocumentGuard_1 = require("../../core/appendonlystores/purgedDocumentGuard");
+const syncStores_1 = require("../../core/appendonlystores/syncStores");
+const types_4 = require("./types");
+/** Token lifetime is an hour on the server; refresh well before that. */
+const PEER_TOKEN_REFRESH_MS = 45 * 60 * 1000;
+const RECONNECT_BASE_MS = 1000;
+const RECONNECT_MAX_MS = 60000;
+const IDLE_SYNC_INTERVAL_MS = 5 * 60 * 1000;
+/** Live pushes are coalesced over this window so a burst becomes one run. */
+const PUSH_COALESCE_MS = 250;
+class PeerReplicator {
+    constructor(peer, host) {
+        this.running = false;
+        this.paused = false;
+        this.sessionState = "disabled";
+        this.eventsConnected = false;
+        this.token = null;
+        this.tokenIssuedAt = 0;
+        this.tokenPromise = null;
+        this.intersection = [];
+        /** Tenant a run that hit its deadline did not finish; the next run starts there. */
+        this.resumeTenantId = null;
+        this.intersectionAt = null;
+        this.tenants = new Map();
+        this.transferredEntries = 0;
+        this.transferredBytes = 0;
+        this.syncInFlight = null;
+        this.pendingScopes = new Set();
+        this.pushTimer = null;
+        this.idleTimer = null;
+        this.reconnectTimer = null;
+        this.reconnectAttempt = 0;
+        this.sseAbort = null;
+        this.irohLink = null;
+        this.unsubscribeBus = null;
+        this.lastError = null;
+        this.lastErrorClass = null;
+        this.lastErrorAt = null;
+        this.lastSyncAt = null;
+        /**
+         * Scan cursors, per (source, target) store pair.
+         *
+         * In-process only, deliberately: a restarted server pays one metadata scan
+         * per pair and then re-establishes the fast path from the store head. Cheap
+         * enough, and it avoids a second on-disk format to keep consistent with the
+         * stores it describes.
+         */
+        this.cursors = new Map();
+        this.peer = peer;
+        this.host = host;
+        const settings = (0, types_4.normalizePeerSettings)(peer);
+        this.direction = settings.direction;
+        this.role = settings.role;
+        this.attachments = settings.attachments;
+        this.logger =
+            host.logger?.createChild(`Peer:${peer.name}`) ??
+                new logging_1.MindooLogger((0, logging_1.getDefaultLogLevel)(), `Peer:${peer.name}`, true);
+    }
+    get name() {
+        return this.peer.name;
+    }
+    get url() {
+        return this.peer.url;
+    }
+    /** Replace the peer config in place (peer CRUD) without dropping cursors. */
+    updatePeer(peer) {
+        this.peer = peer;
+        const settings = (0, types_4.normalizePeerSettings)(peer);
+        this.direction = settings.direction;
+        this.role = settings.role;
+        this.attachments = settings.attachments;
+        // Identity or endpoint may have changed; force a fresh session.
+        this.token = null;
+        this.tokenIssuedAt = 0;
+        void this.irohLink?.close();
+        this.irohLink = null;
+    }
+    /** Roles and endpoint, for `GET /system/cluster/topology`. */
+    topologyEntry() {
+        return {
+            name: this.peer.name,
+            url: this.peer.url ?? null,
+            direction: this.direction,
+            role: this.role,
+            attachments: this.attachments,
+            sessionActive: this.running && !this.paused && this.sessionState === "connected",
+        };
+    }
+    start() {
+        if (this.running)
+            return;
+        if (!this.peer.url) {
+            this.logger.debug(`No url configured, replication disabled`);
+            return;
+        }
+        if (this.direction === "disabled") {
+            this.sessionState = "disabled";
+            return;
+        }
+        if (!(0, types_4.shouldOpenSessionTo)(this.host.localRole, this.role)) {
+            // Spoke to spoke: the hub already carries this traffic. The trust entry
+            // stays so the peer can still authenticate inbound.
+            this.sessionState = "disabled";
+            return;
+        }
+        this.running = true;
+        this.sessionState = "connecting";
+        this.unsubscribeBus = this.host.eventBus.subscribe((event) => {
+            this.onLocalChange(event);
+        });
+        this.idleTimer = setInterval(() => {
+            void this.syncNow().catch(() => {
+                // Errors are already recorded on the status snapshot.
+            });
+        }, IDLE_SYNC_INTERVAL_MS);
+        this.idleTimer.unref?.();
+        void this.bootstrap();
+    }
+    async stop() {
+        this.running = false;
+        this.unsubscribeBus?.();
+        this.unsubscribeBus = null;
+        if (this.pushTimer)
+            clearTimeout(this.pushTimer);
+        if (this.idleTimer)
+            clearInterval(this.idleTimer);
+        if (this.reconnectTimer)
+            clearTimeout(this.reconnectTimer);
+        this.pushTimer = null;
+        this.idleTimer = null;
+        this.reconnectTimer = null;
+        this.sseAbort?.abort();
+        this.sseAbort = null;
+        this.eventsConnected = false;
+        void this.irohLink?.close();
+        this.irohLink = null;
+        this.sessionState = "disabled";
+        try {
+            await this.syncInFlight;
+        }
+        catch {
+            // A sync failing during shutdown is not interesting.
+        }
+    }
+    pause() {
+        this.paused = true;
+        this.sessionState = "paused";
+        this.sseAbort?.abort();
+        this.sseAbort = null;
+        this.eventsConnected = false;
+    }
+    resume() {
+        if (!this.paused)
+            return;
+        this.paused = false;
+        this.sessionState = "connecting";
+        this.reconnectAttempt = 0;
+        void this.bootstrap();
+    }
+    async bootstrap() {
+        if (!this.running || this.paused)
+            return;
+        try {
+            await this.refreshIntersection();
+            await this.syncNow();
+            this.startEventStream();
+            this.reconnectAttempt = 0;
+        }
+        catch (error) {
+            this.recordError(error);
+            this.scheduleReconnect();
+        }
+    }
+    scheduleReconnect() {
+        if (!this.running || this.paused || this.reconnectTimer)
+            return;
+        const delay = Math.min(RECONNECT_MAX_MS, RECONNECT_BASE_MS * 2 ** Math.min(this.reconnectAttempt, 6));
+        this.reconnectAttempt += 1;
+        this.sessionState = "backoff";
+        this.reconnectTimer = setTimeout(() => {
+            this.reconnectTimer = null;
+            void this.bootstrap();
+        }, delay);
+        this.reconnectTimer.unref?.();
+    }
+    // ---------------------------------------------------------------- session
+    async ensureToken() {
+        const fresh = this.token && Date.now() - this.tokenIssuedAt < PEER_TOKEN_REFRESH_MS;
+        if (fresh)
+            return this.token;
+        if (this.tokenPromise)
+            return this.tokenPromise;
+        this.tokenPromise = this.authenticate().finally(() => {
+            this.tokenPromise = null;
+        });
+        return this.tokenPromise;
+    }
+    async authenticate() {
+        const signingKey = await this.host.getSigningKey();
+        if (!signingKey) {
+            throw new Error("Server identity is locked; cannot authenticate with peer");
+        }
+        this.sessionState = "connecting";
+        // The peer looks us up by signing key, not by name: the key is the identity
+        // in `trusted-servers.json`, the name is only a label.
+        const challengeResponse = await this.peerFetch("/system/peer/challenge", {
+            method: "POST",
+            body: { publicsignkey: this.host.localSigningPublicKey },
+            authenticated: false,
+        });
+        const challenge = challengeResponse.challenge;
+        if (!challenge) {
+            throw new Error("Peer did not return a challenge");
+        }
+        const subtle = this.host.cryptoAdapter.getSubtle();
+        const signature = await subtle.sign({ name: "Ed25519" }, signingKey, (0, AuthChallenge_1.authChallengeSigningBytes)(challenge));
+        const authResponse = (await this.peerFetch("/system/peer/authenticate", {
+            method: "POST",
+            body: {
+                challenge,
+                signature: (0, base64_1.bytesToBase64)(new Uint8Array(signature)),
+            },
+            authenticated: false,
+        }));
+        if (!authResponse.success || !authResponse.token) {
+            throw new Error(authResponse.error || "Peer authentication failed");
+        }
+        this.token = authResponse.token;
+        this.tokenIssuedAt = Date.now();
+        this.sessionState = "connected";
+        return this.token;
+    }
+    isIrohPeer() {
+        return typeof this.peer.url === "string" && (0, irohLocator_1.isIrohLocator)(this.peer.url);
+    }
+    async getIrohLink() {
+        if (this.irohLink) {
+            return this.irohLink;
+        }
+        const io = this.host.getIrohStreamIO?.() ?? null;
+        if (!io) {
+            throw new Error(`Iroh peer ${this.peer.name} requires iroh.enabled and a ready Iroh endpoint`);
+        }
+        const parsed = (0, irohLocator_1.parseIrohLocator)(this.peer.url);
+        if (!parsed.ticket) {
+            throw new Error(`Iroh peer ${this.peer.name} url is an endpoint id only; store a full ticket to dial`);
+        }
+        if (!this.host.openIrohLink) {
+            throw new Error(`Iroh peer ${this.peer.name} cannot be dialled: Iroh is not available on this host`);
+        }
+        this.irohLink = this.host.openIrohLink(io, parsed.ticket);
+        return this.irohLink;
+    }
+    async peerFetch(path, init = {}) {
+        const base = this.peer.url;
+        if (!base)
+            throw new Error(`Peer ${this.peer.name} has no url`);
+        if (this.isIrohPeer()) {
+            return this.irohPeerFetch(path, init);
+        }
+        const headers = { "Content-Type": "application/json" };
+        if (init.authenticated !== false) {
+            headers.Authorization = `Bearer ${await this.ensureToken()}`;
+        }
+        const response = await fetch(`${base.replace(/\/$/, "")}${path}`, {
+            method: init.method ?? "GET",
+            headers,
+            body: init.body === undefined ? undefined : JSON.stringify(init.body),
+            signal: init.signal,
+        });
+        if (!response.ok) {
+            const text = await response.text().catch(() => "");
+            if (response.status === 401 || response.status === 403) {
+                // Force a re-handshake on the next call: the peer may have rotated its
+                // trusted-servers entry or restarted with a new challenge store.
+                this.token = null;
+            }
+            throw new Error(`Peer ${this.peer.name} ${path} failed (${response.status}): ${text}`);
+        }
+        return response.json();
+    }
+    async irohPeerFetch(path, init) {
+        const link = await this.getIrohLink();
+        const body = (init.body ?? {});
+        try {
+            if (path === "/system/peer/challenge") {
+                return await link.rpc("peer.challenge", [body.publicsignkey]);
+            }
+            if (path === "/system/peer/authenticate") {
+                const signature = body.signature;
+                const bytes = signature instanceof Uint8Array
+                    ? signature
+                    : (0, base64_1.base64ToBytes)(String(signature ?? ""));
+                return await link.rpc("peer.authenticate", [body.challenge, bytes]);
+            }
+            const token = init.authenticated !== false ? await this.ensureToken() : "";
+            if (path === "/system/peer/tenant-bloom") {
+                return await link.rpc("peer.tenantBloom", [token, body.bloom]);
+            }
+            if (path.startsWith("/system/peer/databases")) {
+                const tenantId = new URL(`http://peer${path}`).searchParams.get("tenantId") ?? "";
+                return await link.rpc("peer.listDatabases", [token, tenantId]);
+            }
+            if (path === "/system/peer/sync") {
+                return await link.rpc("peer.sync", [token, body]);
+            }
+            throw new Error(`No Iroh mapping for ${path}`);
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            if (/Invalid or expired peer token|Unknown peer|Authentication failed/i.test(message)) {
+                this.token = null;
+            }
+            throw new Error(`Peer ${this.peer.name} ${path} failed: ${message}`);
+        }
+    }
+    // ----------------------------------------------------------- intersection
+    /**
+     * Recompute the set of tenants both servers host.
+     *
+     * We send a Bloom summary of our tenant ids; the peer answers with the ids on
+     * *its* side that the filter might contain. Those are candidates only — a
+     * Bloom filter has false positives — so the result is intersected against the
+     * authoritative local list before anything is replicated. That second step is
+     * what makes the false-positive rate a performance question rather than a
+     * correctness one.
+     */
+    async refreshIntersection() {
+        const localTenants = this.host.listTenants();
+        const localSet = new Set(localTenants);
+        const summary = (0, bloom_1.createIdBloomSummary)(localTenants);
+        const response = (await this.peerFetch("/system/peer/tenant-bloom", {
+            method: "POST",
+            body: { bloom: summary },
+        }));
+        const candidates = Array.isArray(response.tenantIds) ? response.tenantIds : [];
+        const shared = candidates.filter((tenantId) => localSet.has(tenantId)).sort();
+        this.intersection = shared;
+        this.intersectionAt = new Date().toISOString();
+        for (const tenantId of this.tenants.keys()) {
+            if (!shared.includes(tenantId))
+                this.tenants.delete(tenantId);
+        }
+        for (const tenantId of shared) {
+            if (!this.tenants.has(tenantId)) {
+                this.tenants.set(tenantId, {
+                    tenantId,
+                    databases: [],
+                    lastSyncAt: null,
+                    lag: 0,
+                    retryQueueDepth: 0,
+                    rejectedEntries: 0,
+                    errorClass: null,
+                    suppressUpToPeerOrder: 0,
+                });
+            }
+        }
+        return shared;
+    }
+    // ------------------------------------------------------------------- sync
+    /**
+     * Run one full replication pass, optionally narrowed to a tenant or database.
+     *
+     * Concurrent calls collapse onto the in-flight run: the caller that arrives
+     * second gets the same promise rather than a second parallel mirror of the
+     * same stores.
+     */
+    async syncNow(scope) {
+        if (this.syncInFlight)
+            return this.syncInFlight;
+        this.syncInFlight = this.runSync(scope).finally(() => {
+            this.syncInFlight = null;
+        });
+        return this.syncInFlight;
+    }
+    /**
+     * Re-offer entries the peer rejected.
+     *
+     * Under the hold policy the cursor is already clamped, so a plain sync run is
+     * the retry — this exists as a named admin action so the UI has something to
+     * call that says what it does.
+     */
+    async retryRejected(scope) {
+        return this.syncNow(scope);
+    }
+    async runSync(scope) {
+        const result = {
+            peer: this.peer.name,
+            tenants: 0,
+            databases: 0,
+            pushed: 0,
+            pulled: 0,
+            rejected: 0,
+            held: 0,
+            errors: [],
+        };
+        if (this.paused || this.direction === "disabled" || !this.peer.url) {
+            return result;
+        }
+        if (this.intersection.length === 0) {
+            await this.refreshIntersection().catch((error) => {
+                this.recordError(error);
+                result.errors.push(errorMessage(error));
+            });
+        }
+        let tenantIds = scope?.tenantId
+            ? this.intersection.filter((id) => id === scope.tenantId)
+            : this.intersection;
+        const resumeAt = this.resumeTenantId ? tenantIds.indexOf(this.resumeTenantId) : -1;
+        if (resumeAt > 0) {
+            tenantIds = [...tenantIds.slice(resumeAt), ...tenantIds.slice(0, resumeAt)];
+        }
+        const pastDeadline = () => scope?.deadline !== undefined && Date.now() >= scope.deadline;
+        for (const tenantId of tenantIds) {
+            if (pastDeadline()) {
+                result.deferred = true;
+                this.resumeTenantId = tenantId;
+                break;
+            }
+            const runtime = this.tenants.get(tenantId);
+            if (!runtime)
+                continue;
+            runtime.rejectedEntries = 0;
+            runtime.retryQueueDepth = 0;
+            runtime.errorClass = null;
+            try {
+                const databases = await this.databasesFor(tenantId, scope?.dbId);
+                // Only a full-tenant run knows the complete list; a scoped run must not
+                // shrink what the status endpoint reports.
+                if (!scope?.dbId)
+                    runtime.databases = databases;
+                result.tenants += 1;
+                for (const [dbId, storeKind] of this.mirrorPlan(databases)) {
+                    if (pastDeadline()) {
+                        result.deferred = true;
+                        break;
+                    }
+                    if (storeKind === types_1.StoreKind.docs)
+                        result.databases += 1;
+                    const pair = await this.syncStorePair(tenantId, dbId, storeKind);
+                    result.pushed += pair.pushed;
+                    result.pulled += pair.pulled;
+                    result.rejected += pair.rejected;
+                    runtime.rejectedEntries += pair.rejected;
+                    this.transferredEntries += pair.pushed + pair.pulled;
+                    this.transferredBytes += pair.bytes;
+                    if (pair.held) {
+                        runtime.retryQueueDepth += 1;
+                        result.held += 1;
+                    }
+                    if (pair.deniedReason) {
+                        // Visible in the status read-model as a backoff rather than an
+                        // outage, so an operator can tell "waiting on a purge that has not
+                        // replicated yet" from "this peer is unreachable".
+                        runtime.errorClass = "access-denied";
+                        this.lastError = pair.deniedReason;
+                        this.lastErrorClass = "access-denied";
+                        this.lastErrorAt = new Date().toISOString();
+                    }
+                }
+                if (result.deferred) {
+                    this.resumeTenantId = tenantId;
+                    break;
+                }
+                runtime.lastSyncAt = new Date().toISOString();
+            }
+            catch (error) {
+                const message = errorMessage(error);
+                runtime.errorClass = (0, types_4.classifyPeerError)(error);
+                result.errors.push(`${tenantId}: ${message}`);
+                this.recordError(error);
+            }
+        }
+        if (!result.deferred && !scope?.tenantId) {
+            this.resumeTenantId = null;
+        }
+        this.lastSyncAt = new Date().toISOString();
+        // A held pair is not a clean run: the backoff has to stay visible until the
+        // entries actually land, or the status flips to healthy while the mirror is
+        // still incomplete.
+        if (result.errors.length === 0 && result.held === 0) {
+            this.lastError = null;
+            this.lastErrorClass = null;
+            this.lastErrorAt = null;
+            this.sessionState = "connected";
+        }
+        return result;
+    }
+    /**
+     * The (database, store) pairs to mirror for one tenant, in order.
+     *
+     * `eager` interleaves — each database's documents are immediately followed by
+     * its attachments, so a database is complete before the next one starts.
+     * `lazy` runs every document store first and only then the attachment stores:
+     * a run that is interrupted (or a link that is slow) has then spent its
+     * bandwidth on the text, which is what makes documents readable, rather than
+     * on blobs that can be fetched later. `never` skips attachments entirely.
+     *
+     * Database order comes from {@link orderDatabasesForReplication} either way —
+     * invariant 1 outranks the attachment policy.
+     */
+    *mirrorPlan(databases) {
+        if (this.attachments === "eager") {
+            for (const dbId of databases) {
+                yield [dbId, types_1.StoreKind.docs];
+                yield [dbId, types_1.StoreKind.attachments];
+            }
+            return;
+        }
+        for (const dbId of databases) {
+            yield [dbId, types_1.StoreKind.docs];
+        }
+        if (this.attachments === "never")
+            return;
+        for (const dbId of databases) {
+            yield [dbId, types_1.StoreKind.attachments];
+        }
+    }
+    /**
+     * Databases to mirror for a tenant: the union of both sides, in invariant
+     * order (invariant 1).
+     *
+     * Union rather than intersection, because a database that exists only on the
+     * peer is exactly the thing a fresh mirror needs to pull.
+     */
+    async databasesFor(tenantId, onlyDbId) {
+        const local = await this.host.listDatabases(tenantId);
+        let remote = [];
+        try {
+            const response = (await this.peerFetch(`/system/peer/databases?tenantId=${encodeURIComponent(tenantId)}`));
+            remote = Array.isArray(response.databases) ? response.databases : [];
+        }
+        catch (error) {
+            this.logger.debug(`Could not list peer databases for ${tenantId}: ${errorMessage(error)}`);
+        }
+        const union = (0, types_4.orderDatabasesForReplication)(new Set([...local, ...remote]));
+        return onlyDbId ? union.filter((dbId) => dbId === onlyDbId) : union;
+    }
+    async syncStorePair(tenantId, dbId, storeKind) {
+        const localStore = await this.host.getLocalStore(tenantId, dbId, storeKind);
+        const remoteStore = await this.createRemoteStore(tenantId, dbId, storeKind);
+        let pushed = 0;
+        let pulled = 0;
+        let bytes = 0;
+        let rejected = 0;
+        let held = false;
+        let deniedReason;
+        // Both directions use `rejectionPolicy: "hold"` (invariant 2). A client may
+        // skip a poisoned entry forever; two servers that are meant to be identical
+        // may not, so the cursor stays clamped until the entry is accepted.
+        //
+        // A denial can also arrive for the batch as a whole rather than per entry —
+        // a purged document or a revoked decryption key makes the target refuse
+        // everything in the request. That is the same condition seen from further
+        // away: hold this pair, leave the cursor where it is, and carry on with the
+        // tenant's other databases. Treating it as a tenant-level failure would let
+        // one purged document stop the whole mirror.
+        if ((0, types_4.directionAllowsPull)(this.direction)) {
+            const getPurgedDocIds = this.host.getPurgedDocIds?.bind(this.host);
+            const pullTarget = dbId === types_1.DIRECTORY_DB_ID || !getPurgedDocIds
+                ? localStore
+                : (0, purgedDocumentGuard_1.guardStoreAgainstPurgedDocuments)(localStore, async () => getPurgedDocIds(tenantId, dbId), this.logger);
+            const result = await this.transfer(remoteStore, pullTarget);
+            pulled = result.transferred;
+            bytes += result.bytes;
+            rejected += result.rejected;
+            held || (held = result.held);
+            deniedReason ?? (deniedReason = result.deniedReason);
+            // The directory is mirrored first, so purges it carries are executed
+            // (and the registry armed) before any database they apply to.
+            if (dbId === types_1.DIRECTORY_DB_ID &&
+                storeKind === types_1.StoreKind.docs &&
+                pulled > 0 &&
+                this.host.onDirectoryPulled) {
+                try {
+                    await this.host.onDirectoryPulled(tenantId);
+                }
+                catch (error) {
+                    this.logger.error(`Executing replicated purge requests for ${tenantId} failed: ${errorMessage(error)}`);
+                }
+            }
+        }
+        if ((0, types_4.directionAllowsPush)(this.direction)) {
+            const result = await this.transfer(localStore, remoteStore);
+            pushed = result.transferred;
+            bytes += result.bytes;
+            rejected += result.rejected;
+            held || (held = result.held);
+            deniedReason ?? (deniedReason = result.deniedReason);
+            if (pushed > 0) {
+                // Remember how far the peer's log advanced because of us, so its SSE
+                // notification for these very entries is recognised as our own echo.
+                const runtime = this.tenants.get(tenantId);
+                const head = await remoteStore.getStoreHead?.().catch(() => null);
+                if (runtime && head?.maxReceiptOrder !== undefined) {
+                    runtime.suppressUpToPeerOrder = Math.max(runtime.suppressUpToPeerOrder, head.maxReceiptOrder);
+                }
+            }
+        }
+        return { pushed, pulled, bytes, rejected, held, deniedReason };
+    }
+    /**
+     * One leg of a store pair, with a batch-wide denial folded into `held`.
+     *
+     * See {@link syncStorePair}: a denial means "not now", so the cursor must
+     * stay put and the pair is retried on the next run. Any other failure is a
+     * real error and propagates to the tenant loop.
+     */
+    async transfer(source, target) {
+        try {
+            const limits = this.host.transferLimits;
+            const result = await (0, syncStores_1.syncEntriesBetweenStores)(source, target, undefined, {
+                logger: this.logger,
+                cursors: this.cursorStore(),
+                rejectionPolicy: "hold",
+                ...(limits?.maxConcurrentBatches ? { maxConcurrentBatches: limits.maxConcurrentBatches } : {}),
+                ...(limits
+                    ? {
+                        transferBatchSize: source.getStoreKind() === types_1.StoreKind.attachments
+                            ? limits.attachmentsBatchSize
+                            : limits.docsBatchSize,
+                    }
+                    : {}),
+            });
+            // Entries the cursor moved past will not be offered again — either
+            // because the refusal can never pass, or because it was supposed to and
+            // did not within the retry budget. This log line is the only place they
+            // are announced: the counters below make them visible in the status
+            // read-model, but not why.
+            const final = result.abandoned ?? [];
+            if (final.length > 0) {
+                const byClass = new Map();
+                for (const entry of final) {
+                    const key = (0, types_3.rejectionClassOf)(entry);
+                    byClass.set(key, (byClass.get(key) ?? 0) + 1);
+                }
+                const breakdown = [...byClass]
+                    .map(([cls, count]) => `${count} ${cls}`)
+                    .join(", ");
+                this.logger.warn(`Peer refused ${final.length} entr${final.length === 1 ? "y" : "ies"} for good ` +
+                    `(${breakdown}); moving the cursor past them. First reason: ` +
+                    (0, types_4.redactEntryIds)(final[0].reason));
+            }
+            return {
+                transferred: result.transferred,
+                bytes: result.transferredBytes ?? 0,
+                rejected: result.rejected?.length ?? 0,
+                held: result.cursorHeldForRetry === true,
+            };
+        }
+        catch (error) {
+            if ((0, types_4.classifyPeerError)(error) !== "access-denied")
+                throw error;
+            this.logger.warn(`Peer refused a batch, holding for retry: ${errorMessage(error)}`);
+            return {
+                transferred: 0,
+                bytes: 0,
+                rejected: 0,
+                held: true,
+                deniedReason: (0, types_4.redactEntryIds)(errorMessage(error)),
+            };
+        }
+    }
+    async createRemoteStore(tenantId, dbId, storeKind) {
+        const encryptionKey = await this.host.getEncryptionKey();
+        if (!encryptionKey) {
+            throw new Error("Server identity is locked; cannot decrypt peer entries");
+        }
+        const signingKey = await this.host.getSigningKey();
+        if (!signingKey) {
+            throw new Error("Server identity is locked; cannot sign for peer");
+        }
+        // The tenant segment belongs in the base url, not just in the config: the
+        // transport builds `${baseUrl}/sync/${storeKind}/...` and the server routes
+        // sync under `/:tenantId/sync/*`. Passing the bare server url — as the old
+        // per-tenant ServerSync did — makes every request land on `/sync/...`,
+        // where Express reads "sync" as the tenant id and answers 404.
+        const transport = this.isIrohPeer()
+            ? new IrohNetworkTransport_1.IrohNetworkTransport((await this.getIrohLink()).asStreamIO(), (0, irohLocator_1.parseIrohLocator)(this.peer.url).ticket, {
+                tenantId,
+                dbId,
+                storeKind,
+            })
+            : new HttpTransport_1.HttpTransport({
+                baseUrl: `${this.peer.url.replace(/\/$/, "")}/${encodeURIComponent(tenantId)}`,
+                tenantId,
+                dbId,
+                storeKind,
+            });
+        const store = new ClientNetworkContentAddressedStore_1.ClientNetworkContentAddressedStore(dbId, storeKind, transport, this.host.cryptoAdapter, this.host.localServerName, signingKey, encryptionKey, this.logger);
+        // The tenant handshake would fail: a peer holds no per-tenant grant. It
+        // carries the cluster-level peer token instead.
+        store.setExternalTokenProvider(() => this.ensureToken());
+        return store;
+    }
+    cursorStore() {
+        return {
+            get: (key) => this.cursors.get(key) ?? null,
+            save: (key, record) => {
+                this.cursors.set(key, record);
+            },
+            delete: (key) => {
+                this.cursors.delete(key);
+            },
+        };
+    }
+    // ------------------------------------------------------------- live push
+    acceptsLocalChange(event) {
+        if (!this.running || this.paused)
+            return false;
+        if (!(0, types_4.directionAllowsPush)(this.direction))
+            return false;
+        // Do not bounce a peer's own write straight back at it. Other peers still
+        // receive it, which is how a spoke's write reaches the mesh via its hub.
+        if (event.originPeer === this.peer.name)
+            return false;
+        if (!this.intersection.includes(event.tenantId))
+            return false;
+        if (this.attachments === "never" && event.storeKind === types_1.StoreKind.attachments)
+            return false;
+        return true;
+    }
+    /**
+     * Push a local change now and resolve when done, instead of coalescing it
+     * on a timer. For hosts that cannot rely on timers outliving the request
+     * that reported the change.
+     */
+    async replicateLocalChange(event) {
+        if (!this.acceptsLocalChange(event))
+            return;
+        try {
+            await this.syncNow({ tenantId: event.tenantId, dbId: event.dbId });
+        }
+        catch (error) {
+            this.recordError(error);
+        }
+    }
+    onLocalChange(event) {
+        if (!this.acceptsLocalChange(event))
+            return;
+        this.pendingScopes.add(`${event.tenantId}\u0000${event.dbId}`);
+        if (this.pushTimer)
+            return;
+        this.pushTimer = setTimeout(() => {
+            this.pushTimer = null;
+            const scopes = [...this.pendingScopes];
+            this.pendingScopes.clear();
+            void this.flushScopes(scopes);
+        }, PUSH_COALESCE_MS);
+        this.pushTimer.unref?.();
+    }
+    /**
+     * Round-robin over the pending (tenant, db) scopes.
+     *
+     * Sequential on purpose: a hub with many spokes should not open a hundred
+     * parallel sessions to one peer, and one busy tenant must not starve the
+     * others — each pending scope gets its turn in the order it was queued.
+     */
+    async flushScopes(scopes) {
+        for (const scope of scopes) {
+            const [tenantId, dbId] = scope.split("\u0000");
+            try {
+                await this.syncNow({ tenantId, dbId });
+            }
+            catch (error) {
+                this.recordError(error);
+            }
+        }
+    }
+    // -------------------------------------------------------------- SSE pull
+    startEventStream() {
+        if (this.host.liveEventStream === false)
+            return;
+        if (!this.running || this.paused)
+            return;
+        if (!(0, types_4.directionAllowsPull)(this.direction))
+            return;
+        if (this.sseAbort)
+            return;
+        const controller = new AbortController();
+        this.sseAbort = controller;
+        void this.consumeEventStream(controller).catch((error) => {
+            if (!controller.signal.aborted) {
+                this.recordError(error);
+                this.scheduleReconnect();
+            }
+        });
+    }
+    async consumeEventStream(controller) {
+        if (this.isIrohPeer()) {
+            await this.consumeIrohEventStream(controller);
+            return;
+        }
+        const base = this.peer.url;
+        const token = await this.ensureToken();
+        const response = await fetch(`${base.replace(/\/$/, "")}/system/peer/events`, {
+            headers: { Authorization: `Bearer ${token}`, Accept: "text/event-stream" },
+            signal: controller.signal,
+        });
+        if (!response.ok || !response.body) {
+            throw new Error(`Peer event stream failed (${response.status})`);
+        }
+        this.sessionState = "connected";
+        this.eventsConnected = true;
+        this.reconnectAttempt = 0;
+        const reader = response.body.getReader();
+        const decoder = new TextDecoder();
+        let buffer = "";
+        while (!controller.signal.aborted) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            buffer += decoder.decode(value, { stream: true });
+            let separator = buffer.indexOf("\n\n");
+            while (separator !== -1) {
+                const frame = buffer.slice(0, separator);
+                buffer = buffer.slice(separator + 2);
+                this.handleEventFrame(frame);
+                separator = buffer.indexOf("\n\n");
+            }
+        }
+        if (this.sseAbort === controller) {
+            this.sseAbort = null;
+            this.eventsConnected = false;
+            // The peer closed the stream. Catch up once (entries may have landed
+            // while we were disconnected) and then re-subscribe.
+            if (this.running && !this.paused) {
+                void this.syncNow().catch(() => undefined);
+                this.scheduleReconnect();
+            }
+        }
+    }
+    async consumeIrohEventStream(controller) {
+        const link = await this.getIrohLink();
+        const token = await this.ensureToken();
+        this.sessionState = "connected";
+        this.eventsConnected = true;
+        this.reconnectAttempt = 0;
+        try {
+            await link.subscribeEvents(token, (event) => this.handleChangeEvent(event), controller.signal);
+        }
+        finally {
+            if (this.sseAbort === controller) {
+                this.sseAbort = null;
+                this.eventsConnected = false;
+                if (this.running && !this.paused && !controller.signal.aborted) {
+                    void this.syncNow().catch(() => undefined);
+                    this.scheduleReconnect();
+                }
+            }
+        }
+    }
+    handleEventFrame(frame) {
+        const dataLine = frame
+            .split("\n")
+            .find((line) => line.startsWith("data:"));
+        if (!dataLine)
+            return;
+        let event;
+        try {
+            event = JSON.parse(dataLine.slice(5).trim());
+        }
+        catch {
+            return;
+        }
+        this.handleChangeEvent(event);
+    }
+    handleChangeEvent(event) {
+        if (!event.tenantId || !event.dbId)
+            return;
+        if (!this.intersection.includes(event.tenantId))
+            return;
+        if (this.attachments === "never" && event.storeKind === types_1.StoreKind.attachments)
+            return;
+        // Echo suppression: this notification describes entries we ourselves just
+        // pushed, so pulling them back would be a round trip for nothing.
+        const runtime = this.tenants.get(event.tenantId);
+        if (runtime &&
+            event.maxReceiptOrder !== undefined &&
+            event.maxReceiptOrder <= runtime.suppressUpToPeerOrder) {
+            return;
+        }
+        this.pendingScopes.add(`${event.tenantId}\u0000${event.dbId}`);
+        if (this.pushTimer)
+            return;
+        this.pushTimer = setTimeout(() => {
+            this.pushTimer = null;
+            const scopes = [...this.pendingScopes];
+            this.pendingScopes.clear();
+            void this.flushScopes(scopes);
+        }, PUSH_COALESCE_MS);
+        this.pushTimer.unref?.();
+    }
+    // ------------------------------------------------------------- reporting
+    recordError(error) {
+        this.lastError = (0, types_4.redactEntryIds)(errorMessage(error));
+        this.lastErrorClass = (0, types_4.classifyPeerError)(error);
+        this.lastErrorAt = new Date().toISOString();
+        if (this.lastErrorClass === "network") {
+            this.sessionState = "unreachable";
+        }
+        else if (this.sessionState !== "paused" && this.sessionState !== "disabled") {
+            this.sessionState = "backoff";
+        }
+        this.logger.warn(`Replication error: ${this.lastError}`);
+    }
+    /**
+     * Status snapshot for `GET /system/cluster/status`.
+     *
+     * Read straight from in-memory replication state — no store scans — so the
+     * admin UI can poll it without competing with replication for I/O. Contains
+     * counts and classified reasons only: never a docId, never an author key.
+     */
+    snapshot() {
+        const tenants = [...this.tenants.values()].map((runtime) => this.tenantStatus(runtime));
+        const { health, reasons } = rollUpHealth(tenants, this.peerLevelReasons());
+        return {
+            name: this.peer.name,
+            url: this.peer.url ?? null,
+            direction: this.direction,
+            role: this.role,
+            attachments: this.attachments,
+            state: this.sessionState,
+            health,
+            reasons,
+            sharedTenants: this.intersection.length,
+            lastIntersectionAt: this.intersectionAt,
+            lastSyncAt: this.lastSyncAt,
+            eventsConnected: this.eventsConnected,
+            lag: tenants.reduce((sum, tenant) => sum + tenant.lag, 0),
+            retryQueueDepth: tenants.reduce((sum, tenant) => sum + tenant.retryQueueDepth, 0),
+            rejectedEntries: tenants.reduce((sum, tenant) => sum + tenant.rejectedEntries, 0),
+            transferredEntries: this.transferredEntries,
+            transferredBytes: this.transferredBytes,
+            lastError: this.lastError,
+            lastErrorClass: this.lastErrorClass,
+            lastErrorAt: this.lastErrorAt,
+        };
+    }
+    tenantSnapshot(tenantId) {
+        const runtime = this.tenants.get(tenantId);
+        return runtime ? this.tenantStatus(runtime) : null;
+    }
+    tenantStatus(runtime) {
+        const reasons = [];
+        if (runtime.retryQueueDepth > 0)
+            reasons.push("entries-awaiting-directory-trust");
+        if (runtime.errorClass === "access-denied")
+            reasons.push("access-denied-backoff");
+        const { health } = rollUpHealth([], reasons);
+        return {
+            tenantId: runtime.tenantId,
+            databases: runtime.databases,
+            lastSyncAt: runtime.lastSyncAt,
+            lag: runtime.lag,
+            retryQueueDepth: runtime.retryQueueDepth,
+            rejectedEntries: runtime.rejectedEntries,
+            health,
+            reasons,
+        };
+    }
+    /** Reasons that come from the link itself rather than from any one tenant. */
+    peerLevelReasons() {
+        const reasons = [];
+        if (!this.peer.url)
+            reasons.push("no-url-configured");
+        if (this.direction === "disabled")
+            reasons.push("session-disabled");
+        if (this.paused)
+            reasons.push("session-paused");
+        if (this.sessionState === "unreachable")
+            reasons.push("peer-unreachable");
+        if (this.lastErrorClass === "auth")
+            reasons.push("peer-auth-failed");
+        if (this.running &&
+            !this.paused &&
+            !this.eventsConnected &&
+            (0, types_4.directionAllowsPull)(this.direction)) {
+            reasons.push("events-disconnected");
+        }
+        if (this.intersectionAt === null && this.running)
+            reasons.push("intersection-stale");
+        return reasons;
+    }
+}
+exports.PeerReplicator = PeerReplicator;
+/**
+ * Collapse reasons into one health verdict, shared by the per-tenant, per-peer
+ * and per-node rollups so all three agree on the thresholds.
+ *
+ * `stalled` outranks `degraded` deliberately. A clamped cursor is the visible
+ * precursor of divergence: the link looks healthy, entries keep flowing, and
+ * yet one database is quietly stuck behind an entry the peer will not accept
+ * until the directory catches up. That is the state an operator most needs to
+ * see, so a transient error elsewhere must not mask it.
+ */
+function rollUpHealth(tenants, extraReasons = []) {
+    const reasons = new Set(extraReasons);
+    for (const tenant of tenants) {
+        for (const reason of tenant.reasons)
+            reasons.add(reason);
+    }
+    let health = "ok";
+    if (reasons.has("session-disabled") || reasons.has("session-paused")) {
+        // Deliberately off is not unhealthy; it is just not running.
+        health = "ok";
+    }
+    else if (reasons.has("peer-unreachable") ||
+        reasons.has("peer-auth-failed") ||
+        reasons.has("no-url-configured")) {
+        health = "down";
+    }
+    else if (reasons.has("entries-awaiting-directory-trust")) {
+        health = "stalled";
+    }
+    else if (reasons.size > 0) {
+        health = "degraded";
+    }
+    return { health, reasons: [...reasons] };
+}
+function errorMessage(error) {
+    if (error instanceof types_2.NetworkError)
+        return `${error.type}: ${error.message}`;
+    return error instanceof Error ? error.message : String(error);
+}
+//# sourceMappingURL=PeerReplicator.js.map