mindoodb 0.0.53 → 0.0.55

package/dist/node/server/peer/PeerReplicator.d.ts

@@ -0,0 +1,232 @@
+/**
+ * Server-to-server replication with one trusted peer.
+ *
+ * One replicator instance per entry in `trusted-servers.json` that carries a
+ * `url`. It owns everything about that link: the peer session, the set of
+ * tenants both sides hold, the mirror loop over every database, the live push
+ * out of {@link SyncEventBus}, and the SSE subscription that pulls the other
+ * direction.
+ *
+ * ## What it replicates
+ *
+ * Everything both servers already have in common, and nothing else. The tenant
+ * set is intersected via a Bloom summary (see {@link refreshIntersection}), so
+ * a server never learns which tenants it does *not* host on the other side. For
+ * each shared tenant it mirrors **all** databases — the union of the local and
+ * remote database lists — in both stores (`docs` and `attachments`).
+ *
+ * ## Convergence invariants
+ *
+ * 1. **Directory first.** Databases are ordered by
+ *    {@link orderDatabasesForReplication}: `directory`, then `userdirectory`,
+ *    then the rest. An application entry signed by a key the peer does not
+ *    trust yet is rejected per entry, so the grants have to land first.
+ * 2. **Cursor holds on rejection.** Unlike a client, a peer must not skip past
+ *    a rejected entry — that is permanent divergence between two servers that
+ *    are supposed to be identical. The peer path passes
+ *    `rejectionPolicy: "hold"`, which clamps the cursor so the next run
+ *    re-offers the entry once the missing trust has replicated.
+ * 3. **Receipts are preserved, not re-stamped.** Enforced on the receiving side
+ *    in `ServerNetworkContentAddressedStore`; this side just transports.
+ */
+import { StoreKind, type ContentAddressedStore } from "../../../core/types";
+import type { CryptoAdapter } from "../../../core/crypto/CryptoAdapter";
+import { Logger } from "../../../core/logging";
+import type { TrustedServer } from "../types";
+import type { SyncEventBus } from "../SyncEventBus";
+import { type ClusterHealth, type ClusterHealthReason, type ClusterPeerStatus, type ClusterTenantStatus, type ClusterTopologyPeer, type PeerRole } from "./types";
+/** What a replicator needs from the surrounding server to do its job. */
+export interface PeerReplicatorHost {
+    cryptoAdapter: CryptoAdapter;
+    /** This server's name, as the peer knows it in its own trusted-servers list. */
+    localServerName: string;
+    /** This server's Ed25519 public key (PEM) — how the peer looks us up. */
+    localSigningPublicKey: string;
+    /** This node's own mesh role, which decides whether spoke-to-spoke is suppressed. */
+    localRole: PeerRole;
+    /** Ed25519 signing key, for the peer challenge. */
+    getSigningKey(): Promise<CryptoKey | undefined>;
+    /** RSA private key, to unwrap entries the peer sends us. */
+    getEncryptionKey(): Promise<CryptoKey | undefined>;
+    /** Tenant ids hosted locally. */
+    listTenants(): string[];
+    /** Database ids held locally for a tenant. */
+    listDatabases(tenantId: string): string[];
+    /** Local store for a (tenant, db, kind) triple, created on demand. */
+    getLocalStore(tenantId: string, dbId: string, storeKind: StoreKind): Promise<ContentAddressedStore>;
+    eventBus: SyncEventBus;
+    logger?: Logger;
+}
+export interface PeerSyncScope {
+    tenantId?: string;
+    dbId?: string;
+}
+export interface PeerSyncRunResult {
+    peer: string;
+    tenants: number;
+    databases: number;
+    pushed: number;
+    pulled: number;
+    rejected: number;
+    /** Pairs whose cursor is clamped waiting for trust to replicate (invariant 2). */
+    held: number;
+    errors: string[];
+}
+export declare class PeerReplicator {
+    private peer;
+    private readonly host;
+    private readonly logger;
+    private direction;
+    private role;
+    private attachments;
+    private running;
+    private paused;
+    private sessionState;
+    private eventsConnected;
+    private token;
+    private tokenIssuedAt;
+    private tokenPromise;
+    private intersection;
+    private intersectionAt;
+    private tenants;
+    private transferredEntries;
+    private transferredBytes;
+    private syncInFlight;
+    private pendingScopes;
+    private pushTimer;
+    private idleTimer;
+    private reconnectTimer;
+    private reconnectAttempt;
+    private sseAbort;
+    private unsubscribeBus;
+    private lastError;
+    private lastErrorClass;
+    private lastErrorAt;
+    private lastSyncAt;
+    /**
+     * Scan cursors, per (source, target) store pair.
+     *
+     * In-process only, deliberately: a restarted server pays one metadata scan
+     * per pair and then re-establishes the fast path from the store head. Cheap
+     * enough, and it avoids a second on-disk format to keep consistent with the
+     * stores it describes.
+     */
+    private readonly cursors;
+    constructor(peer: TrustedServer, host: PeerReplicatorHost);
+    get name(): string;
+    get url(): string | undefined;
+    /** Replace the peer config in place (peer CRUD) without dropping cursors. */
+    updatePeer(peer: TrustedServer): void;
+    /** Roles and endpoint, for `GET /system/cluster/topology`. */
+    topologyEntry(): ClusterTopologyPeer;
+    start(): void;
+    stop(): Promise<void>;
+    pause(): void;
+    resume(): void;
+    private bootstrap;
+    private scheduleReconnect;
+    private ensureToken;
+    private authenticate;
+    private peerFetch;
+    /**
+     * Recompute the set of tenants both servers host.
+     *
+     * We send a Bloom summary of our tenant ids; the peer answers with the ids on
+     * *its* side that the filter might contain. Those are candidates only — a
+     * Bloom filter has false positives — so the result is intersected against the
+     * authoritative local list before anything is replicated. That second step is
+     * what makes the false-positive rate a performance question rather than a
+     * correctness one.
+     */
+    refreshIntersection(): Promise<string[]>;
+    /**
+     * Run one full replication pass, optionally narrowed to a tenant or database.
+     *
+     * Concurrent calls collapse onto the in-flight run: the caller that arrives
+     * second gets the same promise rather than a second parallel mirror of the
+     * same stores.
+     */
+    syncNow(scope?: PeerSyncScope): Promise<PeerSyncRunResult>;
+    /**
+     * Re-offer entries the peer rejected.
+     *
+     * Under the hold policy the cursor is already clamped, so a plain sync run is
+     * the retry — this exists as a named admin action so the UI has something to
+     * call that says what it does.
+     */
+    retryRejected(scope?: PeerSyncScope): Promise<PeerSyncRunResult>;
+    private runSync;
+    /**
+     * The (database, store) pairs to mirror for one tenant, in order.
+     *
+     * `eager` interleaves — each database's documents are immediately followed by
+     * its attachments, so a database is complete before the next one starts.
+     * `lazy` runs every document store first and only then the attachment stores:
+     * a run that is interrupted (or a link that is slow) has then spent its
+     * bandwidth on the text, which is what makes documents readable, rather than
+     * on blobs that can be fetched later. `never` skips attachments entirely.
+     *
+     * Database order comes from {@link orderDatabasesForReplication} either way —
+     * invariant 1 outranks the attachment policy.
+     */
+    private mirrorPlan;
+    /**
+     * Databases to mirror for a tenant: the union of both sides, in invariant
+     * order (invariant 1).
+     *
+     * Union rather than intersection, because a database that exists only on the
+     * peer is exactly the thing a fresh mirror needs to pull.
+     */
+    private databasesFor;
+    private syncStorePair;
+    /**
+     * One leg of a store pair, with a batch-wide denial folded into `held`.
+     *
+     * See {@link syncStorePair}: a denial means "not now", so the cursor must
+     * stay put and the pair is retried on the next run. Any other failure is a
+     * real error and propagates to the tenant loop.
+     */
+    private transfer;
+    private createRemoteStore;
+    private cursorStore;
+    private onLocalChange;
+    /**
+     * Round-robin over the pending (tenant, db) scopes.
+     *
+     * Sequential on purpose: a hub with many spokes should not open a hundred
+     * parallel sessions to one peer, and one busy tenant must not starve the
+     * others — each pending scope gets its turn in the order it was queued.
+     */
+    private flushScopes;
+    private startEventStream;
+    private consumeEventStream;
+    private handleEventFrame;
+    private recordError;
+    /**
+     * Status snapshot for `GET /system/cluster/status`.
+     *
+     * Read straight from in-memory replication state — no store scans — so the
+     * admin UI can poll it without competing with replication for I/O. Contains
+     * counts and classified reasons only: never a docId, never an author key.
+     */
+    snapshot(): ClusterPeerStatus;
+    tenantSnapshot(tenantId: string): ClusterTenantStatus | null;
+    private tenantStatus;
+    /** Reasons that come from the link itself rather than from any one tenant. */
+    private peerLevelReasons;
+}
+/**
+ * Collapse reasons into one health verdict, shared by the per-tenant, per-peer
+ * and per-node rollups so all three agree on the thresholds.
+ *
+ * `stalled` outranks `degraded` deliberately. A clamped cursor is the visible
+ * precursor of divergence: the link looks healthy, entries keep flowing, and
+ * yet one database is quietly stuck behind an entry the peer will not accept
+ * until the directory catches up. That is the state an operator most needs to
+ * see, so a transient error elsewhere must not mask it.
+ */
+export declare function rollUpHealth(tenants: ClusterTenantStatus[], extraReasons?: ClusterHealthReason[]): {
+    health: ClusterHealth;
+    reasons: ClusterHealthReason[];
+};
+//# sourceMappingURL=PeerReplicator.d.ts.map

package/dist/node/server/peer/PeerReplicator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"PeerReplicator.d.ts","sourceRoot":"","sources":["../../../../src/node/server/peer/PeerReplicator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAIH,OAAO,EAAE,SAAS,EAAE,KAAK,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAE5E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oCAAoC,CAAC;AACxE,OAAO,EAAE,MAAM,EAAoC,MAAM,uBAAuB,CAAC;AAQjF,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAmB,MAAM,iBAAiB,CAAC;AACrE,OAAO,EASL,KAAK,aAAa,EAClB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EAEtB,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EAExB,KAAK,QAAQ,EAEd,MAAM,SAAS,CAAC;AAEjB,yEAAyE;AACzE,MAAM,WAAW,kBAAkB;IACjC,aAAa,EAAE,aAAa,CAAC;IAC7B,gFAAgF;IAChF,eAAe,EAAE,MAAM,CAAC;IACxB,yEAAyE;IACzE,qBAAqB,EAAE,MAAM,CAAC;IAC9B,qFAAqF;IACrF,SAAS,EAAE,QAAQ,CAAC;IACpB,mDAAmD;IACnD,aAAa,IAAI,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC,CAAC;IAChD,4DAA4D;IAC5D,gBAAgB,IAAI,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC,CAAC;IACnD,iCAAiC;IACjC,WAAW,IAAI,MAAM,EAAE,CAAC;IACxB,8CAA8C;IAC9C,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC1C,sEAAsE;IACtE,aAAa,CACX,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,SAAS,GACnB,OAAO,CAAC,qBAAqB,CAAC,CAAC;IAClC,QAAQ,EAAE,YAAY,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,kFAAkF;IAClF,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAwBD,qBAAa,cAAc;IACzB,OAAO,CAAC,IAAI,CAAgB;IAC5B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAqB;IAC1C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAEhC,OAAO,CAAC,SAAS,CAAoB;IACrC,OAAO,CAAC,IAAI,CAAW;IACvB,OAAO,CAAC,WAAW,CAAqB;IAExC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,YAAY,CAAmC;IACvD,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,KAAK,CAAuB;IACpC,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,YAAY,CAAgC;IAEpD,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,cAAc,CAAuB;IAC7C,OAAO,CAAC,OAAO,CAAoC;IACnD,OAAO,CAAC,kBAAkB,CAAK;IAC/B,OAAO,CAAC,gBAAgB,CAAK;IAE7B,OAAO,CAAC,YAAY,CAA2C;IAC/D,OAAO,CAAC,aAAa,CAAqB;IAC1C,OAAO,CAAC,SAAS,CAA+B;IAChD,OAAO,CAAC,SAAS,CAA+B;IAChD,OAAO,CAAC,cAAc,CAA+B;IACrD,OAAO,CAAC,gBAAgB,CAAK;IAC7B,OAAO,CAAC,QAAQ,CAAgC;IAChD,OAAO,CAAC,cAAc,CAA6B;IAEnD,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,cAAc,CAAkC;IACxD,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,UAAU,CAAuB;IAEzC;;;;;;;OAOG;IACH,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2C;gBAEvD,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,kBAAkB;IAYzD,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,IAAI,GAAG,IAAI,MAAM,GAAG,SAAS,CAE5B;IAED,6EAA6E;IAC7E,UAAU,CAAC,IAAI,EAAE,aAAa,GAAG,IAAI;IAWrC,8DAA8D;IAC9D,aAAa,IAAI,mBAAmB;IAWpC,KAAK,IAAI,IAAI;IAiCP,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAqB3B,KAAK,IAAI,IAAI;IAQb,MAAM,IAAI,IAAI;YAQA,SAAS;IAavB,OAAO,CAAC,iBAAiB;YAiBX,WAAW;YAWX,YAAY;YA4CZ,SAAS;IAsCvB;;;;;;;;;OASG;IACG,mBAAmB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAsC9C;;;;;;OAMG;IACG,OAAO,CAAC,KAAK,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAQhE;;;;;;OAMG;IACG,aAAa,CAAC,KAAK,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,iBAAiB,CAAC;YAIxD,OAAO;IAqFrB;;;;;;;;;;;;OAYG;IACH,OAAO,CAAE,UAAU;IAiBnB;;;;;;OAMG;YACW,YAAY;YAeZ,aAAa;IAkE3B;;;;;;OAMG;YACW,QAAQ;YAmCR,iBAAiB;IA0C/B,OAAO,CAAC,WAAW;IAcnB,OAAO,CAAC,aAAa;IAoBrB;;;;;;OAMG;YACW,WAAW;IAazB,OAAO,CAAC,gBAAgB;YAeV,kBAAkB;IA4ChC,OAAO,CAAC,gBAAgB;IAwCxB,OAAO,CAAC,WAAW;IAYnB;;;;;;OAMG;IACH,QAAQ,IAAI,iBAAiB;IA4B7B,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI;IAK5D,OAAO,CAAC,YAAY;IAkBpB,8EAA8E;IAC9E,OAAO,CAAC,gBAAgB;CAkBzB;AAED;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAC1B,OAAO,EAAE,mBAAmB,EAAE,EAC9B,YAAY,GAAE,mBAAmB,EAAO,GACvC;IAAE,MAAM,EAAE,aAAa,CAAC;IAAC,OAAO,EAAE,mBAAmB,EAAE,CAAA;CAAE,CAsB3D"}