hungry-ghost-hive 0.45.0 → 0.47.0

package/dist/cluster/runtime.d.ts

@@ -1,5 +1,7 @@
 import type { Database } from 'sql.js';
 import type { ClusterConfig } from '../config/schema.js';
+import { type MembershipJoinRequest, type MembershipJoinResponse, type MembershipLeaveRequest, type MembershipLeaveResponse } from './cluster-http-server.js';
+import { type ClusterEvent, type VersionVector } from './replication.js';
 type NodeRole = 'leader' | 'follower' | 'candidate';
 interface ClusterRuntimeOptions {
     hiveDir?: string;
@@ -17,6 +19,9 @@ export interface ClusterStatus {
     is_leader: boolean;
     leader_id: string | null;
     leader_url: string | null;
+    fencing_token: number;
+    leader_lease_valid: boolean;
+    leader_lease_duration_ms: number;
     raft_commit_index: number;
     raft_last_applied: number;
     raft_last_log_index: number;
@@ -24,12 +29,38 @@ export interface ClusterStatus {
         id: string;
         url: string;
     }>;
+    /** True while the node is performing snapshot-based catch-up and not yet election-eligible. */
+    is_catching_up: boolean;
 }
 export interface ClusterSyncResult {
     local_events_emitted: number;
     imported_events_applied: number;
     merged_duplicate_stories: number;
     durable_log_entries_appended: number;
+    log_entries_compacted: number;
+    cluster_events_pruned: number;
+    /** True when this sync triggered snapshot-based recovery rather than delta sync. */
+    used_snapshot_recovery: boolean;
+    /** Number of rows applied from the snapshot (0 when delta sync was used). */
+    catch_up_applied: number;
+    /** Total rows in the snapshot (0 when delta sync was used). */
+    catch_up_total: number;
+}
+export interface PeerReplicationLag {
+    peer_id: string;
+    peer_url: string;
+    reachable: boolean;
+    events_behind: number;
+    last_sync_at: string | null;
+    last_sync_duration_ms: number | null;
+    last_sync_events_applied: number;
+}
+export interface ReplicationLagSummary {
+    node_id: string;
+    total_local_events: number;
+    version_vector: VersionVector;
+    peers: PeerReplicationLag[];
+    last_sync_at: string | null;
 }
 export declare class ClusterRuntime {
     private readonly config;
@@ -38,6 +69,11 @@ export declare class ClusterRuntime {
     private stopping;
     private eventCache;
     private versionVectorCache;
+    private lastCompactionAt;
+    private peerLagMap;
+    private lastSyncAt;
+    /** Cached full snapshot refreshed on every sync, served to recovering nodes. */
+    private cachedSnapshot;
     private readonly raft;
     private readonly heartbeat;
     private readonly httpServer;
@@ -46,16 +82,58 @@ export declare class ClusterRuntime {
     stop(): Promise<void>;
     isEnabled(): boolean;
     isLeader(): boolean;
+    getReplicationLag(): ReplicationLagSummary;
     getStatus(): ClusterStatus;
     sync(db: Database): Promise<ClusterSyncResult>;
+    handleMembershipJoin(request: MembershipJoinRequest): MembershipJoinResponse;
+    handleMembershipLeave(request: MembershipLeaveRequest): MembershipLeaveResponse;
+    private maybeCompact;
     private refreshCache;
     private pullEventsFromPeers;
+    /**
+     * Returns true when the delta response is missing events the peer should have.
+     * This happens when the peer's event cache has been truncated (log compaction)
+     * and can no longer provide all events since our last known version.
+     */
+    private isDeltaInsufficient;
+    /**
+     * Requests a full snapshot from the given peer and applies it locally.
+     * Marks the node as no longer catching up once complete.
+     * Returns { applied, total } on success, null on failure.
+     */
+    private recoverFromSnapshot;
+    /**
+     * Applies a snapshot to the local database, upserting all rows from all tables.
+     * Stores the snapshot's version vector so future delta requests start from here.
+     */
+    private applySnapshot;
+    /**
+     * Builds a full snapshot of all replicated tables from the current db state.
+     * Called during sync to keep cachedSnapshot fresh for the HTTP endpoint.
+     */
+    private buildSnapshot;
     private requestDelta;
+    private requestSnapshot;
     private getDeltaFromCache;
     private postJson;
+    private getJson;
     private handleBackgroundError;
     private validateNetworkSecurity;
 }
+export declare function fetchReplicationLag(config: ClusterConfig): Promise<ReplicationLagSummary | null>;
+/**
+ * Fetches recent cluster events from the local runtime via the delta endpoint.
+ * Uses an empty version vector to request recent events up to the given limit.
+ */
+export declare function fetchLocalClusterEvents(config: ClusterConfig, limit?: number): Promise<ClusterEvent[] | null>;
+/**
+ * POSTs to the local cluster runtime at the given path.
+ */
+export declare function postToLocalCluster<T>(config: ClusterConfig, path: string, body: unknown): Promise<T | null>;
+/**
+ * POSTs to a peer cluster node at the given URL and path.
+ */
+export declare function postToPeerCluster<T>(peerUrl: string, path: string, body: unknown, options: ClusterStatusFetchOptions): Promise<T | null>;
 export declare function fetchLocalClusterStatus(config: ClusterConfig): Promise<ClusterStatus | null>;
 export declare function fetchClusterStatusFromUrl(url: string, options: ClusterStatusFetchOptions): Promise<ClusterStatus | null>;
 export {};

package/dist/cluster/runtime.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runtime.d.ts","sourceRoot":"","sources":["../../src/cluster/runtime.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AACvC,OAAO,KAAK,EAAE,aAAa,EAAqB,MAAM,qBAAqB,CAAC;AAe5E,KAAK,QAAQ,GAAG,QAAQ,GAAG,UAAU,GAAG,WAAW,CAAC;AAEpD,UAAU,qBAAqB;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,yBAAyB;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAOD,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,QAAQ,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,iBAAiB;IAChC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,CAAC;IAChC,wBAAwB,EAAE,MAAM,CAAC;IACjC,4BAA4B,EAAE,MAAM,CAAC;CACtC;AAED,qBAAa,cAAc;IAYvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAZ1B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,QAAQ,CAAS;IAEzB,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,kBAAkB,CAAqB;IAE/C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAmB;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAoB;gBAG5B,MAAM,EAAE,aAAa,EACrB,OAAO,GAAE,qBAA0B;IAwBhD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAoBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAiB3B,SAAS,IAAI,OAAO;IAIpB,QAAQ,IAAI,OAAO;IAKnB,SAAS,IAAI,aAAa;IAmBpB,IAAI,CAAC,EAAE,EAAE,QAAQ,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAmCpD,OAAO,CAAC,YAAY;YAKN,mBAAmB;YAkBnB,YAAY;IAW1B,OAAO,CAAC,iBAAiB;YASX,QAAQ;IAmBtB,OAAO,CAAC,qBAAqB;IAO7B,OAAO,CAAC,uBAAuB;CAQhC;AAED,wBAAsB,uBAAuB,CAC3C,MAAM,EAAE,aAAa,GACpB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAyB/B;AAED,wBAAsB,yBAAyB,CAC7C,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAa/B"}
+{"version":3,"file":"runtime.d.ts","sourceRoot":"","sources":["../../src/cluster/runtime.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AACvC,OAAO,KAAK,EAAE,aAAa,EAAqB,MAAM,qBAAqB,CAAC;AAG5E,OAAO,EAEL,KAAK,qBAAqB,EAC1B,KAAK,sBAAsB,EAC3B,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC7B,MAAM,0BAA0B,CAAC;AAGlC,OAAO,EAWL,KAAK,YAAY,EACjB,KAAK,aAAa,EACnB,MAAM,kBAAkB,CAAC;AAG1B,KAAK,QAAQ,GAAG,QAAQ,GAAG,UAAU,GAAG,WAAW,CAAC;AAEpD,UAAU,qBAAqB;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,yBAAyB;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAQD,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,QAAQ,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,wBAAwB,EAAE,MAAM,CAAC;IACjC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC1C,+FAA+F;IAC/F,cAAc,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,iBAAiB;IAChC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,CAAC;IAChC,wBAAwB,EAAE,MAAM,CAAC;IACjC,4BAA4B,EAAE,MAAM,CAAC;IACrC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,oFAAoF;IACpF,sBAAsB,EAAE,OAAO,CAAC;IAChC,6EAA6E;IAC7E,gBAAgB,EAAE,MAAM,CAAC;IACzB,+DAA+D;IAC/D,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,wBAAwB,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,EAAE,aAAa,CAAC;IAC9B,KAAK,EAAE,kBAAkB,EAAE,CAAC;IAC5B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,qBAAa,cAAc;IAkBvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAlB1B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,QAAQ,CAAS;IAEzB,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,kBAAkB,CAAqB;IAC/C,OAAO,CAAC,gBAAgB,CAAK;IAC7B,OAAO,CAAC,UAAU,CAAyC;IAC3D,OAAO,CAAC,UAAU,CAAuB;IAEzC,gFAAgF;IAChF,OAAO,CAAC,cAAc,CAAgC;IAEtD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAmB;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAoB;gBAG5B,MAAM,EAAE,aAAa,EACrB,OAAO,GAAE,qBAA0B;IAmChD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAoBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAiB3B,SAAS,IAAI,OAAO;IAIpB,QAAQ,IAAI,OAAO;IAKnB,iBAAiB,IAAI,qBAAqB;IAwB1C,SAAS,IAAI,aAAa;IAuBpB,IAAI,CAAC,EAAE,EAAE,QAAQ,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAoDpD,oBAAoB,CAAC,OAAO,EAAE,qBAAqB,GAAG,sBAAsB;IA6D5E,qBAAqB,CAAC,OAAO,EAAE,sBAAsB,GAAG,uBAAuB;IA2C/E,OAAO,CAAC,YAAY;IA0CpB,OAAO,CAAC,YAAY;YAKN,mBAAmB;IA0FjC;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;IA0B3B;;;;OAIG;YACW,mBAAmB;IA8BjC;;;OAGG;IACH,OAAO,CAAC,aAAa;IAwBrB;;;OAGG;IACH,OAAO,CAAC,aAAa;YAiBP,YAAY;YAYZ,eAAe;IAI7B,OAAO,CAAC,iBAAiB;YASX,QAAQ;YAmBR,OAAO;IAYrB,OAAO,CAAC,qBAAqB;IAO7B,OAAO,CAAC,uBAAuB;CAQhC;AAED,wBAAsB,mBAAmB,CACvC,MAAM,EAAE,aAAa,GACpB,OAAO,CAAC,qBAAqB,GAAG,IAAI,CAAC,CAYvC;AAED;;;GAGG;AACH,wBAAsB,uBAAuB,CAC3C,MAAM,EAAE,aAAa,EACrB,KAAK,GAAE,MAAW,GACjB,OAAO,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,CAchC;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CAAC,CAAC,EACxC,MAAM,EAAE,aAAa,EACrB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,OAAO,GACZ,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAUnB;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,CAAC,EACvC,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,OAAO,EACb,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAMnB;AAED,wBAAsB,uBAAuB,CAC3C,MAAM,EAAE,aAAa,GACpB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CA6B/B;AAED,wBAAsB,yBAAyB,CAC7C,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAa/B"}

package/dist/cluster/runtime.js

@@ -1,9 +1,11 @@
 // Licensed under the Hungry Ghost Hive License. See LICENSE.
 import { join } from 'path';
-import { ClusterHttpServer } from './cluster-http-server.js';
+import { queryAll } from '../db/client.js';
+import { REPLICATED_TABLES } from './adapters.js';
+import { ClusterHttpServer, } from './cluster-http-server.js';
 import { HeartbeatManager } from './heartbeat-manager.js';
 import { RaftStateMachine } from './raft-state-machine.js';
-import { applyRemoteEvents, ensureClusterTables, getAllClusterEvents, getVersionVector, mergeSimilarStories, scanLocalChanges, } from './replication.js';
+import { applyRemoteEvents, ensureClusterTables, getAllClusterEvents, getClusterEventCount, getEffectiveVersionVector, getVersionVector, mergeSimilarStories, pruneClusterEvents, scanLocalChanges, setSnapshotVersionVector, } from './replication.js';
 export class ClusterRuntime {
     config;
     options;
@@ -11,6 +13,11 @@ export class ClusterRuntime {
     stopping = false;
     eventCache = [];
     versionVectorCache = {};
+    lastCompactionAt = 0;
+    peerLagMap = new Map();
+    lastSyncAt = null;
+    /** Cached full snapshot refreshed on every sync, served to recovering nodes. */
+    cachedSnapshot = null;
     raft;
     heartbeat;
     httpServer;
@@ -27,6 +34,10 @@ export class ClusterRuntime {
             postJson: (peer, path, body) => this.postJson(peer, path, body),
             isActive: () => this.started && !this.stopping,
             handleBackgroundError: error => this.handleBackgroundError(error),
+            onPeersUpdated: peers => {
+                // Follower received updated peer list from leader via heartbeat
+                this.raft.setPeers(peers);
+            },
         });
         this.httpServer = new ClusterHttpServer(config, {
             getStatus: () => this.getStatus(),
@@ -34,6 +45,13 @@ export class ClusterRuntime {
             handleHeartbeat: body => this.heartbeat.handleHeartbeat(body),
             getDeltaFromCache: (vector, limit) => this.getDeltaFromCache(vector, limit),
             getVersionVectorCache: () => this.versionVectorCache,
+            getReplicationLag: () => this.getReplicationLag(),
+            getFencingToken: () => this.raft.getFencingToken(),
+            validateFencingToken: token => this.raft.validateFencingToken(token),
+            isLeaderLeaseValid: () => this.raft.isLeaderLeaseValid(),
+            handleMembershipJoin: body => this.handleMembershipJoin(body),
+            handleMembershipLeave: body => this.handleMembershipLeave(body),
+            getSnapshot: () => this.cachedSnapshot ?? { version_vector: {}, tables: {} },
         });
     }
     async start() {
@@ -72,6 +90,26 @@ export class ClusterRuntime {
             return true;
         return this.raft.role === 'leader';
     }
+    getReplicationLag() {
+        return {
+            node_id: this.config.node_id,
+            total_local_events: this.eventCache.length,
+            version_vector: { ...this.versionVectorCache },
+            peers: this.raft
+                .getPeers()
+                .filter(p => p.id !== this.config.node_id)
+                .map(p => this.peerLagMap.get(p.id) || {
+                peer_id: p.id,
+                peer_url: p.url,
+                reachable: false,
+                events_behind: 0,
+                last_sync_at: null,
+                last_sync_duration_ms: null,
+                last_sync_events_applied: 0,
+            }),
+            last_sync_at: this.lastSyncAt,
+        };
+    }
     getStatus() {
         const raftState = this.raft.getRaftStoreState();
         return {
@@ -83,10 +121,14 @@ export class ClusterRuntime {
             is_leader: this.isLeader(),
             leader_id: this.raft.leaderId,
             leader_url: this.raft.getLeaderUrl(),
+            fencing_token: this.raft.getFencingToken(),
+            leader_lease_valid: this.raft.isLeaderLeaseValid(),
+            leader_lease_duration_ms: this.raft.leaderLeaseDurationMs,
             raft_commit_index: raftState?.commit_index || 0,
             raft_last_applied: raftState?.last_applied || 0,
             raft_last_log_index: raftState?.last_log_index || 0,
-            peers: this.config.peers.map(peer => ({ id: peer.id, url: peer.url })),
+            peers: this.raft.getPeers().map(peer => ({ id: peer.id, url: peer.url })),
+            is_catching_up: this.raft.isCatchingUp,
         };
     }
     async sync(db) {
@@ -96,49 +138,339 @@ export class ClusterRuntime {
                 imported_events_applied: 0,
                 merged_duplicate_stories: 0,
                 durable_log_entries_appended: 0,
+                log_entries_compacted: 0,
+                cluster_events_pruned: 0,
+                used_snapshot_recovery: false,
+                catch_up_applied: 0,
+                catch_up_total: 0,
             };
         }
         const hiveDir = this.options.hiveDir || join(process.cwd(), '.hive');
         this.raft.initializeRaftStore(hiveDir);
         ensureClusterTables(db, this.config.node_id);
+        // Refresh snapshot cache so the HTTP endpoint always serves current data
+        this.cachedSnapshot = this.buildSnapshot(db);
         const localEventsBefore = scanLocalChanges(db, this.config.node_id);
-        const imported = await this.pullEventsFromPeers(db);
+        const { imported, usedSnapshot, catchUpApplied, catchUpTotal } = await this.pullEventsFromPeers(db);
         const merged = mergeSimilarStories(db, this.config.story_similarity_threshold);
-        const localEventsAfter = imported > 0 || merged > 0 ? scanLocalChanges(db, this.config.node_id) : 0;
+        const localEventsAfter = imported > 0 || merged > 0 || usedSnapshot ? scanLocalChanges(db, this.config.node_id) : 0;
         this.refreshCache(db);
         const durableLogEntriesAppended = this.raft.appendClusterEventsToDurableLog(getAllClusterEvents(db));
+        // Run compaction if thresholds are met and enough time has elapsed
+        const { logCompacted, eventsPruned } = this.maybeCompact(db);
         return {
             local_events_emitted: localEventsBefore + localEventsAfter,
             imported_events_applied: imported,
             merged_duplicate_stories: merged,
             durable_log_entries_appended: durableLogEntriesAppended,
+            log_entries_compacted: logCompacted,
+            cluster_events_pruned: eventsPruned,
+            used_snapshot_recovery: usedSnapshot,
+            catch_up_applied: catchUpApplied,
+            catch_up_total: catchUpTotal,
         };
     }
+    handleMembershipJoin(request) {
+        const peers = this.raft.getPeers();
+        const leaderUrl = this.raft.getLeaderUrl();
+        // If not the leader, redirect to leader
+        if (this.raft.role !== 'leader') {
+            return {
+                success: false,
+                leader_id: this.raft.leaderId,
+                leader_url: leaderUrl,
+                peers: peers.map(p => ({ id: p.id, url: p.url })),
+                term: this.raft.currentTerm,
+            };
+        }
+        // Check if peer already exists
+        const existing = peers.find(p => p.id === request.node_id);
+        if (existing) {
+            // Update URL if changed
+            if (existing.url !== request.url) {
+                const updated = peers.map(p => p.id === request.node_id ? { id: p.id, url: request.url } : p);
+                this.raft.setPeers(updated);
+                this.raft.appendDurableEntry('membership_change', {
+                    action: 'update',
+                    node_id: request.node_id,
+                    url: request.url,
+                    peer_count: updated.length,
+                });
+            }
+            return {
+                success: true,
+                leader_id: this.raft.leaderId,
+                leader_url: this.config.public_url,
+                peers: this.raft.getPeers().map(p => ({ id: p.id, url: p.url })),
+                term: this.raft.currentTerm,
+            };
+        }
+        // Add new peer
+        const newPeer = { id: request.node_id, url: request.url };
+        const updated = [...peers, newPeer];
+        this.raft.setPeers(updated);
+        this.raft.appendDurableEntry('membership_change', {
+            action: 'join',
+            node_id: request.node_id,
+            url: request.url,
+            peer_count: updated.length,
+        });
+        return {
+            success: true,
+            leader_id: this.raft.leaderId,
+            leader_url: this.config.public_url,
+            peers: updated.map(p => ({ id: p.id, url: p.url })),
+            term: this.raft.currentTerm,
+        };
+    }
+    handleMembershipLeave(request) {
+        const peers = this.raft.getPeers();
+        // If not the leader, cannot process leave
+        if (this.raft.role !== 'leader') {
+            return {
+                success: false,
+                peers: peers.map(p => ({ id: p.id, url: p.url })),
+            };
+        }
+        // Cannot remove self (leader) — leader must transfer leadership first
+        if (request.node_id === this.config.node_id) {
+            return {
+                success: false,
+                peers: peers.map(p => ({ id: p.id, url: p.url })),
+            };
+        }
+        const existing = peers.find(p => p.id === request.node_id);
+        if (!existing) {
+            // Already gone
+            return {
+                success: true,
+                peers: peers.map(p => ({ id: p.id, url: p.url })),
+            };
+        }
+        const updated = peers.filter(p => p.id !== request.node_id);
+        this.raft.setPeers(updated);
+        this.raft.appendDurableEntry('membership_change', {
+            action: 'leave',
+            node_id: request.node_id,
+            peer_count: updated.length,
+        });
+        return {
+            success: true,
+            peers: updated.map(p => ({ id: p.id, url: p.url })),
+        };
+    }
+    maybeCompact(db) {
+        const now = Date.now();
+        const interval = this.config.compaction_interval_ms ?? 300000;
+        // Respect minimum interval between compaction runs
+        if (interval > 0 && now - this.lastCompactionAt < interval) {
+            return { logCompacted: 0, eventsPruned: 0 };
+        }
+        let logCompacted = 0;
+        let eventsPruned = 0;
+        // Compact raft log if threshold exceeded
+        const maxLogEntries = this.config.max_log_entries ?? 10000;
+        if (maxLogEntries > 0) {
+            const logCount = this.raft.getLogEntryCount();
+            if (logCount > maxLogEntries) {
+                const versionVector = getVersionVector(db);
+                const result = this.raft.createSnapshotAndCompact(versionVector);
+                logCompacted = result.entries_removed;
+            }
+        }
+        // Prune cluster_events if threshold exceeded
+        const maxEvents = this.config.max_cluster_events ?? 50000;
+        if (maxEvents > 0) {
+            const eventCount = getClusterEventCount(db);
+            if (eventCount > maxEvents) {
+                eventsPruned = pruneClusterEvents(db, maxEvents);
+                if (eventsPruned > 0) {
+                    this.refreshCache(db);
+                }
+            }
+        }
+        if (logCompacted > 0 || eventsPruned > 0) {
+            this.lastCompactionAt = now;
+        }
+        return { logCompacted, eventsPruned };
+    }
     refreshCache(db) {
         this.eventCache = getAllClusterEvents(db).slice(-20000);
         this.versionVectorCache = getVersionVector(db);
     }
     async pullEventsFromPeers(db) {
-        if (this.config.peers.length === 0)
-            return 0;
-        let applied = 0;
-        for (const peer of this.config.peers) {
+        const peers = this.raft.getPeers();
+        if (peers.length === 0) {
+            return { imported: 0, usedSnapshot: false, catchUpApplied: 0, catchUpTotal: 0 };
+        }
+        let imported = 0;
+        const syncTimestamp = new Date().toISOString();
+        this.lastSyncAt = syncTimestamp;
+        for (const peer of peers) {
             if (peer.id === this.config.node_id)
                 continue;
-            const localVector = getVersionVector(db);
+            const localVector = getEffectiveVersionVector(db);
+            const syncStart = Date.now();
             const response = await this.requestDelta(peer, localVector, 4000);
-            if (!response || response.events.length === 0)
+            if (!response) {
+                this.peerLagMap.set(peer.id, {
+                    peer_id: peer.id,
+                    peer_url: peer.url,
+                    reachable: false,
+                    events_behind: 0,
+                    last_sync_at: syncTimestamp,
+                    last_sync_duration_ms: Date.now() - syncStart,
+                    last_sync_events_applied: 0,
+                });
+                continue;
+            }
+            // If the peer advertises a higher fencing token, step down
+            if (typeof response.fencing_token === 'number' &&
+                response.fencing_token > this.raft.currentTerm) {
+                this.raft.stepDown(response.fencing_token, null);
+            }
+            // Detect if the delta is insufficient (peer's log was truncated past our position)
+            if (this.isDeltaInsufficient(localVector, response.version_vector, response.events)) {
+                const recovery = await this.recoverFromSnapshot(db, peer);
+                if (recovery !== null) {
+                    this.peerLagMap.set(peer.id, {
+                        peer_id: peer.id,
+                        peer_url: peer.url,
+                        reachable: true,
+                        events_behind: 0,
+                        last_sync_at: syncTimestamp,
+                        last_sync_duration_ms: Date.now() - syncStart,
+                        last_sync_events_applied: recovery.applied,
+                    });
+                    return {
+                        imported: 0,
+                        usedSnapshot: true,
+                        catchUpApplied: recovery.applied,
+                        catchUpTotal: recovery.total,
+                    };
+                }
+                // Snapshot recovery failed — fall through and apply whatever delta we have
+            }
+            const eventsBehind = response.events.length;
+            const peerApplied = eventsBehind > 0 ? applyRemoteEvents(db, this.config.node_id, response.events) : 0;
+            imported += peerApplied;
+            this.peerLagMap.set(peer.id, {
+                peer_id: peer.id,
+                peer_url: peer.url,
+                reachable: true,
+                events_behind: eventsBehind,
+                last_sync_at: syncTimestamp,
+                last_sync_duration_ms: Date.now() - syncStart,
+                last_sync_events_applied: peerApplied,
+            });
+        }
+        // If we had been catching up and now the effective vector matches peers, mark done
+        if (this.raft.isCatchingUp) {
+            this.raft.isCatchingUp = false;
+        }
+        return { imported, usedSnapshot: false, catchUpApplied: 0, catchUpTotal: 0 };
+    }
+    /**
+     * Returns true when the delta response is missing events the peer should have.
+     * This happens when the peer's event cache has been truncated (log compaction)
+     * and can no longer provide all events since our last known version.
+     */
+    isDeltaInsufficient(localVector, peerVector, receivedEvents) {
+        // Count how many events we actually received per actor
+        const received = {};
+        for (const event of receivedEvents) {
+            received[event.version.actor_id] = (received[event.version.actor_id] ?? 0) + 1;
+        }
+        for (const [actorId, peerCounter] of Object.entries(peerVector)) {
+            const localCounter = localVector[actorId] ?? 0;
+            const needed = peerCounter - localCounter;
+            if (needed <= 0)
                 continue;
-            applied += applyRemoteEvents(db, this.config.node_id, response.events);
+            const receivedCount = received[actorId] ?? 0;
+            if (receivedCount < needed) {
+                // We're missing events for this actor that the peer should have
+                return true;
+            }
         }
-        return applied;
+        return false;
+    }
+    /**
+     * Requests a full snapshot from the given peer and applies it locally.
+     * Marks the node as no longer catching up once complete.
+     * Returns { applied, total } on success, null on failure.
+     */
+    async recoverFromSnapshot(db, peer) {
+        this.raft.isCatchingUp = true;
+        this.raft.appendDurableEntry('runtime', {
+            event: 'snapshot_recovery_start',
+            node_id: this.config.node_id,
+            peer_id: peer.id,
+        });
+        const snapshot = await this.requestSnapshot(peer);
+        if (!snapshot) {
+            return null;
+        }
+        const { applied, total } = this.applySnapshot(db, snapshot);
+        this.raft.isCatchingUp = false;
+        this.raft.appendDurableEntry('runtime', {
+            event: 'snapshot_recovery_complete',
+            node_id: this.config.node_id,
+            peer_id: peer.id,
+            rows_applied: applied,
+            rows_total: total,
+        });
+        return { applied, total };
+    }
+    /**
+     * Applies a snapshot to the local database, upserting all rows from all tables.
+     * Stores the snapshot's version vector so future delta requests start from here.
+     */
+    applySnapshot(db, snapshot) {
+        let applied = 0;
+        let total = 0;
+        for (const adapter of REPLICATED_TABLES) {
+            const rows = snapshot.tables[adapter.table];
+            if (!rows)
+                continue;
+            total += rows.length;
+            for (const row of rows) {
+                adapter.upsert(db, row.payload);
+                applied++;
+            }
+        }
+        // Record the snapshot version vector so future delta requests
+        // only ask for events newer than this snapshot
+        setSnapshotVersionVector(db, snapshot.version_vector);
+        return { applied, total };
+    }
+    /**
+     * Builds a full snapshot of all replicated tables from the current db state.
+     * Called during sync to keep cachedSnapshot fresh for the HTTP endpoint.
+     */
+    buildSnapshot(db) {
+        const tables = {};
+        for (const adapter of REPLICATED_TABLES) {
+            const rows = queryAll(db, adapter.selectSql);
+            tables[adapter.table] = rows.map(row => ({
+                rowId: adapter.rowId(row),
+                payload: adapter.payload(row),
+            }));
+        }
+        return {
+            version_vector: getVersionVector(db),
+            tables,
+        };
     }
     async requestDelta(peer, versionVector, limit) {
         return this.postJson(peer, '/cluster/v1/events/delta', {
             version_vector: versionVector,
             limit,
+            fencing_token: this.raft.getFencingToken(),
         });
     }
+    async requestSnapshot(peer) {
+        return this.getJson(peer, '/cluster/v1/snapshot');
+    }
     getDeltaFromCache(remoteVersionVector, limit) {
         return this.eventCache
             .filter(event => {
@@ -155,6 +487,11 @@ export class ClusterRuntime {
             body,
         });
     }
+    async getJson(peer, path) {
+        const normalizedBase = peer.url.endsWith('/') ? peer.url : `${peer.url}/`;
+        const url = new URL(path.replace(/^\//, ''), normalizedBase).toString();
+        return fetchClusterStatusOrPostJson(url, this.config.request_timeout_ms, this.config.auth_token, { method: 'GET' });
+    }
     handleBackgroundError(error) {
         if (!this.started || this.stopping)
             return;
@@ -171,6 +508,48 @@ export class ClusterRuntime {
         throw new Error(`Cluster auth_token is required when listen_host is not loopback (received: ${this.config.listen_host})`);
     }
 }
+export async function fetchReplicationLag(config) {
+    if (!config.enabled)
+        return null;
+    const host = config.listen_host === '0.0.0.0' ? '127.0.0.1' : config.listen_host;
+    const url = `http://${host}:${config.listen_port}/cluster/v1/replication-lag`;
+    return fetchClusterStatusOrPostJson(url, config.request_timeout_ms, config.auth_token, { method: 'GET' });
+}
+/**
+ * Fetches recent cluster events from the local runtime via the delta endpoint.
+ * Uses an empty version vector to request recent events up to the given limit.
+ */
+export async function fetchLocalClusterEvents(config, limit = 50) {
+    if (!config.enabled)
+        return null;
+    const host = config.listen_host === '0.0.0.0' ? '127.0.0.1' : config.listen_host;
+    const url = `http://${host}:${config.listen_port}/cluster/v1/events/delta`;
+    const response = await fetchClusterStatusOrPostJson(url, config.request_timeout_ms, config.auth_token, { method: 'POST', body: { version_vector: {}, limit } });
+    return response?.events ?? null;
+}
+/**
+ * POSTs to the local cluster runtime at the given path.
+ */
+export async function postToLocalCluster(config, path, body) {
+    if (!config.enabled)
+        return null;
+    const host = config.listen_host === '0.0.0.0' ? '127.0.0.1' : config.listen_host;
+    const url = `http://${host}:${config.listen_port}${path}`;
+    return fetchClusterStatusOrPostJson(url, config.request_timeout_ms, config.auth_token, {
+        method: 'POST',
+        body,
+    });
+}
+/**
+ * POSTs to a peer cluster node at the given URL and path.
+ */
+export async function postToPeerCluster(peerUrl, path, body, options) {
+    const url = `${peerUrl.replace(/\/$/, '')}${path}`;
+    return fetchClusterStatusOrPostJson(url, options.timeoutMs, options.authToken, {
+        method: 'POST',
+        body,
+    });
+}
 export async function fetchLocalClusterStatus(config) {
     if (!config.enabled) {
         return {
@@ -182,10 +561,14 @@ export async function fetchLocalClusterStatus(config) {
             is_leader: true,
             leader_id: config.node_id,
             leader_url: null,
+            fencing_token: 0,
+            leader_lease_valid: true,
+            leader_lease_duration_ms: config.leader_lease_ms ?? config.heartbeat_interval_ms * 3,
             raft_commit_index: 0,
             raft_last_applied: 0,
             raft_last_log_index: 0,
             peers: config.peers.map(peer => ({ id: peer.id, url: peer.url })),
+            is_catching_up: false,
         };
     }
     const host = config.listen_host === '0.0.0.0' ? '127.0.0.1' : config.listen_host;
@@ -228,10 +611,14 @@ function parseClusterStatus(input) {
         is_leader: input.is_leader === true,
         leader_id: typeof input.leader_id === 'string' ? input.leader_id : null,
         leader_url: typeof input.leader_url === 'string' ? input.leader_url : null,
+        fencing_token: toInt(input.fencing_token),
+        leader_lease_valid: input.leader_lease_valid === true,
+        leader_lease_duration_ms: toInt(input.leader_lease_duration_ms),
         raft_commit_index: toInt(input.raft_commit_index),
         raft_last_applied: toInt(input.raft_last_applied),
         raft_last_log_index: toInt(input.raft_last_log_index),
         peers,
+        is_catching_up: input.is_catching_up === true,
     };
 }
 async function fetchClusterStatusOrPostJson(url, timeoutMs, authToken, options) {