hungry-ghost-hive 0.44.0 → 0.46.0

package/src/cluster/distributed-system.test.ts

@@ -646,6 +646,174 @@ describe('durable raft metadata store', () => {
   });
 });
 
+describe('log compaction and snapshotting', () => {
+  it('creates a snapshot and compacts the raft log', () => {
+    const dir = mkdtempSync(join(tmpdir(), 'hive-compaction-'));
+    tempDirs.push(dir);
+
+    const store = new RaftMetadataStore({ clusterDir: dir, nodeId: 'node-compact' });
+    store.setState({ current_term: 5 });
+
+    // Append 20 entries
+    for (let i = 0; i < 20; i++) {
+      store.appendEntry({ type: 'runtime', metadata: { seq: i } });
+    }
+
+    expect(store.getLogEntryCount()).toBe(20);
+    expect(store.getState().last_log_index).toBe(20);
+
+    // Create snapshot at current state
+    const snapshot = store.createSnapshot({ 'node-compact': 10 });
+    expect(snapshot.last_included_index).toBe(20);
+    expect(snapshot.last_included_term).toBe(5);
+    expect(snapshot.version_vector).toEqual({ 'node-compact': 10 });
+    expect(existsSync(join(dir, 'raft-snapshot.json'))).toBe(true);
+
+    // Compact the log
+    const result = store.compactLog();
+    expect(result.entries_removed).toBe(20);
+    expect(result.entries_retained).toBe(0);
+    expect(result.snapshot_index).toBe(20);
+    expect(store.getLogEntryCount()).toBe(0);
+  });
+
+  it('retains entries after the snapshot index during compaction', () => {
+    const dir = mkdtempSync(join(tmpdir(), 'hive-compaction-'));
+    tempDirs.push(dir);
+
+    const store = new RaftMetadataStore({ clusterDir: dir, nodeId: 'node-retain' });
+
+    // Append 10 entries
+    for (let i = 0; i < 10; i++) {
+      store.appendEntry({ type: 'runtime', metadata: { seq: i } });
+    }
+
+    // Snapshot at index 5 (manually set to test partial compaction)
+    const snapshot = store.createSnapshot({ 'node-retain': 5 });
+    expect(snapshot.last_included_index).toBe(10);
+
+    // Append 5 more entries AFTER snapshot
+    for (let i = 0; i < 5; i++) {
+      store.appendEntry({ type: 'runtime', metadata: { seq: 10 + i } });
+    }
+
+    expect(store.getLogEntryCount()).toBe(15);
+
+    const result = store.compactLog();
+    expect(result.entries_removed).toBe(10);
+    expect(result.entries_retained).toBe(5);
+    expect(store.getLogEntryCount()).toBe(5);
+  });
+
+  it('restores snapshot and event IDs across restarts', () => {
+    const dir = mkdtempSync(join(tmpdir(), 'hive-snapshot-restart-'));
+    tempDirs.push(dir);
+
+    const first = new RaftMetadataStore({ clusterDir: dir, nodeId: 'node-snap' });
+    first.setState({ current_term: 3 });
+
+    const events: ClusterEvent[] = [
+      buildStoryEvent({
+        event_id: 'node-a:1',
+        row_id: 'STORY-A',
+        version: { actor_id: 'node-a', actor_counter: 1, logical_ts: 1000 },
+      }),
+      buildStoryEvent({
+        event_id: 'node-b:1',
+        row_id: 'STORY-B',
+        version: { actor_id: 'node-b', actor_counter: 1, logical_ts: 2000 },
+      }),
+    ];
+    first.appendClusterEvents(events, 3);
+    first.createSnapshot({ 'node-a': 1, 'node-b': 1 });
+    first.compactLog();
+
+    // Restart
+    const second = new RaftMetadataStore({ clusterDir: dir, nodeId: 'node-snap' });
+    const restored = second.getState();
+    const snap = second.getSnapshot();
+
+    expect(restored.last_log_index).toBeGreaterThanOrEqual(2);
+    expect(snap).not.toBeNull();
+    expect(snap!.version_vector).toEqual({ 'node-a': 1, 'node-b': 1 });
+
+    // Event IDs should be restored from snapshot
+    expect(second.hasEvent('node-a:1')).toBe(true);
+    expect(second.hasEvent('node-b:1')).toBe(true);
+
+    // Deduplication should still work
+    const deduped = second.appendClusterEvents(events, 3);
+    expect(deduped).toBe(0);
+  });
+
+  it('compaction without snapshot is a no-op', () => {
+    const dir = mkdtempSync(join(tmpdir(), 'hive-no-snap-'));
+    tempDirs.push(dir);
+
+    const store = new RaftMetadataStore({ clusterDir: dir, nodeId: 'node-nosn' });
+    for (let i = 0; i < 5; i++) {
+      store.appendEntry({ type: 'runtime', metadata: { seq: i } });
+    }
+
+    const result = store.compactLog();
+    expect(result.entries_removed).toBe(0);
+    expect(result.entries_retained).toBe(0);
+    expect(store.getLogEntryCount()).toBe(5);
+  });
+});
+
+describe('cluster_events pruning', () => {
+  it('prunes old events keeping the most recent ones', async () => {
+    const db = await createTestDatabase();
+    ensureClusterTables(db, 'node-prune');
+
+    // Emit 10 events
+    for (let i = 0; i < 10; i++) {
+      const { emitLocalEvent } = await import('./events.js');
+      emitLocalEvent(db, 'node-prune', {
+        table_name: 'stories',
+        row_id: `STORY-${i}`,
+        op: 'upsert',
+        payload: storyPayload(`STORY-${i}`),
+      });
+    }
+
+    const { getClusterEventCount, pruneClusterEvents } = await import('./events.js');
+    expect(getClusterEventCount(db)).toBe(10);
+
+    // Prune to keep only 5
+    const pruned = pruneClusterEvents(db, 5);
+    expect(pruned).toBe(5);
+    expect(getClusterEventCount(db)).toBe(5);
+
+    db.close();
+  });
+
+  it('does not prune when count is below threshold', async () => {
+    const db = await createTestDatabase();
+    ensureClusterTables(db, 'node-no-prune');
+
+    const { emitLocalEvent } = await import('./events.js');
+    for (let i = 0; i < 3; i++) {
+      emitLocalEvent(db, 'node-no-prune', {
+        table_name: 'stories',
+        row_id: `STORY-${i}`,
+        op: 'upsert',
+        payload: storyPayload(`STORY-${i}`),
+      });
+    }
+
+    const { getClusterEventCount, pruneClusterEvents } = await import('./events.js');
+    expect(getClusterEventCount(db)).toBe(3);
+
+    const pruned = pruneClusterEvents(db, 10);
+    expect(pruned).toBe(0);
+    expect(getClusterEventCount(db)).toBe(3);
+
+    db.close();
+  });
+});
+
 function storyPayload(
   id: string,
   overrides: Partial<Record<string, unknown>> = {}

package/src/cluster/events.ts

@@ -96,6 +96,13 @@ export function ensureClusterTables(db: Database, nodeId: string): void {
   `
   );
 
+  // Add snapshot_version_vector column if it doesn't exist yet (backward-compat migration)
+  try {
+    run(db, 'ALTER TABLE cluster_state ADD COLUMN snapshot_version_vector TEXT');
+  } catch {
+    // Column already exists — ignore
+  }
+
   const state = queryOne<{ id: number }>(db, 'SELECT id FROM cluster_state WHERE id = 1');
   const now = new Date().toISOString();
 
@@ -128,6 +135,53 @@ export function getVersionVector(db: Database): VersionVector {
   return vector;
 }
 
+/**
+ * Returns the snapshot version vector stored after the last snapshot-based recovery.
+ * Empty object if no snapshot has been applied.
+ */
+export function getSnapshotVersionVector(db: Database): VersionVector {
+  const row = queryOne<{ snapshot_version_vector: string | null }>(
+    db,
+    'SELECT snapshot_version_vector FROM cluster_state WHERE id = 1'
+  );
+
+  if (!row?.snapshot_version_vector) return {};
+
+  try {
+    return JSON.parse(row.snapshot_version_vector) as VersionVector;
+  } catch {
+    return {};
+  }
+}
+
+/**
+ * Persists the snapshot version vector so that future delta requests start
+ * from this point rather than from the (empty) event log.
+ */
+export function setSnapshotVersionVector(db: Database, vector: VersionVector): void {
+  run(db, 'UPDATE cluster_state SET snapshot_version_vector = ? WHERE id = 1', [
+    JSON.stringify(vector),
+  ]);
+}
+
+/**
+ * Returns the effective version vector for delta-sync requests.
+ * Takes the max per actor between the event-log-derived vector and any
+ * snapshot vector stored from a previous snapshot-based recovery.
+ * This prevents re-requesting events that were already covered by a snapshot.
+ */
+export function getEffectiveVersionVector(db: Database): VersionVector {
+  const eventVector = getVersionVector(db);
+  const snapshotVector = getSnapshotVersionVector(db);
+
+  const effective: VersionVector = { ...snapshotVector };
+  for (const [actor, counter] of Object.entries(eventVector)) {
+    effective[actor] = Math.max(effective[actor] ?? 0, counter);
+  }
+
+  return effective;
+}
+
 export function getAllClusterEvents(db: Database): ClusterEvent[] {
   const rows = queryAll<ClusterEventRow>(
     db,
@@ -204,6 +258,42 @@ export function emitLocalEvent(
   );
 }
 
+/**
+ * Prune old cluster_events rows, retaining only the most recent `retainCount` events.
+ * Returns the number of rows deleted.
+ */
+export function pruneClusterEvents(db: Database, retainCount: number): number {
+  if (retainCount <= 0) return 0;
+
+  const countRow = queryOne<{ total: number }>(db, 'SELECT COUNT(*) as total FROM cluster_events');
+  const total = countRow?.total || 0;
+
+  if (total <= retainCount) return 0;
+
+  // Delete events that are not in the most recent `retainCount` by logical_ts ordering.
+  // We keep the newest events and delete the oldest.
+  run(
+    db,
+    `
+    DELETE FROM cluster_events
+    WHERE event_id NOT IN (
+      SELECT event_id FROM cluster_events
+      ORDER BY logical_ts DESC, actor_id DESC, actor_counter DESC
+      LIMIT ?
+    )
+  `,
+    [retainCount]
+  );
+
+  const afterRow = queryOne<{ total: number }>(db, 'SELECT COUNT(*) as total FROM cluster_events');
+  return total - (afterRow?.total || 0);
+}
+
+export function getClusterEventCount(db: Database): number {
+  const row = queryOne<{ total: number }>(db, 'SELECT COUNT(*) as total FROM cluster_events');
+  return row?.total || 0;
+}
+
 export function fetchTableSnapshots(db: Database, adapter: TableAdapter): TableRowSnapshot[] {
   const rows = queryAll<Record<string, unknown>>(db, adapter.selectSql);
 

package/src/cluster/heartbeat-manager.ts

@@ -6,11 +6,14 @@ import type { RaftStateMachine } from './raft-state-machine.js';
 interface HeartbeatRequest {
   term: number;
   leader_id: string;
+  fencing_token: number;
+  peers?: Array<{ id: string; url: string }>;
 }
 
 interface HeartbeatResponse {
   term: number;
   success: boolean;
+  fencing_token: number;
 }
 
 export interface HeartbeatManagerDeps {
@@ -18,6 +21,7 @@ export interface HeartbeatManagerDeps {
   postJson: <T>(peer: ClusterPeerConfig, path: string, body: unknown) => Promise<T | null>;
   isActive: () => boolean;
   handleBackgroundError: (error: unknown) => void;
+  onPeersUpdated?: (peers: ClusterPeerConfig[]) => void;
 }
 
 export class HeartbeatManager {
@@ -47,20 +51,23 @@ export class HeartbeatManager {
     if (!this.deps.isActive()) return;
 
     const { raft } = this.deps;
+    const peers = raft.getPeers();
 
     const heartbeat: HeartbeatRequest = {
       term: raft.currentTerm,
       leader_id: this.config.node_id,
+      fencing_token: raft.getFencingToken(),
+      peers: peers.map(p => ({ id: p.id, url: p.url })),
     };
 
     raft.appendDurableEntry('heartbeat_sent', {
       term: raft.currentTerm,
       leader_id: this.config.node_id,
-      peer_count: this.config.peers.filter(peer => peer.id !== this.config.node_id).length,
+      peer_count: peers.filter(peer => peer.id !== this.config.node_id).length,
     });
 
     await Promise.all(
-      this.config.peers
+      peers
         .filter(peer => peer.id !== this.config.node_id)
         .map(async peer => {
           const response = await this.deps.postJson<HeartbeatResponse>(
@@ -69,8 +76,11 @@ export class HeartbeatManager {
             heartbeat
           );
 
-          if (response && response.term > raft.currentTerm) {
-            raft.stepDown(response.term, peer.id);
+          if (response) {
+            const remoteTerm = Math.max(response.term, response.fencing_token ?? 0);
+            if (remoteTerm > raft.currentTerm) {
+              raft.stepDown(remoteTerm, peer.id);
+            }
           }
         })
     );
@@ -82,9 +92,16 @@ export class HeartbeatManager {
     const request = body as Partial<HeartbeatRequest>;
     const term = Number(request.term || 0);
     const leaderId = typeof request.leader_id === 'string' ? request.leader_id : null;
+    const fencingToken = Number(request.fencing_token ?? term);
 
+    // Reject heartbeats from stale leaders
     if (term < raft.currentTerm) {
-      return { term: raft.currentTerm, success: false };
+      return { term: raft.currentTerm, success: false, fencing_token: raft.getFencingToken() };
+    }
+
+    // Reject if fencing token doesn't match the heartbeat term
+    if (fencingToken < term) {
+      return { term: raft.currentTerm, success: false, fencing_token: raft.getFencingToken() };
     }
 
     const changed =
@@ -98,15 +115,40 @@ export class HeartbeatManager {
       raft.persistRaftState();
     }
 
+    // Update lease: record that we received a valid heartbeat now
+    raft.lastHeartbeatReceivedAt = Date.now();
     raft.resetElectionDeadline();
 
+    // Apply peer list from leader if present
+    const requestPeers = (request as { peers?: unknown }).peers;
+    if (Array.isArray(requestPeers)) {
+      const parsed = parsePeerList(requestPeers);
+      if (parsed.length > 0) {
+        raft.setPeers(parsed);
+        this.deps.onPeersUpdated?.(parsed);
+      }
+    }
+
     if (changed) {
       raft.appendDurableEntry('heartbeat_received', {
         term,
         leader_id: leaderId,
+        fencing_token: fencingToken,
       });
     }
 
-    return { term: raft.currentTerm, success: true };
+    return { term: raft.currentTerm, success: true, fencing_token: raft.getFencingToken() };
+  }
+}
+
+function parsePeerList(input: unknown[]): ClusterPeerConfig[] {
+  const peers: ClusterPeerConfig[] = [];
+  for (const item of input) {
+    if (!item || typeof item !== 'object') continue;
+    const p = item as { id?: unknown; url?: unknown };
+    if (typeof p.id === 'string' && typeof p.url === 'string') {
+      peers.push({ id: p.id, url: p.url });
+    }
   }
+  return peers;
 }