hungry-ghost-hive 0.45.0 → 0.46.0

package/src/cluster/raft-store.ts

@@ -4,6 +4,7 @@ import { createHash } from 'crypto';
 import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'fs';
 import { join } from 'path';
 import type { ClusterEvent } from './replication.js';
+import type { RaftSnapshot, VersionVector } from './types.js';
 
 export type DurableLogEntryType =
   | 'runtime'
@@ -13,7 +14,8 @@ export type DurableLogEntryType =
   | 'heartbeat_sent'
   | 'heartbeat_received'
   | 'state_transition'
-  | 'cluster_event';
+  | 'cluster_event'
+  | 'membership_change';
 
 export interface DurableRaftState {
   node_id: string;
@@ -43,6 +45,15 @@ export interface DurableRaftLogEntry {
   created_at: string;
 }
 
+export interface CompactionResult {
+  /** Number of log entries removed */
+  entries_removed: number;
+  /** Number of log entries retained (after snapshot index) */
+  entries_retained: number;
+  /** The snapshot index */
+  snapshot_index: number;
+}
+
 interface RaftStoreOptions {
   clusterDir: string;
   nodeId: string;
@@ -51,19 +62,23 @@ interface RaftStoreOptions {
 export class RaftMetadataStore {
   private readonly statePath: string;
   private readonly logPath: string;
+  private readonly snapshotPath: string;
   private readonly nodeId: string;
 
   private state: DurableRaftState;
   private knownEventIds = new Set<string>();
+  private snapshot: RaftSnapshot | null = null;
 
   constructor(options: RaftStoreOptions) {
     this.nodeId = options.nodeId;
     this.statePath = join(options.clusterDir, 'raft-state.json');
     this.logPath = join(options.clusterDir, 'raft-log.ndjson');
+    this.snapshotPath = join(options.clusterDir, 'raft-snapshot.json');
 
     mkdirSync(options.clusterDir, { recursive: true });
 
     this.state = this.loadOrCreateState();
+    this.loadSnapshot();
     this.rebuildFromLog();
     this.persistState();
   }
@@ -180,6 +195,157 @@ export class RaftMetadataStore {
     return this.knownEventIds.has(eventId);
   }
 
+  getSnapshot(): RaftSnapshot | null {
+    return this.snapshot;
+  }
+
+  getLogEntryCount(): number {
+    if (!existsSync(this.logPath)) return 0;
+    try {
+      const content = readFileSync(this.logPath, 'utf-8');
+      if (!content.trim()) return 0;
+      return content.split('\n').filter(Boolean).length;
+    } catch {
+      return 0;
+    }
+  }
+
+  /**
+   * Create a snapshot at the current state, capturing the version vector
+   * and known event IDs for deduplication continuity.
+   */
+  createSnapshot(versionVector: VersionVector): RaftSnapshot {
+    const snapshot: RaftSnapshot = {
+      last_included_index: this.state.last_log_index,
+      last_included_term: this.state.last_log_term,
+      version_vector: { ...versionVector },
+      known_event_ids: Array.from(this.knownEventIds),
+      created_at: new Date().toISOString(),
+    };
+
+    this.persistSnapshot(snapshot);
+    this.snapshot = snapshot;
+    return snapshot;
+  }
+
+  /**
+   * Compact the raft log by removing all entries at or before the snapshot index.
+   * Only entries after the snapshot index are retained.
+   */
+  compactLog(): CompactionResult {
+    if (!this.snapshot) {
+      return { entries_removed: 0, entries_retained: 0, snapshot_index: 0 };
+    }
+
+    const snapshotIndex = this.snapshot.last_included_index;
+
+    if (!existsSync(this.logPath)) {
+      return { entries_removed: 0, entries_retained: 0, snapshot_index: snapshotIndex };
+    }
+
+    let content: string;
+    try {
+      content = readFileSync(this.logPath, 'utf-8');
+    } catch {
+      return { entries_removed: 0, entries_retained: 0, snapshot_index: snapshotIndex };
+    }
+
+    if (!content.trim()) {
+      return { entries_removed: 0, entries_retained: 0, snapshot_index: snapshotIndex };
+    }
+
+    const lines = content.split('\n').filter(Boolean);
+    const retained: string[] = [];
+    let removed = 0;
+
+    for (const line of lines) {
+      try {
+        const entry = JSON.parse(line) as Partial<DurableRaftLogEntry>;
+        const index = toNonNegativeInt(entry.index);
+        if (index > snapshotIndex) {
+          retained.push(line);
+        } else {
+          removed++;
+        }
+      } catch {
+        // Drop malformed lines during compaction
+        removed++;
+      }
+    }
+
+    // Atomic write: write to temp file then rename
+    try {
+      const temp = `${this.logPath}.tmp`;
+      writeFileSync(temp, retained.length > 0 ? retained.join('\n') + '\n' : '', 'utf-8');
+      renameSync(temp, this.logPath);
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
+        throw error;
+      }
+    }
+
+    return {
+      entries_removed: removed,
+      entries_retained: retained.length,
+      snapshot_index: snapshotIndex,
+    };
+  }
+
+  private loadSnapshot(): void {
+    if (!existsSync(this.snapshotPath)) return;
+
+    try {
+      const raw = readFileSync(this.snapshotPath, 'utf-8');
+      const parsed = JSON.parse(raw) as Partial<RaftSnapshot>;
+
+      if (
+        typeof parsed.last_included_index !== 'number' ||
+        typeof parsed.last_included_term !== 'number'
+      ) {
+        return;
+      }
+
+      this.snapshot = {
+        last_included_index: parsed.last_included_index,
+        last_included_term: parsed.last_included_term,
+        version_vector: parsed.version_vector || {},
+        known_event_ids: Array.isArray(parsed.known_event_ids) ? parsed.known_event_ids : [],
+        created_at: parsed.created_at || new Date().toISOString(),
+      };
+
+      // Restore known event IDs from snapshot
+      for (const id of this.snapshot.known_event_ids) {
+        this.knownEventIds.add(id);
+      }
+
+      // Ensure state reflects at least the snapshot's progress
+      this.state.last_log_index = Math.max(
+        this.state.last_log_index,
+        this.snapshot.last_included_index
+      );
+      this.state.last_log_term = Math.max(
+        this.state.last_log_term,
+        this.snapshot.last_included_term
+      );
+      this.state.commit_index = Math.max(this.state.commit_index, this.state.last_log_index);
+      this.state.last_applied = Math.max(this.state.last_applied, this.state.commit_index);
+    } catch {
+      // Ignore corrupt snapshots; the log is still authoritative.
+    }
+  }
+
+  private persistSnapshot(snapshot: RaftSnapshot): void {
+    try {
+      const temp = `${this.snapshotPath}.tmp`;
+      writeFileSync(temp, JSON.stringify(snapshot, null, 2), 'utf-8');
+      renameSync(temp, this.snapshotPath);
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
+        throw error;
+      }
+    }
+  }
+
   private loadOrCreateState(): DurableRaftState {
     if (existsSync(this.statePath)) {
       try {

package/src/cluster/replication-lag.test.ts

@@ -0,0 +1,284 @@
+// Licensed under the Hungry Ghost Hive License. See LICENSE.
+
+import { mkdirSync, mkdtempSync, rmSync } from 'fs';
+import { createServer } from 'net';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { afterEach, describe, expect, it } from 'vitest';
+import type { ClusterConfig } from '../config/schema.js';
+import { createDatabase } from '../db/client.js';
+import { ClusterRuntime, fetchReplicationLag } from './runtime.js';
+
+const tempRoots: string[] = [];
+
+afterEach(() => {
+  for (const root of tempRoots.splice(0)) {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
+describe('replication lag tracking', () => {
+  it('returns null when cluster is disabled', async () => {
+    const result = await fetchReplicationLag({
+      enabled: false,
+      node_id: 'node-test',
+      listen_host: '127.0.0.1',
+      listen_port: 9999,
+      public_url: 'http://127.0.0.1:9999',
+      peers: [],
+      heartbeat_interval_ms: 2000,
+      election_timeout_min_ms: 3000,
+      election_timeout_max_ms: 6000,
+      sync_interval_ms: 5000,
+      request_timeout_ms: 5000,
+      story_similarity_threshold: 0.92,
+    });
+
+    expect(result).toBeNull();
+  });
+
+  it('getReplicationLag returns empty peers when no peers configured', async () => {
+    if (!(await canListenOnLocalhost())) return;
+
+    const root = mkdtempSync(join(tmpdir(), 'hive-repl-lag-'));
+    tempRoots.push(root);
+    const hiveDir = join(root, '.hive');
+    mkdirSync(hiveDir, { recursive: true });
+
+    const { runtime } = await startRuntimeWithRetries(hiveDir, {
+      enabled: true,
+      node_id: 'node-lag-test',
+      listen_host: '127.0.0.1',
+      peers: [],
+      heartbeat_interval_ms: 100,
+      election_timeout_min_ms: 150,
+      election_timeout_max_ms: 250,
+      sync_interval_ms: 200,
+      request_timeout_ms: 500,
+      story_similarity_threshold: 0.92,
+    });
+
+    const lag = runtime.getReplicationLag();
+    expect(lag.node_id).toBe('node-lag-test');
+    expect(lag.peers).toEqual([]);
+    expect(lag.last_sync_at).toBeNull();
+    expect(lag.total_local_events).toBe(0);
+    expect(lag.version_vector).toEqual({});
+
+    await runtime.stop();
+  });
+
+  it('tracks peer lag after sync with unreachable peer', async () => {
+    if (!(await canListenOnLocalhost())) return;
+
+    const root = mkdtempSync(join(tmpdir(), 'hive-repl-lag-unreach-'));
+    tempRoots.push(root);
+    const hiveDir = join(root, '.hive');
+    mkdirSync(hiveDir, { recursive: true });
+
+    const db = await createDatabase(join(hiveDir, 'hive.db'));
+
+    const { runtime } = await startRuntimeWithRetries(hiveDir, {
+      enabled: true,
+      node_id: 'node-lag-unreach',
+      listen_host: '127.0.0.1',
+      peers: [
+        { id: 'node-lag-unreach', url: 'http://127.0.0.1:1' },
+        { id: 'peer-ghost', url: 'http://127.0.0.1:19999' },
+      ],
+      heartbeat_interval_ms: 100,
+      election_timeout_min_ms: 150,
+      election_timeout_max_ms: 250,
+      sync_interval_ms: 200,
+      request_timeout_ms: 500,
+      story_similarity_threshold: 0.92,
+    });
+
+    await runtime.sync(db.db);
+
+    const lag = runtime.getReplicationLag();
+    expect(lag.node_id).toBe('node-lag-unreach');
+    expect(lag.peers).toHaveLength(1);
+    expect(lag.peers[0].peer_id).toBe('peer-ghost');
+    expect(lag.peers[0].reachable).toBe(false);
+    expect(lag.peers[0].last_sync_at).not.toBeNull();
+    expect(lag.peers[0].last_sync_duration_ms).toBeGreaterThanOrEqual(0);
+    expect(lag.last_sync_at).not.toBeNull();
+
+    await runtime.stop();
+    db.close();
+  });
+
+  it('serves replication-lag via HTTP endpoint', async () => {
+    if (!(await canListenOnLocalhost())) return;
+
+    const root = mkdtempSync(join(tmpdir(), 'hive-repl-lag-http-'));
+    tempRoots.push(root);
+    const hiveDir = join(root, '.hive');
+    mkdirSync(hiveDir, { recursive: true });
+
+    const { runtime, config } = await startRuntimeWithRetries(hiveDir, {
+      enabled: true,
+      node_id: 'node-lag-http',
+      listen_host: '127.0.0.1',
+      peers: [],
+      heartbeat_interval_ms: 100,
+      election_timeout_min_ms: 150,
+      election_timeout_max_ms: 250,
+      sync_interval_ms: 200,
+      request_timeout_ms: 500,
+      story_similarity_threshold: 0.92,
+    });
+
+    const url = `http://127.0.0.1:${config.listen_port}/cluster/v1/replication-lag`;
+    const response = await fetch(url);
+    expect(response.ok).toBe(true);
+
+    const body = (await response.json()) as {
+      node_id: string;
+      peers: unknown[];
+      last_sync_at: string | null;
+    };
+    expect(body.node_id).toBe('node-lag-http');
+    expect(body.peers).toEqual([]);
+    expect(body.last_sync_at).toBeNull();
+
+    await runtime.stop();
+  });
+
+  it('tracks lag for reachable peer with events', async () => {
+    if (!(await canListenOnLocalhost())) return;
+
+    const root = mkdtempSync(join(tmpdir(), 'hive-repl-lag-two-'));
+    tempRoots.push(root);
+    const hiveDirA = join(root, '.hive-a');
+    const hiveDirB = join(root, '.hive-b');
+    mkdirSync(hiveDirA, { recursive: true });
+    mkdirSync(hiveDirB, { recursive: true });
+
+    const dbA = await createDatabase(join(hiveDirA, 'hive.db'));
+    const dbB = await createDatabase(join(hiveDirB, 'hive.db'));
+
+    // Start node B first (the peer)
+    const { runtime: runtimeB, config: configB } = await startRuntimeWithRetries(hiveDirB, {
+      enabled: true,
+      node_id: 'node-b',
+      listen_host: '127.0.0.1',
+      peers: [],
+      heartbeat_interval_ms: 100,
+      election_timeout_min_ms: 150,
+      election_timeout_max_ms: 250,
+      sync_interval_ms: 200,
+      request_timeout_ms: 500,
+      story_similarity_threshold: 0.92,
+    });
+
+    // Insert data on node B and sync so it has events in cache
+    dbB.db.run(
+      `INSERT INTO stories (id, requirement_id, team_id, title, description, status, created_at, updated_at)
+       VALUES ('STORY-B1', NULL, NULL, 'Story from B', 'Test story', 'planned', ?, ?)`,
+      [new Date().toISOString(), new Date().toISOString()]
+    );
+    await runtimeB.sync(dbB.db);
+    dbB.save();
+
+    // Start node A with node B as a peer
+    const { runtime: runtimeA } = await startRuntimeWithRetries(hiveDirA, {
+      enabled: true,
+      node_id: 'node-a',
+      listen_host: '127.0.0.1',
+      peers: [
+        { id: 'node-a', url: 'http://127.0.0.1:1' },
+        { id: 'node-b', url: `http://127.0.0.1:${configB.listen_port}` },
+      ],
+      heartbeat_interval_ms: 100,
+      election_timeout_min_ms: 150,
+      election_timeout_max_ms: 250,
+      sync_interval_ms: 200,
+      request_timeout_ms: 500,
+      story_similarity_threshold: 0.92,
+    });
+
+    // Sync node A - it should pull events from B
+    await runtimeA.sync(dbA.db);
+
+    const lag = runtimeA.getReplicationLag();
+    expect(lag.node_id).toBe('node-a');
+    expect(lag.peers).toHaveLength(1);
+    expect(lag.peers[0].peer_id).toBe('node-b');
+    expect(lag.peers[0].reachable).toBe(true);
+    expect(lag.peers[0].events_behind).toBeGreaterThanOrEqual(0);
+    expect(lag.peers[0].last_sync_at).not.toBeNull();
+    expect(lag.peers[0].last_sync_duration_ms).toBeGreaterThanOrEqual(0);
+    expect(lag.last_sync_at).not.toBeNull();
+
+    await runtimeA.stop();
+    await runtimeB.stop();
+    dbA.close();
+    dbB.close();
+  });
+});
+
+async function getFreePort(): Promise<number> {
+  return new Promise((resolve, reject) => {
+    const server = createServer();
+    server.once('error', reject);
+    server.listen(0, '127.0.0.1', () => {
+      const address = server.address();
+      if (!address || typeof address === 'string') {
+        server.close(() => reject(new Error('Failed to allocate free port')));
+        return;
+      }
+
+      const port = address.port;
+      server.close(err => {
+        if (err) {
+          reject(err);
+          return;
+        }
+        resolve(port);
+      });
+    });
+  });
+}
+
+async function canListenOnLocalhost(): Promise<boolean> {
+  try {
+    await getFreePort();
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function startRuntimeWithRetries(
+  hiveDir: string,
+  baseConfig: Omit<ClusterConfig, 'listen_port' | 'public_url'>,
+  attempts = 5
+): Promise<{ runtime: ClusterRuntime; config: ClusterConfig }> {
+  let lastError: unknown;
+
+  for (let i = 0; i < attempts; i++) {
+    const port = await getFreePort();
+    const config: ClusterConfig = {
+      ...baseConfig,
+      listen_port: port,
+      public_url: `http://127.0.0.1:${port}`,
+    };
+
+    const runtime = new ClusterRuntime(config, { hiveDir });
+
+    try {
+      await runtime.start();
+      return { runtime, config };
+    } catch (error) {
+      lastError = error;
+      const err = error as NodeJS.ErrnoException;
+      if (err.code !== 'EADDRINUSE') {
+        throw error;
+      }
+    }
+  }
+
+  throw lastError instanceof Error ? lastError : new Error('Failed to start cluster runtime');
+}

package/src/cluster/replication.ts

@@ -4,6 +4,7 @@
 export type {
   ClusterEvent,
   ClusterEventVersion,
+  RaftSnapshot,
   ReplicatedTable,
   ReplicationOp,
   VersionVector,
@@ -13,8 +14,13 @@ export type {
 export {
   ensureClusterTables,
   getAllClusterEvents,
+  getClusterEventCount,
   getDeltaEvents,
+  getEffectiveVersionVector,
+  getSnapshotVersionVector,
   getVersionVector,
+  pruneClusterEvents,
+  setSnapshotVersionVector,
 } from './events.js';
 
 // Re-export sync functions