hungry-ghost-hive 0.45.0 → 0.46.0

package/src/cli/commands/req.ts

@@ -14,7 +14,12 @@ import { createAgent, getTechLead, updateAgent } from '../../db/queries/agents.j
 import { createLog } from '../../db/queries/logs.js';
 import { createRequirement, updateRequirement } from '../../db/queries/requirements.js';
 import { getAllTeams } from '../../db/queries/teams.js';
-import { isTmuxAvailable, spawnTmuxSession } from '../../tmux/manager.js';
+import {
+  isTmuxAvailable,
+  isTmuxSessionRunning,
+  sendToTmuxSession,
+  spawnTmuxSession,
+} from '../../tmux/manager.js';
 import { getTechLeadSessionName } from '../../utils/instance.js';
 import { withHiveContext } from '../../utils/with-hive-context.js';
 import { startDashboard } from '../dashboard/index.js';
@@ -222,24 +227,32 @@ export const reqCommand = new Command('req')
           );
 
           try {
-            // Build CLI command using the configured runtime for Tech Lead
-            const chromeEnabled =
-              config.agents?.chrome_enabled === true && techLeadCliTool === 'claude';
-            const commandArgs = getCliRuntimeBuilder(techLeadCliTool).buildSpawnCommand(
-              techLeadModel,
-              techLeadSafetyMode,
-              { chrome: chromeEnabled }
-            );
+            const sessionAlreadyRunning = await isTmuxSessionRunning(sessionName);
+
+            if (sessionAlreadyRunning) {
+              // Session is already running — send the prompt to the existing session
+              // rather than killing it and losing in-progress work.
+              await sendToTmuxSession(sessionName, techLeadPrompt);
+            } else {
+              // Build CLI command using the configured runtime for Tech Lead
+              const chromeEnabled =
+                config.agents?.chrome_enabled === true && techLeadCliTool === 'claude';
+              const commandArgs = getCliRuntimeBuilder(techLeadCliTool).buildSpawnCommand(
+                techLeadModel,
+                techLeadSafetyMode,
+                { chrome: chromeEnabled }
+              );
 
-            // Pass the prompt as initialPrompt so it's included as a CLI positional
-            // argument via $(cat ...). This delivers the full multi-line prompt
-            // reliably without tmux send-keys newline issues.
-            await spawnTmuxSession({
-              sessionName,
-              workDir: root,
-              commandArgs,
-              initialPrompt: techLeadPrompt,
-            });
+              // Pass the prompt as initialPrompt so it's included as a CLI positional
+              // argument via $(cat ...). This delivers the full multi-line prompt
+              // reliably without tmux send-keys newline issues.
+              await spawnTmuxSession({
+                sessionName,
+                workDir: root,
+                commandArgs,
+                initialPrompt: techLeadPrompt,
+              });
+            }
 
             // Update agent and log spawning/planning events (atomic transaction)
             await withTransaction(db.db, () => {

package/src/cluster/cluster-http-server.ts

@@ -7,21 +7,53 @@ import type { ClusterEvent, VersionVector } from './replication.js';
 interface DeltaRequest {
   version_vector: VersionVector;
   limit?: number;
+  fencing_token?: number;
 }
 
 interface DeltaResponse {
   events: ClusterEvent[];
   version_vector: VersionVector;
+  fencing_token: number;
 }
 
 const MAX_CLUSTER_REQUEST_BODY_BYTES = 1024 * 1024; // 1 MiB
 
+export interface MembershipJoinRequest {
+  node_id: string;
+  url: string;
+}
+
+export interface MembershipJoinResponse {
+  success: boolean;
+  leader_id: string | null;
+  leader_url: string | null;
+  peers: Array<{ id: string; url: string }>;
+  term: number;
+}
+
+export interface MembershipLeaveRequest {
+  node_id: string;
+}
+
+export interface MembershipLeaveResponse {
+  success: boolean;
+  peers: Array<{ id: string; url: string }>;
+}
+
 export interface ClusterHttpHandlers {
   getStatus: () => unknown;
   handleVoteRequest: (body: unknown) => unknown;
   handleHeartbeat: (body: unknown) => unknown;
   getDeltaFromCache: (vector: VersionVector, limit: number) => ClusterEvent[];
   getVersionVectorCache: () => VersionVector;
+  getReplicationLag: () => unknown;
+  getFencingToken: () => number;
+  validateFencingToken: (token: number) => boolean;
+  isLeaderLeaseValid: () => boolean;
+  handleMembershipJoin: (body: MembershipJoinRequest) => MembershipJoinResponse;
+  handleMembershipLeave: (body: MembershipLeaveRequest) => MembershipLeaveResponse;
+  /** Returns a full snapshot of all replicated tables for state recovery. */
+  getSnapshot: () => unknown;
 }
 
 export class ClusterHttpServer {
@@ -72,6 +104,11 @@ export class ClusterHttpServer {
         return;
       }
 
+      if (method === 'GET' && path === '/cluster/v1/snapshot') {
+        sendJson(res, 200, this.handlers.getSnapshot());
+        return;
+      }
+
       if (method === 'POST' && path === '/cluster/v1/election/request-vote') {
         const body = await readJsonBody(req);
         const response = this.handlers.handleVoteRequest(body);
@@ -86,8 +123,25 @@ export class ClusterHttpServer {
         return;
       }
 
+      if (method === 'GET' && path === '/cluster/v1/replication-lag') {
+        sendJson(res, 200, this.handlers.getReplicationLag());
+        return;
+      }
+
       if (method === 'POST' && path === '/cluster/v1/events/delta') {
         const body = (await readJsonBody(req)) as Partial<DeltaRequest>;
+
+        // Validate fencing token if provided — reject stale-leader requests
+        if (typeof body.fencing_token === 'number') {
+          if (!this.handlers.validateFencingToken(body.fencing_token)) {
+            sendJson(res, 409, {
+              error: 'Fencing token rejected: stale leader epoch',
+              fencing_token: this.handlers.getFencingToken(),
+            });
+            return;
+          }
+        }
+
         const vector = toVersionVector(body.version_vector);
         const limit =
           typeof body.limit === 'number' && Number.isFinite(body.limit) && body.limit > 0
@@ -98,10 +152,36 @@ export class ClusterHttpServer {
         sendJson(res, 200, {
           events,
           version_vector: this.handlers.getVersionVectorCache(),
+          fencing_token: this.handlers.getFencingToken(),
         } satisfies DeltaResponse);
         return;
       }
 
+      if (method === 'POST' && path === '/cluster/v1/membership/join') {
+        const body = (await readJsonBody(req)) as Partial<MembershipJoinRequest>;
+        if (typeof body.node_id !== 'string' || typeof body.url !== 'string') {
+          sendJson(res, 400, { error: 'node_id and url are required' });
+          return;
+        }
+        const response = this.handlers.handleMembershipJoin({
+          node_id: body.node_id,
+          url: body.url,
+        });
+        sendJson(res, response.success ? 200 : 307, response);
+        return;
+      }
+
+      if (method === 'POST' && path === '/cluster/v1/membership/leave') {
+        const body = (await readJsonBody(req)) as Partial<MembershipLeaveRequest>;
+        if (typeof body.node_id !== 'string') {
+          sendJson(res, 400, { error: 'node_id is required' });
+          return;
+        }
+        const response = this.handlers.handleMembershipLeave({ node_id: body.node_id });
+        sendJson(res, response.success ? 200 : 400, response);
+        return;
+      }
+
       sendJson(res, 404, { error: 'Not found' });
     } catch (error) {
       if (error instanceof HttpRequestError) {

package/src/cluster/distributed-runtime-coverage.test.ts

@@ -131,10 +131,14 @@ describe('distributed runtime transport and status', () => {
       is_leader: false,
       leader_id: null,
       leader_url: null,
+      fencing_token: 0,
+      leader_lease_valid: false,
+      leader_lease_duration_ms: 0,
       raft_commit_index: 12,
       raft_last_applied: 0,
       raft_last_log_index: 9,
       peers: [{ id: 'peer-a', url: 'http://127.0.0.1:8080' }],
+      is_catching_up: false,
     } satisfies ClusterStatus);
   });
 
@@ -477,6 +481,11 @@ describe('distributed runtime sync behavior', () => {
       imported_events_applied: 0,
       merged_duplicate_stories: 0,
       durable_log_entries_appended: 0,
+      log_entries_compacted: 0,
+      cluster_events_pruned: 0,
+      used_snapshot_recovery: false,
+      catch_up_applied: 0,
+      catch_up_total: 0,
     });
 
     db.close();

package/src/cluster/distributed-system.test.ts

@@ -646,6 +646,174 @@ describe('durable raft metadata store', () => {
   });
 });
 
+describe('log compaction and snapshotting', () => {
+  it('creates a snapshot and compacts the raft log', () => {
+    const dir = mkdtempSync(join(tmpdir(), 'hive-compaction-'));
+    tempDirs.push(dir);
+
+    const store = new RaftMetadataStore({ clusterDir: dir, nodeId: 'node-compact' });
+    store.setState({ current_term: 5 });
+
+    // Append 20 entries
+    for (let i = 0; i < 20; i++) {
+      store.appendEntry({ type: 'runtime', metadata: { seq: i } });
+    }
+
+    expect(store.getLogEntryCount()).toBe(20);
+    expect(store.getState().last_log_index).toBe(20);
+
+    // Create snapshot at current state
+    const snapshot = store.createSnapshot({ 'node-compact': 10 });
+    expect(snapshot.last_included_index).toBe(20);
+    expect(snapshot.last_included_term).toBe(5);
+    expect(snapshot.version_vector).toEqual({ 'node-compact': 10 });
+    expect(existsSync(join(dir, 'raft-snapshot.json'))).toBe(true);
+
+    // Compact the log
+    const result = store.compactLog();
+    expect(result.entries_removed).toBe(20);
+    expect(result.entries_retained).toBe(0);
+    expect(result.snapshot_index).toBe(20);
+    expect(store.getLogEntryCount()).toBe(0);
+  });
+
+  it('retains entries after the snapshot index during compaction', () => {
+    const dir = mkdtempSync(join(tmpdir(), 'hive-compaction-'));
+    tempDirs.push(dir);
+
+    const store = new RaftMetadataStore({ clusterDir: dir, nodeId: 'node-retain' });
+
+    // Append 10 entries
+    for (let i = 0; i < 10; i++) {
+      store.appendEntry({ type: 'runtime', metadata: { seq: i } });
+    }
+
+    // Snapshot at index 5 (manually set to test partial compaction)
+    const snapshot = store.createSnapshot({ 'node-retain': 5 });
+    expect(snapshot.last_included_index).toBe(10);
+
+    // Append 5 more entries AFTER snapshot
+    for (let i = 0; i < 5; i++) {
+      store.appendEntry({ type: 'runtime', metadata: { seq: 10 + i } });
+    }
+
+    expect(store.getLogEntryCount()).toBe(15);
+
+    const result = store.compactLog();
+    expect(result.entries_removed).toBe(10);
+    expect(result.entries_retained).toBe(5);
+    expect(store.getLogEntryCount()).toBe(5);
+  });
+
+  it('restores snapshot and event IDs across restarts', () => {
+    const dir = mkdtempSync(join(tmpdir(), 'hive-snapshot-restart-'));
+    tempDirs.push(dir);
+
+    const first = new RaftMetadataStore({ clusterDir: dir, nodeId: 'node-snap' });
+    first.setState({ current_term: 3 });
+
+    const events: ClusterEvent[] = [
+      buildStoryEvent({
+        event_id: 'node-a:1',
+        row_id: 'STORY-A',
+        version: { actor_id: 'node-a', actor_counter: 1, logical_ts: 1000 },
+      }),
+      buildStoryEvent({
+        event_id: 'node-b:1',
+        row_id: 'STORY-B',
+        version: { actor_id: 'node-b', actor_counter: 1, logical_ts: 2000 },
+      }),
+    ];
+    first.appendClusterEvents(events, 3);
+    first.createSnapshot({ 'node-a': 1, 'node-b': 1 });
+    first.compactLog();
+
+    // Restart
+    const second = new RaftMetadataStore({ clusterDir: dir, nodeId: 'node-snap' });
+    const restored = second.getState();
+    const snap = second.getSnapshot();
+
+    expect(restored.last_log_index).toBeGreaterThanOrEqual(2);
+    expect(snap).not.toBeNull();
+    expect(snap!.version_vector).toEqual({ 'node-a': 1, 'node-b': 1 });
+
+    // Event IDs should be restored from snapshot
+    expect(second.hasEvent('node-a:1')).toBe(true);
+    expect(second.hasEvent('node-b:1')).toBe(true);
+
+    // Deduplication should still work
+    const deduped = second.appendClusterEvents(events, 3);
+    expect(deduped).toBe(0);
+  });
+
+  it('compaction without snapshot is a no-op', () => {
+    const dir = mkdtempSync(join(tmpdir(), 'hive-no-snap-'));
+    tempDirs.push(dir);
+
+    const store = new RaftMetadataStore({ clusterDir: dir, nodeId: 'node-nosn' });
+    for (let i = 0; i < 5; i++) {
+      store.appendEntry({ type: 'runtime', metadata: { seq: i } });
+    }
+
+    const result = store.compactLog();
+    expect(result.entries_removed).toBe(0);
+    expect(result.entries_retained).toBe(0);
+    expect(store.getLogEntryCount()).toBe(5);
+  });
+});
+
+describe('cluster_events pruning', () => {
+  it('prunes old events keeping the most recent ones', async () => {
+    const db = await createTestDatabase();
+    ensureClusterTables(db, 'node-prune');
+
+    // Emit 10 events
+    for (let i = 0; i < 10; i++) {
+      const { emitLocalEvent } = await import('./events.js');
+      emitLocalEvent(db, 'node-prune', {
+        table_name: 'stories',
+        row_id: `STORY-${i}`,
+        op: 'upsert',
+        payload: storyPayload(`STORY-${i}`),
+      });
+    }
+
+    const { getClusterEventCount, pruneClusterEvents } = await import('./events.js');
+    expect(getClusterEventCount(db)).toBe(10);
+
+    // Prune to keep only 5
+    const pruned = pruneClusterEvents(db, 5);
+    expect(pruned).toBe(5);
+    expect(getClusterEventCount(db)).toBe(5);
+
+    db.close();
+  });
+
+  it('does not prune when count is below threshold', async () => {
+    const db = await createTestDatabase();
+    ensureClusterTables(db, 'node-no-prune');
+
+    const { emitLocalEvent } = await import('./events.js');
+    for (let i = 0; i < 3; i++) {
+      emitLocalEvent(db, 'node-no-prune', {
+        table_name: 'stories',
+        row_id: `STORY-${i}`,
+        op: 'upsert',
+        payload: storyPayload(`STORY-${i}`),
+      });
+    }
+
+    const { getClusterEventCount, pruneClusterEvents } = await import('./events.js');
+    expect(getClusterEventCount(db)).toBe(3);
+
+    const pruned = pruneClusterEvents(db, 10);
+    expect(pruned).toBe(0);
+    expect(getClusterEventCount(db)).toBe(3);
+
+    db.close();
+  });
+});
+
 function storyPayload(
   id: string,
   overrides: Partial<Record<string, unknown>> = {}

package/src/cluster/events.ts

@@ -96,6 +96,13 @@ export function ensureClusterTables(db: Database, nodeId: string): void {
   `
   );
 
+  // Add snapshot_version_vector column if it doesn't exist yet (backward-compat migration)
+  try {
+    run(db, 'ALTER TABLE cluster_state ADD COLUMN snapshot_version_vector TEXT');
+  } catch {
+    // Column already exists — ignore
+  }
+
   const state = queryOne<{ id: number }>(db, 'SELECT id FROM cluster_state WHERE id = 1');
   const now = new Date().toISOString();
 
@@ -128,6 +135,53 @@ export function getVersionVector(db: Database): VersionVector {
   return vector;
 }
 
+/**
+ * Returns the snapshot version vector stored after the last snapshot-based recovery.
+ * Empty object if no snapshot has been applied.
+ */
+export function getSnapshotVersionVector(db: Database): VersionVector {
+  const row = queryOne<{ snapshot_version_vector: string | null }>(
+    db,
+    'SELECT snapshot_version_vector FROM cluster_state WHERE id = 1'
+  );
+
+  if (!row?.snapshot_version_vector) return {};
+
+  try {
+    return JSON.parse(row.snapshot_version_vector) as VersionVector;
+  } catch {
+    return {};
+  }
+}
+
+/**
+ * Persists the snapshot version vector so that future delta requests start
+ * from this point rather than from the (empty) event log.
+ */
+export function setSnapshotVersionVector(db: Database, vector: VersionVector): void {
+  run(db, 'UPDATE cluster_state SET snapshot_version_vector = ? WHERE id = 1', [
+    JSON.stringify(vector),
+  ]);
+}
+
+/**
+ * Returns the effective version vector for delta-sync requests.
+ * Takes the max per actor between the event-log-derived vector and any
+ * snapshot vector stored from a previous snapshot-based recovery.
+ * This prevents re-requesting events that were already covered by a snapshot.
+ */
+export function getEffectiveVersionVector(db: Database): VersionVector {
+  const eventVector = getVersionVector(db);
+  const snapshotVector = getSnapshotVersionVector(db);
+
+  const effective: VersionVector = { ...snapshotVector };
+  for (const [actor, counter] of Object.entries(eventVector)) {
+    effective[actor] = Math.max(effective[actor] ?? 0, counter);
+  }
+
+  return effective;
+}
+
 export function getAllClusterEvents(db: Database): ClusterEvent[] {
   const rows = queryAll<ClusterEventRow>(
     db,
@@ -204,6 +258,42 @@ export function emitLocalEvent(
   );
 }
 
+/**
+ * Prune old cluster_events rows, retaining only the most recent `retainCount` events.
+ * Returns the number of rows deleted.
+ */
+export function pruneClusterEvents(db: Database, retainCount: number): number {
+  if (retainCount <= 0) return 0;
+
+  const countRow = queryOne<{ total: number }>(db, 'SELECT COUNT(*) as total FROM cluster_events');
+  const total = countRow?.total || 0;
+
+  if (total <= retainCount) return 0;
+
+  // Delete events that are not in the most recent `retainCount` by logical_ts ordering.
+  // We keep the newest events and delete the oldest.
+  run(
+    db,
+    `
+    DELETE FROM cluster_events
+    WHERE event_id NOT IN (
+      SELECT event_id FROM cluster_events
+      ORDER BY logical_ts DESC, actor_id DESC, actor_counter DESC
+      LIMIT ?
+    )
+  `,
+    [retainCount]
+  );
+
+  const afterRow = queryOne<{ total: number }>(db, 'SELECT COUNT(*) as total FROM cluster_events');
+  return total - (afterRow?.total || 0);
+}
+
+export function getClusterEventCount(db: Database): number {
+  const row = queryOne<{ total: number }>(db, 'SELECT COUNT(*) as total FROM cluster_events');
+  return row?.total || 0;
+}
+
 export function fetchTableSnapshots(db: Database, adapter: TableAdapter): TableRowSnapshot[] {
   const rows = queryAll<Record<string, unknown>>(db, adapter.selectSql);
 

package/src/cluster/heartbeat-manager.ts

@@ -6,11 +6,14 @@ import type { RaftStateMachine } from './raft-state-machine.js';
 interface HeartbeatRequest {
   term: number;
   leader_id: string;
+  fencing_token: number;
+  peers?: Array<{ id: string; url: string }>;
 }
 
 interface HeartbeatResponse {
   term: number;
   success: boolean;
+  fencing_token: number;
 }
 
 export interface HeartbeatManagerDeps {
@@ -18,6 +21,7 @@ export interface HeartbeatManagerDeps {
   postJson: <T>(peer: ClusterPeerConfig, path: string, body: unknown) => Promise<T | null>;
   isActive: () => boolean;
   handleBackgroundError: (error: unknown) => void;
+  onPeersUpdated?: (peers: ClusterPeerConfig[]) => void;
 }
 
 export class HeartbeatManager {
@@ -47,20 +51,23 @@ export class HeartbeatManager {
     if (!this.deps.isActive()) return;
 
     const { raft } = this.deps;
+    const peers = raft.getPeers();
 
     const heartbeat: HeartbeatRequest = {
       term: raft.currentTerm,
       leader_id: this.config.node_id,
+      fencing_token: raft.getFencingToken(),
+      peers: peers.map(p => ({ id: p.id, url: p.url })),
     };
 
     raft.appendDurableEntry('heartbeat_sent', {
       term: raft.currentTerm,
       leader_id: this.config.node_id,
-      peer_count: this.config.peers.filter(peer => peer.id !== this.config.node_id).length,
+      peer_count: peers.filter(peer => peer.id !== this.config.node_id).length,
     });
 
     await Promise.all(
-      this.config.peers
+      peers
         .filter(peer => peer.id !== this.config.node_id)
         .map(async peer => {
           const response = await this.deps.postJson<HeartbeatResponse>(
@@ -69,8 +76,11 @@ export class HeartbeatManager {
             heartbeat
           );
 
-          if (response && response.term > raft.currentTerm) {
-            raft.stepDown(response.term, peer.id);
+          if (response) {
+            const remoteTerm = Math.max(response.term, response.fencing_token ?? 0);
+            if (remoteTerm > raft.currentTerm) {
+              raft.stepDown(remoteTerm, peer.id);
+            }
           }
         })
     );
@@ -82,9 +92,16 @@ export class HeartbeatManager {
     const request = body as Partial<HeartbeatRequest>;
     const term = Number(request.term || 0);
     const leaderId = typeof request.leader_id === 'string' ? request.leader_id : null;
+    const fencingToken = Number(request.fencing_token ?? term);
 
+    // Reject heartbeats from stale leaders
     if (term < raft.currentTerm) {
-      return { term: raft.currentTerm, success: false };
+      return { term: raft.currentTerm, success: false, fencing_token: raft.getFencingToken() };
+    }
+
+    // Reject if fencing token doesn't match the heartbeat term
+    if (fencingToken < term) {
+      return { term: raft.currentTerm, success: false, fencing_token: raft.getFencingToken() };
     }
 
     const changed =
@@ -98,15 +115,40 @@ export class HeartbeatManager {
       raft.persistRaftState();
     }
 
+    // Update lease: record that we received a valid heartbeat now
+    raft.lastHeartbeatReceivedAt = Date.now();
     raft.resetElectionDeadline();
 
+    // Apply peer list from leader if present
+    const requestPeers = (request as { peers?: unknown }).peers;
+    if (Array.isArray(requestPeers)) {
+      const parsed = parsePeerList(requestPeers);
+      if (parsed.length > 0) {
+        raft.setPeers(parsed);
+        this.deps.onPeersUpdated?.(parsed);
+      }
+    }
+
     if (changed) {
       raft.appendDurableEntry('heartbeat_received', {
         term,
         leader_id: leaderId,
+        fencing_token: fencingToken,
       });
     }
 
-    return { term: raft.currentTerm, success: true };
+    return { term: raft.currentTerm, success: true, fencing_token: raft.getFencingToken() };
+  }
+}
+
+function parsePeerList(input: unknown[]): ClusterPeerConfig[] {
+  const peers: ClusterPeerConfig[] = [];
+  for (const item of input) {
+    if (!item || typeof item !== 'object') continue;
+    const p = item as { id?: unknown; url?: unknown };
+    if (typeof p.id === 'string' && typeof p.url === 'string') {
+      peers.push({ id: p.id, url: p.url });
+    }
   }
+  return peers;
 }