hungry-ghost-hive 0.45.0 → 0.47.0

package/src/cli/commands/req.ts

@@ -14,7 +14,12 @@ import { createAgent, getTechLead, updateAgent } from '../../db/queries/agents.j
 import { createLog } from '../../db/queries/logs.js';
 import { createRequirement, updateRequirement } from '../../db/queries/requirements.js';
 import { getAllTeams } from '../../db/queries/teams.js';
-import { isTmuxAvailable, spawnTmuxSession } from '../../tmux/manager.js';
+import {
+  isTmuxAvailable,
+  isTmuxSessionRunning,
+  sendToTmuxSession,
+  spawnTmuxSession,
+} from '../../tmux/manager.js';
 import { getTechLeadSessionName } from '../../utils/instance.js';
 import { withHiveContext } from '../../utils/with-hive-context.js';
 import { startDashboard } from '../dashboard/index.js';
@@ -222,24 +227,32 @@ export const reqCommand = new Command('req')
           );
 
           try {
-            // Build CLI command using the configured runtime for Tech Lead
-            const chromeEnabled =
-              config.agents?.chrome_enabled === true && techLeadCliTool === 'claude';
-            const commandArgs = getCliRuntimeBuilder(techLeadCliTool).buildSpawnCommand(
-              techLeadModel,
-              techLeadSafetyMode,
-              { chrome: chromeEnabled }
-            );
+            const sessionAlreadyRunning = await isTmuxSessionRunning(sessionName);
+
+            if (sessionAlreadyRunning) {
+              // Session is already running — send the prompt to the existing session
+              // rather than killing it and losing in-progress work.
+              await sendToTmuxSession(sessionName, techLeadPrompt);
+            } else {
+              // Build CLI command using the configured runtime for Tech Lead
+              const chromeEnabled =
+                config.agents?.chrome_enabled === true && techLeadCliTool === 'claude';
+              const commandArgs = getCliRuntimeBuilder(techLeadCliTool).buildSpawnCommand(
+                techLeadModel,
+                techLeadSafetyMode,
+                { chrome: chromeEnabled }
+              );
 
-            // Pass the prompt as initialPrompt so it's included as a CLI positional
-            // argument via $(cat ...). This delivers the full multi-line prompt
-            // reliably without tmux send-keys newline issues.
-            await spawnTmuxSession({
-              sessionName,
-              workDir: root,
-              commandArgs,
-              initialPrompt: techLeadPrompt,
-            });
+              // Pass the prompt as initialPrompt so it's included as a CLI positional
+              // argument via $(cat ...). This delivers the full multi-line prompt
+              // reliably without tmux send-keys newline issues.
+              await spawnTmuxSession({
+                sessionName,
+                workDir: root,
+                commandArgs,
+                initialPrompt: techLeadPrompt,
+              });
+            }
 
             // Update agent and log spawning/planning events (atomic transaction)
             await withTransaction(db.db, () => {

package/src/cli/dashboard/index.ts

@@ -83,13 +83,24 @@ export async function startDashboard(options: DashboardOptions = {}): Promise<vo
     tags: true,
   });
 
+  // Paused flag — set to true while a panel is handling tmux attachment so
+  // the background scheduleRefresh() timer skips its render and avoids a
+  // double-render race after the terminal is restored.
+  let refreshPaused = false;
+  const pauseRefresh = () => {
+    refreshPaused = true;
+  };
+  const resumeRefresh = () => {
+    refreshPaused = false;
+  };
+
   // Create panels
-  const agentsPanel = createAgentsPanel(screen, db.db);
+  const agentsPanel = createAgentsPanel(screen, db.db, pauseRefresh, resumeRefresh);
   const storiesPanel = createStoriesPanel(screen, db.db);
   const pipelinePanel = createPipelinePanel(screen, db.db);
   const activityPanel = createActivityPanel(screen, db.db);
   const mergeQueuePanel = createMergeQueuePanel(screen, db.db);
-  const escalationsPanel = createEscalationsPanel(screen, db.db);
+  const escalationsPanel = createEscalationsPanel(screen, db.db, pauseRefresh, resumeRefresh);
 
   // Footer
   blessed.box({
@@ -113,6 +124,7 @@ export async function startDashboard(options: DashboardOptions = {}): Promise<vo
 
   // Refresh function - only reloads database when file has changed
   const refresh = async () => {
+    if (refreshPaused) return;
     try {
       // Check if database file has been modified
       const currentMtime = statSync(dbPath).mtimeMs;

package/src/cli/dashboard/panels/agents.ts

@@ -19,7 +19,12 @@ function debugLog(msg: string) {
 // Store agents for selection lookup
 let currentAgents: AgentRow[] = [];
 
-export function createAgentsPanel(screen: Widgets.Screen, db: Database): Widgets.ListElement {
+export function createAgentsPanel(
+  screen: Widgets.Screen,
+  db: Database,
+  pauseRefresh: () => void,
+  resumeRefresh: () => void
+): Widgets.ListElement {
   const list = blessed.list({
     parent: screen,
     top: 1,
@@ -59,6 +64,9 @@ export function createAgentsPanel(screen: Widgets.Screen, db: Database): Widgets
       const agent = currentAgents[agentIndex];
       debugLog(`Selected agent: ${agent.id}, tmux: ${agent.tmux_session}`);
       if (agent.tmux_session) {
+        // Pause the background refresh timer so it cannot fire a concurrent
+        // render while we are restoring the terminal after tmux detach.
+        pauseRefresh();
         // Temporarily suspend blessed while attached to tmux.
         const resumeScreen = screen.program.pause();
 
@@ -69,8 +77,6 @@ export function createAgentsPanel(screen: Widgets.Screen, db: Database): Widgets
           });
         } finally {
           // Restore dashboard in the same process (do not spawn nested dashboards).
-          // spawnSync blocks the event loop so the refresh timer cannot fire and
-          // the captured `db` reference remains valid.
           resumeScreen();
           // Force full redraw — after tmux detach the terminal buffer is clobbered.
           // Use realloc() (dirty=true) so every cell is marked for redraw, preventing
@@ -82,6 +88,9 @@ export function createAgentsPanel(screen: Widgets.Screen, db: Database): Widgets
             debugLog(`Failed to refresh agents panel after tmux detach: ${err}`);
           }
           screen.render();
+          // Allow the background refresh timer to render again now that the
+          // single explicit render above has fully completed.
+          resumeRefresh();
         }
 
         return;

package/src/cli/dashboard/panels/escalations.ts

@@ -21,7 +21,12 @@ function summarizeEscalationReason(reason: string): string {
   return truncate(reason, 38);
 }
 
-export function createEscalationsPanel(screen: Widgets.Screen, db: Database): Widgets.ListElement {
+export function createEscalationsPanel(
+  screen: Widgets.Screen,
+  db: Database,
+  pauseRefresh: () => void,
+  resumeRefresh: () => void
+): Widgets.ListElement {
   const list = blessed.list({
     parent: screen,
     top: '55%+5',
@@ -53,6 +58,9 @@ export function createEscalationsPanel(screen: Widgets.Screen, db: Database): Wi
     if (selectedIndex >= 0 && selectedIndex < currentEscalations.length) {
       const escalation = currentEscalations[selectedIndex];
 
+      // Pause the background refresh timer so it cannot fire a concurrent
+      // render while we are restoring the terminal after tmux detach.
+      pauseRefresh();
       // Temporarily suspend blessed while handling escalation.
       const resumeScreen = screen.program.pause();
 
@@ -102,6 +110,9 @@ export function createEscalationsPanel(screen: Widgets.Screen, db: Database): Wi
           console.error('Failed to update escalations panel:', err);
         }
         screen.render();
+        // Allow the background refresh timer to render again now that the
+        // single explicit render above has fully completed.
+        resumeRefresh();
       }
 
       return;

package/src/cluster/cluster-http-server.ts

@@ -7,21 +7,53 @@ import type { ClusterEvent, VersionVector } from './replication.js';
 interface DeltaRequest {
   version_vector: VersionVector;
   limit?: number;
+  fencing_token?: number;
 }
 
 interface DeltaResponse {
   events: ClusterEvent[];
   version_vector: VersionVector;
+  fencing_token: number;
 }
 
 const MAX_CLUSTER_REQUEST_BODY_BYTES = 1024 * 1024; // 1 MiB
 
+export interface MembershipJoinRequest {
+  node_id: string;
+  url: string;
+}
+
+export interface MembershipJoinResponse {
+  success: boolean;
+  leader_id: string | null;
+  leader_url: string | null;
+  peers: Array<{ id: string; url: string }>;
+  term: number;
+}
+
+export interface MembershipLeaveRequest {
+  node_id: string;
+}
+
+export interface MembershipLeaveResponse {
+  success: boolean;
+  peers: Array<{ id: string; url: string }>;
+}
+
 export interface ClusterHttpHandlers {
   getStatus: () => unknown;
   handleVoteRequest: (body: unknown) => unknown;
   handleHeartbeat: (body: unknown) => unknown;
   getDeltaFromCache: (vector: VersionVector, limit: number) => ClusterEvent[];
   getVersionVectorCache: () => VersionVector;
+  getReplicationLag: () => unknown;
+  getFencingToken: () => number;
+  validateFencingToken: (token: number) => boolean;
+  isLeaderLeaseValid: () => boolean;
+  handleMembershipJoin: (body: MembershipJoinRequest) => MembershipJoinResponse;
+  handleMembershipLeave: (body: MembershipLeaveRequest) => MembershipLeaveResponse;
+  /** Returns a full snapshot of all replicated tables for state recovery. */
+  getSnapshot: () => unknown;
 }
 
 export class ClusterHttpServer {
@@ -72,6 +104,11 @@ export class ClusterHttpServer {
         return;
       }
 
+      if (method === 'GET' && path === '/cluster/v1/snapshot') {
+        sendJson(res, 200, this.handlers.getSnapshot());
+        return;
+      }
+
       if (method === 'POST' && path === '/cluster/v1/election/request-vote') {
         const body = await readJsonBody(req);
         const response = this.handlers.handleVoteRequest(body);
@@ -86,8 +123,25 @@ export class ClusterHttpServer {
         return;
       }
 
+      if (method === 'GET' && path === '/cluster/v1/replication-lag') {
+        sendJson(res, 200, this.handlers.getReplicationLag());
+        return;
+      }
+
       if (method === 'POST' && path === '/cluster/v1/events/delta') {
         const body = (await readJsonBody(req)) as Partial<DeltaRequest>;
+
+        // Validate fencing token if provided — reject stale-leader requests
+        if (typeof body.fencing_token === 'number') {
+          if (!this.handlers.validateFencingToken(body.fencing_token)) {
+            sendJson(res, 409, {
+              error: 'Fencing token rejected: stale leader epoch',
+              fencing_token: this.handlers.getFencingToken(),
+            });
+            return;
+          }
+        }
+
         const vector = toVersionVector(body.version_vector);
         const limit =
           typeof body.limit === 'number' && Number.isFinite(body.limit) && body.limit > 0
@@ -98,10 +152,36 @@ export class ClusterHttpServer {
         sendJson(res, 200, {
           events,
           version_vector: this.handlers.getVersionVectorCache(),
+          fencing_token: this.handlers.getFencingToken(),
         } satisfies DeltaResponse);
         return;
       }
 
+      if (method === 'POST' && path === '/cluster/v1/membership/join') {
+        const body = (await readJsonBody(req)) as Partial<MembershipJoinRequest>;
+        if (typeof body.node_id !== 'string' || typeof body.url !== 'string') {
+          sendJson(res, 400, { error: 'node_id and url are required' });
+          return;
+        }
+        const response = this.handlers.handleMembershipJoin({
+          node_id: body.node_id,
+          url: body.url,
+        });
+        sendJson(res, response.success ? 200 : 307, response);
+        return;
+      }
+
+      if (method === 'POST' && path === '/cluster/v1/membership/leave') {
+        const body = (await readJsonBody(req)) as Partial<MembershipLeaveRequest>;
+        if (typeof body.node_id !== 'string') {
+          sendJson(res, 400, { error: 'node_id is required' });
+          return;
+        }
+        const response = this.handlers.handleMembershipLeave({ node_id: body.node_id });
+        sendJson(res, response.success ? 200 : 400, response);
+        return;
+      }
+
       sendJson(res, 404, { error: 'Not found' });
     } catch (error) {
       if (error instanceof HttpRequestError) {

package/src/cluster/distributed-runtime-coverage.test.ts

@@ -131,10 +131,14 @@ describe('distributed runtime transport and status', () => {
       is_leader: false,
       leader_id: null,
       leader_url: null,
+      fencing_token: 0,
+      leader_lease_valid: false,
+      leader_lease_duration_ms: 0,
       raft_commit_index: 12,
       raft_last_applied: 0,
       raft_last_log_index: 9,
       peers: [{ id: 'peer-a', url: 'http://127.0.0.1:8080' }],
+      is_catching_up: false,
     } satisfies ClusterStatus);
   });
 
@@ -477,6 +481,11 @@ describe('distributed runtime sync behavior', () => {
       imported_events_applied: 0,
       merged_duplicate_stories: 0,
       durable_log_entries_appended: 0,
+      log_entries_compacted: 0,
+      cluster_events_pruned: 0,
+      used_snapshot_recovery: false,
+      catch_up_applied: 0,
+      catch_up_total: 0,
     });
 
     db.close();

package/src/cluster/distributed-system.test.ts

@@ -646,6 +646,174 @@ describe('durable raft metadata store', () => {
   });
 });
 
+describe('log compaction and snapshotting', () => {
+  it('creates a snapshot and compacts the raft log', () => {
+    const dir = mkdtempSync(join(tmpdir(), 'hive-compaction-'));
+    tempDirs.push(dir);
+
+    const store = new RaftMetadataStore({ clusterDir: dir, nodeId: 'node-compact' });
+    store.setState({ current_term: 5 });
+
+    // Append 20 entries
+    for (let i = 0; i < 20; i++) {
+      store.appendEntry({ type: 'runtime', metadata: { seq: i } });
+    }
+
+    expect(store.getLogEntryCount()).toBe(20);
+    expect(store.getState().last_log_index).toBe(20);
+
+    // Create snapshot at current state
+    const snapshot = store.createSnapshot({ 'node-compact': 10 });
+    expect(snapshot.last_included_index).toBe(20);
+    expect(snapshot.last_included_term).toBe(5);
+    expect(snapshot.version_vector).toEqual({ 'node-compact': 10 });
+    expect(existsSync(join(dir, 'raft-snapshot.json'))).toBe(true);
+
+    // Compact the log
+    const result = store.compactLog();
+    expect(result.entries_removed).toBe(20);
+    expect(result.entries_retained).toBe(0);
+    expect(result.snapshot_index).toBe(20);
+    expect(store.getLogEntryCount()).toBe(0);
+  });
+
+  it('retains entries after the snapshot index during compaction', () => {
+    const dir = mkdtempSync(join(tmpdir(), 'hive-compaction-'));
+    tempDirs.push(dir);
+
+    const store = new RaftMetadataStore({ clusterDir: dir, nodeId: 'node-retain' });
+
+    // Append 10 entries
+    for (let i = 0; i < 10; i++) {
+      store.appendEntry({ type: 'runtime', metadata: { seq: i } });
+    }
+
+    // Snapshot at index 5 (manually set to test partial compaction)
+    const snapshot = store.createSnapshot({ 'node-retain': 5 });
+    expect(snapshot.last_included_index).toBe(10);
+
+    // Append 5 more entries AFTER snapshot
+    for (let i = 0; i < 5; i++) {
+      store.appendEntry({ type: 'runtime', metadata: { seq: 10 + i } });
+    }
+
+    expect(store.getLogEntryCount()).toBe(15);
+
+    const result = store.compactLog();
+    expect(result.entries_removed).toBe(10);
+    expect(result.entries_retained).toBe(5);
+    expect(store.getLogEntryCount()).toBe(5);
+  });
+
+  it('restores snapshot and event IDs across restarts', () => {
+    const dir = mkdtempSync(join(tmpdir(), 'hive-snapshot-restart-'));
+    tempDirs.push(dir);
+
+    const first = new RaftMetadataStore({ clusterDir: dir, nodeId: 'node-snap' });
+    first.setState({ current_term: 3 });
+
+    const events: ClusterEvent[] = [
+      buildStoryEvent({
+        event_id: 'node-a:1',
+        row_id: 'STORY-A',
+        version: { actor_id: 'node-a', actor_counter: 1, logical_ts: 1000 },
+      }),
+      buildStoryEvent({
+        event_id: 'node-b:1',
+        row_id: 'STORY-B',
+        version: { actor_id: 'node-b', actor_counter: 1, logical_ts: 2000 },
+      }),
+    ];
+    first.appendClusterEvents(events, 3);
+    first.createSnapshot({ 'node-a': 1, 'node-b': 1 });
+    first.compactLog();
+
+    // Restart
+    const second = new RaftMetadataStore({ clusterDir: dir, nodeId: 'node-snap' });
+    const restored = second.getState();
+    const snap = second.getSnapshot();
+
+    expect(restored.last_log_index).toBeGreaterThanOrEqual(2);
+    expect(snap).not.toBeNull();
+    expect(snap!.version_vector).toEqual({ 'node-a': 1, 'node-b': 1 });
+
+    // Event IDs should be restored from snapshot
+    expect(second.hasEvent('node-a:1')).toBe(true);
+    expect(second.hasEvent('node-b:1')).toBe(true);
+
+    // Deduplication should still work
+    const deduped = second.appendClusterEvents(events, 3);
+    expect(deduped).toBe(0);
+  });
+
+  it('compaction without snapshot is a no-op', () => {
+    const dir = mkdtempSync(join(tmpdir(), 'hive-no-snap-'));
+    tempDirs.push(dir);
+
+    const store = new RaftMetadataStore({ clusterDir: dir, nodeId: 'node-nosn' });
+    for (let i = 0; i < 5; i++) {
+      store.appendEntry({ type: 'runtime', metadata: { seq: i } });
+    }
+
+    const result = store.compactLog();
+    expect(result.entries_removed).toBe(0);
+    expect(result.entries_retained).toBe(0);
+    expect(store.getLogEntryCount()).toBe(5);
+  });
+});
+
+describe('cluster_events pruning', () => {
+  it('prunes old events keeping the most recent ones', async () => {
+    const db = await createTestDatabase();
+    ensureClusterTables(db, 'node-prune');
+
+    // Emit 10 events
+    for (let i = 0; i < 10; i++) {
+      const { emitLocalEvent } = await import('./events.js');
+      emitLocalEvent(db, 'node-prune', {
+        table_name: 'stories',
+        row_id: `STORY-${i}`,
+        op: 'upsert',
+        payload: storyPayload(`STORY-${i}`),
+      });
+    }
+
+    const { getClusterEventCount, pruneClusterEvents } = await import('./events.js');
+    expect(getClusterEventCount(db)).toBe(10);
+
+    // Prune to keep only 5
+    const pruned = pruneClusterEvents(db, 5);
+    expect(pruned).toBe(5);
+    expect(getClusterEventCount(db)).toBe(5);
+
+    db.close();
+  });
+
+  it('does not prune when count is below threshold', async () => {
+    const db = await createTestDatabase();
+    ensureClusterTables(db, 'node-no-prune');
+
+    const { emitLocalEvent } = await import('./events.js');
+    for (let i = 0; i < 3; i++) {
+      emitLocalEvent(db, 'node-no-prune', {
+        table_name: 'stories',
+        row_id: `STORY-${i}`,
+        op: 'upsert',
+        payload: storyPayload(`STORY-${i}`),
+      });
+    }
+
+    const { getClusterEventCount, pruneClusterEvents } = await import('./events.js');
+    expect(getClusterEventCount(db)).toBe(3);
+
+    const pruned = pruneClusterEvents(db, 10);
+    expect(pruned).toBe(0);
+    expect(getClusterEventCount(db)).toBe(3);
+
+    db.close();
+  });
+});
+
 function storyPayload(
   id: string,
   overrides: Partial<Record<string, unknown>> = {}

package/src/cluster/events.ts

@@ -96,6 +96,13 @@ export function ensureClusterTables(db: Database, nodeId: string): void {
   `
   );
 
+  // Add snapshot_version_vector column if it doesn't exist yet (backward-compat migration)
+  try {
+    run(db, 'ALTER TABLE cluster_state ADD COLUMN snapshot_version_vector TEXT');
+  } catch {
+    // Column already exists — ignore
+  }
+
   const state = queryOne<{ id: number }>(db, 'SELECT id FROM cluster_state WHERE id = 1');
   const now = new Date().toISOString();
 
@@ -128,6 +135,53 @@ export function getVersionVector(db: Database): VersionVector {
   return vector;
 }
 
+/**
+ * Returns the snapshot version vector stored after the last snapshot-based recovery.
+ * Empty object if no snapshot has been applied.
+ */
+export function getSnapshotVersionVector(db: Database): VersionVector {
+  const row = queryOne<{ snapshot_version_vector: string | null }>(
+    db,
+    'SELECT snapshot_version_vector FROM cluster_state WHERE id = 1'
+  );
+
+  if (!row?.snapshot_version_vector) return {};
+
+  try {
+    return JSON.parse(row.snapshot_version_vector) as VersionVector;
+  } catch {
+    return {};
+  }
+}
+
+/**
+ * Persists the snapshot version vector so that future delta requests start
+ * from this point rather than from the (empty) event log.
+ */
+export function setSnapshotVersionVector(db: Database, vector: VersionVector): void {
+  run(db, 'UPDATE cluster_state SET snapshot_version_vector = ? WHERE id = 1', [
+    JSON.stringify(vector),
+  ]);
+}
+
+/**
+ * Returns the effective version vector for delta-sync requests.
+ * Takes the max per actor between the event-log-derived vector and any
+ * snapshot vector stored from a previous snapshot-based recovery.
+ * This prevents re-requesting events that were already covered by a snapshot.
+ */
+export function getEffectiveVersionVector(db: Database): VersionVector {
+  const eventVector = getVersionVector(db);
+  const snapshotVector = getSnapshotVersionVector(db);
+
+  const effective: VersionVector = { ...snapshotVector };
+  for (const [actor, counter] of Object.entries(eventVector)) {
+    effective[actor] = Math.max(effective[actor] ?? 0, counter);
+  }
+
+  return effective;
+}
+
 export function getAllClusterEvents(db: Database): ClusterEvent[] {
   const rows = queryAll<ClusterEventRow>(
     db,
@@ -204,6 +258,42 @@ export function emitLocalEvent(
   );
 }
 
+/**
+ * Prune old cluster_events rows, retaining only the most recent `retainCount` events.
+ * Returns the number of rows deleted.
+ */
+export function pruneClusterEvents(db: Database, retainCount: number): number {
+  if (retainCount <= 0) return 0;
+
+  const countRow = queryOne<{ total: number }>(db, 'SELECT COUNT(*) as total FROM cluster_events');
+  const total = countRow?.total || 0;
+
+  if (total <= retainCount) return 0;
+
+  // Delete events that are not in the most recent `retainCount` by logical_ts ordering.
+  // We keep the newest events and delete the oldest.
+  run(
+    db,
+    `
+    DELETE FROM cluster_events
+    WHERE event_id NOT IN (
+      SELECT event_id FROM cluster_events
+      ORDER BY logical_ts DESC, actor_id DESC, actor_counter DESC
+      LIMIT ?
+    )
+  `,
+    [retainCount]
+  );
+
+  const afterRow = queryOne<{ total: number }>(db, 'SELECT COUNT(*) as total FROM cluster_events');
+  return total - (afterRow?.total || 0);
+}
+
+export function getClusterEventCount(db: Database): number {
+  const row = queryOne<{ total: number }>(db, 'SELECT COUNT(*) as total FROM cluster_events');
+  return row?.total || 0;
+}
+
 export function fetchTableSnapshots(db: Database, adapter: TableAdapter): TableRowSnapshot[] {
   const rows = queryAll<Record<string, unknown>>(db, adapter.selectSql);