hungry-ghost-hive 0.45.0 → 0.47.0

package/src/cli/commands/cluster.ts

@@ -2,7 +2,18 @@
 
 import chalk from 'chalk';
 import { Command } from 'commander';
-import { fetchClusterStatusFromUrl, fetchLocalClusterStatus } from '../../cluster/runtime.js';
+import type {
+  MembershipJoinResponse,
+  MembershipLeaveResponse,
+} from '../../cluster/cluster-http-server.js';
+import {
+  fetchClusterStatusFromUrl,
+  fetchLocalClusterEvents,
+  fetchLocalClusterStatus,
+  fetchReplicationLag,
+  postToLocalCluster,
+  postToPeerCluster,
+} from '../../cluster/runtime.js';
 import { loadConfig } from '../../config/loader.js';
 import { findHiveRoot, getHivePaths } from '../../utils/paths.js';
 
@@ -13,6 +24,15 @@ interface PeerStatus {
   status: Awaited<ReturnType<typeof fetchLocalClusterStatus>>;
 }
 
+interface PeerHealth {
+  id: string;
+  url: string;
+  reachable: boolean;
+  latencyMs: number | null;
+  role: string | null;
+  term: number | null;
+}
+
 export const clusterCommand = new Command('cluster').description('Distributed cluster operations');
 
 clusterCommand
@@ -122,5 +142,470 @@ clusterCommand
       );
     }
 
+    // Fetch and display replication lag
+    const lagSummary = await fetchReplicationLag(config.cluster);
+    if (lagSummary && lagSummary.peers.length > 0) {
+      console.log(chalk.bold('\nReplication Lag'));
+      if (lagSummary.last_sync_at) {
+        console.log(chalk.gray(`Last sync: ${lagSummary.last_sync_at}`));
+      }
+
+      for (const peer of lagSummary.peers) {
+        if (!peer.reachable) {
+          console.log(
+            `  ${chalk.red('UNREACHABLE')} ${peer.peer_id} ${chalk.gray(`(last sync: ${peer.last_sync_at || 'never'})`)}`
+          );
+          continue;
+        }
+
+        const lagColor =
+          peer.events_behind === 0
+            ? chalk.green
+            : peer.events_behind > 100
+              ? chalk.red
+              : chalk.yellow;
+        const lagLabel =
+          peer.events_behind === 0 ? 'IN_SYNC' : `${peer.events_behind} events behind`;
+        const duration =
+          peer.last_sync_duration_ms !== null ? ` ${peer.last_sync_duration_ms}ms` : '';
+        console.log(
+          `  ${lagColor(lagLabel)} ${peer.peer_id} applied=${peer.last_sync_events_applied}${duration}`
+        );
+      }
+    }
+
+    console.log();
+  });
+
+clusterCommand
+  .command('replication-lag')
+  .description('Show per-peer replication lag details')
+  .option('--json', 'Output as JSON')
+  .action(async (options: { json?: boolean }) => {
+    const root = findHiveRoot();
+    if (!root) {
+      console.error(chalk.red('Not in a Hive workspace.'));
+      process.exit(1);
+    }
+
+    const paths = getHivePaths(root);
+    const config = loadConfig(paths.hiveDir);
+
+    if (!config.cluster.enabled) {
+      if (options.json) {
+        console.log(JSON.stringify({ enabled: false }, null, 2));
+      } else {
+        console.log(chalk.yellow('Cluster mode is disabled.'));
+      }
+      return;
+    }
+
+    const lagSummary = await fetchReplicationLag(config.cluster);
+
+    if (!lagSummary) {
+      if (options.json) {
+        console.log(
+          JSON.stringify({ error: 'Unable to fetch replication lag from local runtime' }, null, 2)
+        );
+      } else {
+        console.log(chalk.red('Unable to fetch replication lag from local runtime.'));
+        console.log(chalk.gray('Start it with: hive manager start'));
+      }
+      return;
+    }
+
+    if (options.json) {
+      console.log(JSON.stringify(lagSummary, null, 2));
+      return;
+    }
+
+    console.log(chalk.bold('\nReplication Lag Summary\n'));
+    console.log(chalk.gray(`Node: ${lagSummary.node_id}`));
+    console.log(chalk.gray(`Local events: ${lagSummary.total_local_events}`));
+    console.log(chalk.gray(`Last sync: ${lagSummary.last_sync_at || 'never'}`));
+
+    if (Object.keys(lagSummary.version_vector).length > 0) {
+      console.log(chalk.bold('\nVersion Vector'));
+      for (const [actor, counter] of Object.entries(lagSummary.version_vector)) {
+        console.log(chalk.gray(`  ${actor}: ${counter}`));
+      }
+    }
+
+    console.log(chalk.bold('\nPeer Lag'));
+    if (lagSummary.peers.length === 0) {
+      console.log(chalk.gray('  No peers configured.'));
+    } else {
+      for (const peer of lagSummary.peers) {
+        if (!peer.reachable) {
+          console.log(`  ${chalk.red('UNREACHABLE')} ${peer.peer_id} ${chalk.gray(peer.peer_url)}`);
+          console.log(chalk.gray(`    Last sync: ${peer.last_sync_at || 'never'}`));
+          continue;
+        }
+
+        const lagColor =
+          peer.events_behind === 0
+            ? chalk.green
+            : peer.events_behind > 100
+              ? chalk.red
+              : chalk.yellow;
+        const lagLabel =
+          peer.events_behind === 0 ? 'IN_SYNC' : `${peer.events_behind} events behind`;
+        console.log(`  ${lagColor(lagLabel)} ${peer.peer_id} ${chalk.gray(peer.peer_url)}`);
+        console.log(
+          chalk.gray(
+            `    applied=${peer.last_sync_events_applied} duration=${peer.last_sync_duration_ms ?? '-'}ms last_sync=${peer.last_sync_at || 'never'}`
+          )
+        );
+      }
+    }
+
+    console.log();
+  });
+
+clusterCommand
+  .command('health')
+  .description('Check connectivity and latency to all cluster peers')
+  .option('--json', 'Output as JSON')
+  .action(async (options: { json?: boolean }) => {
+    const root = findHiveRoot();
+    if (!root) {
+      console.error(chalk.red('Not in a Hive workspace.'));
+      process.exit(1);
+    }
+
+    const paths = getHivePaths(root);
+    const config = loadConfig(paths.hiveDir);
+
+    if (!config.cluster.enabled) {
+      if (options.json) {
+        console.log(JSON.stringify({ enabled: false }, null, 2));
+      } else {
+        console.log(chalk.yellow('Cluster mode is disabled.'));
+      }
+      return;
+    }
+
+    const peerHealthResults: PeerHealth[] = await Promise.all(
+      config.cluster.peers.map(async peer => {
+        const start = Date.now();
+        const status = await fetchClusterStatusFromUrl(
+          `${peer.url.replace(/\/$/, '')}/cluster/v1/status`,
+          {
+            authToken: config.cluster.auth_token,
+            timeoutMs: config.cluster.request_timeout_ms,
+          }
+        );
+        const latencyMs = status ? Date.now() - start : null;
+
+        return {
+          id: peer.id,
+          url: peer.url,
+          reachable: status !== null,
+          latencyMs,
+          role: status?.role ?? null,
+          term: status?.term ?? null,
+        };
+      })
+    );
+
+    // Also check self
+    const selfStart = Date.now();
+    const selfStatus = await fetchLocalClusterStatus(config.cluster);
+    const selfLatencyMs = selfStatus ? Date.now() - selfStart : null;
+
+    const selfHealth: PeerHealth = {
+      id: config.cluster.node_id,
+      url: config.cluster.public_url,
+      reachable: selfStatus !== null,
+      latencyMs: selfLatencyMs,
+      role: selfStatus?.role ?? null,
+      term: selfStatus?.term ?? null,
+    };
+
+    const allNodes = [selfHealth, ...peerHealthResults];
+    const reachableCount = allNodes.filter(n => n.reachable).length;
+
+    if (options.json) {
+      console.log(
+        JSON.stringify(
+          {
+            enabled: true,
+            total_nodes: allNodes.length,
+            reachable: reachableCount,
+            nodes: allNodes,
+          },
+          null,
+          2
+        )
+      );
+      return;
+    }
+
+    console.log(chalk.bold('\nCluster Health\n'));
+    console.log(chalk.gray(`${reachableCount}/${allNodes.length} nodes reachable`));
+
+    for (const node of allNodes) {
+      const isSelf = node.id === config.cluster.node_id;
+      const label = isSelf ? chalk.cyan('(self)') : '';
+
+      if (!node.reachable) {
+        console.log(`  ${chalk.red('✗')} ${node.id} ${label} ${chalk.gray(node.url)}`);
+        console.log(`    ${chalk.red('UNREACHABLE')}`);
+      } else {
+        const latency = node.latencyMs !== null ? `${node.latencyMs}ms` : '?';
+        const roleLabel = node.role?.toUpperCase() ?? 'UNKNOWN';
+        console.log(`  ${chalk.green('✓')} ${node.id} ${label} ${chalk.gray(node.url)}`);
+        console.log(
+          `    ${chalk.gray(roleLabel)} term=${node.term ?? '?'} latency=${chalk.cyan(latency)}`
+        );
+      }
+    }
+
     console.log();
   });
+
+clusterCommand
+  .command('events')
+  .description('Show recent cluster replication events')
+  .option('--limit <n>', 'Maximum number of events to show', '50')
+  .option('--table <name>', 'Filter by table name')
+  .option('--json', 'Output as JSON')
+  .action(async (options: { limit?: string; table?: string; json?: boolean }) => {
+    const root = findHiveRoot();
+    if (!root) {
+      console.error(chalk.red('Not in a Hive workspace.'));
+      process.exit(1);
+    }
+
+    const paths = getHivePaths(root);
+    const config = loadConfig(paths.hiveDir);
+
+    if (!config.cluster.enabled) {
+      if (options.json) {
+        console.log(JSON.stringify({ enabled: false, events: [] }, null, 2));
+      } else {
+        console.log(chalk.yellow('Cluster mode is disabled.'));
+      }
+      return;
+    }
+
+    const limit = Math.max(1, Math.min(1000, parseInt(options.limit ?? '50', 10) || 50));
+    const events = await fetchLocalClusterEvents(config.cluster, limit);
+
+    if (!events) {
+      if (options.json) {
+        console.log(JSON.stringify({ error: 'Local cluster runtime is not reachable.' }, null, 2));
+      } else {
+        console.error(chalk.red('Local cluster runtime is not reachable.'));
+        console.log(chalk.gray('Start it with: hive manager start'));
+      }
+      process.exit(1);
+    }
+
+    const filtered = options.table ? events.filter(e => e.table_name === options.table) : events;
+
+    if (options.json) {
+      console.log(
+        JSON.stringify({ enabled: true, total: filtered.length, events: filtered }, null, 2)
+      );
+      return;
+    }
+
+    console.log(chalk.bold(`\nCluster Events (${filtered.length})\n`));
+
+    if (filtered.length === 0) {
+      console.log(chalk.gray('No events found.'));
+      console.log();
+      return;
+    }
+
+    for (const event of filtered) {
+      const opColor = event.op === 'upsert' ? chalk.green : chalk.red;
+      const ts = new Date(event.created_at).toLocaleString();
+      console.log(
+        `${chalk.gray(ts)} ${opColor(event.op.toUpperCase())} ${chalk.cyan(event.table_name)} ${event.row_id}`
+      );
+      console.log(
+        chalk.gray(
+          `  actor=${event.version.actor_id} counter=${event.version.actor_counter} logical_ts=${event.version.logical_ts}`
+        )
+      );
+    }
+
+    console.log();
+  });
+
+clusterCommand
+  .command('join')
+  .description('Join this node to an existing cluster via a peer URL')
+  .argument('<peer-url>', 'URL of an existing cluster node (e.g. http://10.0.0.2:8787)')
+  .option('--json', 'Output as JSON')
+  .action(async (peerUrl: string, options: { json?: boolean }) => {
+    const root = findHiveRoot();
+    if (!root) {
+      console.error(chalk.red('Not in a Hive workspace.'));
+      process.exit(1);
+    }
+
+    const paths = getHivePaths(root);
+    const config = loadConfig(paths.hiveDir);
+
+    if (!config.cluster.enabled) {
+      console.error(chalk.red('Cluster mode is disabled.'));
+      console.log(chalk.gray('Set `cluster.enabled: true` in `.hive/hive.config.yaml` to enable.'));
+      process.exit(1);
+    }
+
+    const response = await postToPeerCluster<MembershipJoinResponse>(
+      peerUrl,
+      '/cluster/v1/membership/join',
+      { node_id: config.cluster.node_id, url: config.cluster.public_url },
+      { authToken: config.cluster.auth_token, timeoutMs: config.cluster.request_timeout_ms }
+    );
+
+    if (!response) {
+      if (options.json) {
+        console.log(JSON.stringify({ success: false, error: 'Peer unreachable.' }, null, 2));
+      } else {
+        console.error(chalk.red(`Failed to reach peer at ${peerUrl}.`));
+      }
+      process.exit(1);
+    }
+
+    if (!response.success && response.leader_url) {
+      // Peer redirected us to the leader
+      const leaderResponse = await postToPeerCluster<MembershipJoinResponse>(
+        response.leader_url,
+        '/cluster/v1/membership/join',
+        { node_id: config.cluster.node_id, url: config.cluster.public_url },
+        { authToken: config.cluster.auth_token, timeoutMs: config.cluster.request_timeout_ms }
+      );
+
+      if (options.json) {
+        console.log(
+          JSON.stringify(
+            leaderResponse ?? { success: false, error: 'Leader unreachable.' },
+            null,
+            2
+          )
+        );
+      } else if (leaderResponse?.success) {
+        console.log(chalk.green(`✓ Joined cluster via leader ${response.leader_id ?? 'unknown'}.`));
+        console.log(chalk.gray(`Peers: ${leaderResponse.peers.map(p => p.id).join(', ')}`));
+      } else {
+        console.error(chalk.red('Failed to join cluster via leader.'));
+        process.exit(1);
+      }
+      return;
+    }
+
+    if (options.json) {
+      console.log(JSON.stringify(response, null, 2));
+      return;
+    }
+
+    if (response.success) {
+      console.log(chalk.green(`✓ Joined cluster.`));
+      console.log(chalk.gray(`Leader: ${response.leader_id ?? 'unknown'}`));
+      console.log(chalk.gray(`Peers: ${response.peers.map(p => p.id).join(', ')}`));
+    } else {
+      console.error(chalk.red('Failed to join cluster (node not leader and no leader available).'));
+      process.exit(1);
+    }
+  });
+
+clusterCommand
+  .command('leave')
+  .description('Gracefully remove this node from the cluster')
+  .option('--json', 'Output as JSON')
+  .action(async (options: { json?: boolean }) => {
+    const root = findHiveRoot();
+    if (!root) {
+      console.error(chalk.red('Not in a Hive workspace.'));
+      process.exit(1);
+    }
+
+    const paths = getHivePaths(root);
+    const config = loadConfig(paths.hiveDir);
+
+    if (!config.cluster.enabled) {
+      console.error(chalk.red('Cluster mode is disabled.'));
+      process.exit(1);
+    }
+
+    // First get local status to find the leader
+    const localStatus = await fetchLocalClusterStatus(config.cluster);
+
+    if (!localStatus) {
+      if (options.json) {
+        console.log(
+          JSON.stringify(
+            { success: false, error: 'Local cluster runtime is not reachable.' },
+            null,
+            2
+          )
+        );
+      } else {
+        console.error(chalk.red('Local cluster runtime is not reachable.'));
+        console.log(chalk.gray('Start it with: hive manager start'));
+      }
+      process.exit(1);
+    }
+
+    // If we are the leader, cannot leave (would need to transfer leadership first)
+    if (localStatus.is_leader) {
+      if (options.json) {
+        console.log(
+          JSON.stringify(
+            {
+              success: false,
+              error: 'This node is the leader. Transfer leadership before leaving.',
+            },
+            null,
+            2
+          )
+        );
+      } else {
+        console.error(chalk.red('This node is the current leader and cannot leave directly.'));
+        console.log(chalk.gray('Wait for a new leader to be elected, then retry.'));
+      }
+      process.exit(1);
+    }
+
+    // POST leave to the local runtime (which will forward to leader or handle directly)
+    const response = await postToLocalCluster<MembershipLeaveResponse>(
+      config.cluster,
+      '/cluster/v1/membership/leave',
+      { node_id: config.cluster.node_id }
+    );
+
+    if (!response) {
+      if (options.json) {
+        console.log(
+          JSON.stringify(
+            { success: false, error: 'Failed to contact local cluster runtime.' },
+            null,
+            2
+          )
+        );
+      } else {
+        console.error(chalk.red('Failed to contact local cluster runtime.'));
+      }
+      process.exit(1);
+    }
+
+    if (options.json) {
+      console.log(JSON.stringify(response, null, 2));
+      return;
+    }
+
+    if (response.success) {
+      console.log(chalk.green(`✓ Left cluster successfully.`));
+      console.log(
+        chalk.gray(`Remaining peers: ${response.peers.map(p => p.id).join(', ') || 'none'}`)
+      );
+    } else {
+      console.error(chalk.red('Failed to leave cluster.'));
+      process.exit(1);
+    }
+  });

package/src/cli/commands/manager/index.ts

@@ -257,7 +257,8 @@ managerCommand
   .command('start')
   .description('Start the manager daemon (runs every 60s)')
   .option('-i, --interval <seconds>', 'Check interval in seconds', '60')
-  .option('-v, --verbose', 'Show detailed manager check logs')
+  .option('-v, --verbose', 'Show detailed manager check logs (default: true)')
+  .option('--no-verbose', 'Suppress verbose manager check logs')
   .option('--once', 'Run once and exit')
   .action(async (options: { interval: string; verbose?: boolean; once?: boolean }) => {
     const { root, paths } = withHiveRoot(ctx => ctx);
@@ -310,7 +311,7 @@ managerCommand
       );
     }
 
-    const verbose = options.verbose === true;
+    const verbose = options.verbose !== false;
     let checkInProgress = false;
     let checkQueued = false;
 
@@ -386,7 +387,8 @@ managerCommand
 managerCommand
   .command('check')
   .description('Run a single manager check')
-  .option('-v, --verbose', 'Show detailed manager check logs')
+  .option('-v, --verbose', 'Show detailed manager check logs (default: true)')
+  .option('--no-verbose', 'Suppress verbose manager check logs')
   .action(async (options: { verbose?: boolean }) => {
     const { root, paths } = withHiveRoot(ctx => ctx);
     const config = loadConfig(paths.hiveDir);
@@ -411,7 +413,7 @@ managerCommand
       }
     }
 
-    await managerCheck(root, config, undefined, options.verbose === true);
+    await managerCheck(root, config, undefined, options.verbose !== false);
   });
 
 // Run health check to sync agents with tmux

package/src/cli/commands/req-spawn.test.ts

@@ -0,0 +1,153 @@
+// Licensed under the Hungry Ghost Hive License. See LICENSE.
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+// Mock all external dependencies so we can exercise the session-routing logic
+// without a real filesystem, database, or tmux.
+
+vi.mock('../../cli-runtimes/index.js', () => ({
+  getCliRuntimeBuilder: vi.fn(() => ({
+    buildSpawnCommand: vi.fn(() => ['claude', '--dangerously-skip-permissions']),
+  })),
+  resolveRuntimeModelForCli: vi.fn(model => model),
+}));
+
+vi.mock('../../cluster/runtime.js', () => ({
+  fetchLocalClusterStatus: vi.fn(() => null),
+}));
+
+vi.mock('../../config/loader.js', () => ({
+  loadConfig: vi.fn(() => ({
+    models: {
+      tech_lead: {
+        cli_tool: 'claude',
+        safety_mode: 'bypassPermissions',
+        model: 'claude-sonnet-4-6',
+      },
+    },
+    integrations: {
+      project_management: { provider: 'none' },
+    },
+    agents: { chrome_enabled: false },
+    cluster: { enabled: false },
+  })),
+}));
+
+vi.mock('../../connectors/registry.js', () => ({
+  registry: {
+    getProjectManagement: vi.fn(() => null),
+  },
+}));
+
+vi.mock('../../db/client.js', () => ({
+  withTransaction: vi.fn(async (_db: unknown, fn: () => unknown) => fn()),
+}));
+
+vi.mock('../../db/queries/agents.js', () => ({
+  createAgent: vi.fn(() => ({ id: 'agent-1' })),
+  getTechLead: vi.fn(() => ({ id: 'agent-1' })),
+  updateAgent: vi.fn(),
+}));
+
+vi.mock('../../db/queries/logs.js', () => ({
+  createLog: vi.fn(),
+}));
+
+vi.mock('../../db/queries/requirements.js', () => ({
+  createRequirement: vi.fn(() => ({ id: 'req-1', godmode: 0 })),
+  updateRequirement: vi.fn(),
+}));
+
+vi.mock('../../db/queries/teams.js', () => ({
+  getAllTeams: vi.fn(() => [{ id: 'team-1', name: 'alpha' }]),
+}));
+
+vi.mock('../../tmux/manager.js', () => ({
+  isTmuxAvailable: vi.fn(),
+  isTmuxSessionRunning: vi.fn(),
+  sendToTmuxSession: vi.fn(),
+  spawnTmuxSession: vi.fn(),
+}));
+
+vi.mock('../../utils/instance.js', () => ({
+  getTechLeadSessionName: vi.fn(() => 'hive-tech-lead'),
+}));
+
+vi.mock('../../utils/with-hive-context.js', () => ({
+  withHiveContext: vi.fn(
+    (cb: (ctx: { root: string; paths: { hiveDir: string }; db: { db: object } }) => unknown) =>
+      cb({ root: '/tmp/hive', paths: { hiveDir: '/tmp/hive/.hive' }, db: { db: {} } })
+  ),
+}));
+
+vi.mock('../dashboard/index.js', () => ({
+  startDashboard: vi.fn(),
+}));
+
+// ora returns a chainable spinner stub
+vi.mock('ora', () => ({
+  default: vi.fn(() => ({
+    start: vi.fn().mockReturnThis(),
+    succeed: vi.fn().mockReturnThis(),
+    fail: vi.fn().mockReturnThis(),
+    warn: vi.fn().mockReturnThis(),
+    text: '',
+  })),
+}));
+
+import * as tmuxManager from '../../tmux/manager.js';
+import { reqCommand } from './req.js';
+
+describe('req command - tech lead session routing', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.mocked(tmuxManager.isTmuxAvailable).mockResolvedValue(true);
+  });
+
+  it('spawns a new session when no session is running', async () => {
+    vi.mocked(tmuxManager.isTmuxSessionRunning).mockResolvedValue(false);
+
+    await reqCommand.parseAsync(['node', 'req', 'Build a new feature', '--target-branch', 'main'], {
+      from: 'user',
+    });
+
+    expect(tmuxManager.spawnTmuxSession).toHaveBeenCalledOnce();
+    expect(tmuxManager.sendToTmuxSession).not.toHaveBeenCalled();
+  });
+
+  it('sends prompt to existing session when session is already running', async () => {
+    vi.mocked(tmuxManager.isTmuxSessionRunning).mockResolvedValue(true);
+
+    await reqCommand.parseAsync(['node', 'req', 'Add another feature', '--target-branch', 'main'], {
+      from: 'user',
+    });
+
+    expect(tmuxManager.sendToTmuxSession).toHaveBeenCalledOnce();
+    expect(tmuxManager.spawnTmuxSession).not.toHaveBeenCalled();
+  });
+
+  it('sends prompt to existing session with the correct session name', async () => {
+    vi.mocked(tmuxManager.isTmuxSessionRunning).mockResolvedValue(true);
+
+    await reqCommand.parseAsync(['node', 'req', 'Fix the scheduler', '--target-branch', 'main'], {
+      from: 'user',
+    });
+
+    expect(tmuxManager.sendToTmuxSession).toHaveBeenCalledWith(
+      'hive-tech-lead',
+      expect.any(String)
+    );
+  });
+
+  it('spawns session with correct session name when session is not running', async () => {
+    vi.mocked(tmuxManager.isTmuxSessionRunning).mockResolvedValue(false);
+
+    await reqCommand.parseAsync(['node', 'req', 'Fix the scheduler', '--target-branch', 'main'], {
+      from: 'user',
+    });
+
+    expect(tmuxManager.spawnTmuxSession).toHaveBeenCalledWith(
+      expect.objectContaining({ sessionName: 'hive-tech-lead' })
+    );
+  });
+});