agent-relay 2.0.28 → 2.0.32

package/packages/daemon/src/migrations/index.test.ts

@@ -0,0 +1,195 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import Database from 'better-sqlite3';
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import {
+  ensureMigrationsTable,
+  getAppliedMigrations,
+  isMigrationApplied,
+  recordMigration,
+  calculateChecksum,
+  runMigrations,
+  getPendingMigrations,
+  verifyMigrations,
+  loadMigrationFiles,
+} from './index.js';
+
+describe('Migration Runner', () => {
+  let testDir: string;
+  let db: Database.Database;
+
+  beforeEach(async () => {
+    // Create temp test directory
+    testDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'relay-migrations-test-'));
+    db = new Database(path.join(testDir, 'test.sqlite'));
+    db.pragma('journal_mode = WAL');
+  });
+
+  afterEach(() => {
+    db.close();
+    try {
+      fs.rmSync(testDir, { recursive: true, force: true });
+    } catch {
+      // Ignore cleanup errors
+    }
+  });
+
+  describe('ensureMigrationsTable', () => {
+    it('should create __migrations table if not exists', () => {
+      ensureMigrationsTable(db);
+
+      const tables = db.prepare(`
+        SELECT name FROM sqlite_master WHERE type='table' AND name='__migrations'
+      `).all();
+
+      expect(tables).toHaveLength(1);
+    });
+
+    it('should be idempotent', () => {
+      ensureMigrationsTable(db);
+      ensureMigrationsTable(db);
+
+      const tables = db.prepare(`
+        SELECT name FROM sqlite_master WHERE type='table' AND name='__migrations'
+      `).all();
+
+      expect(tables).toHaveLength(1);
+    });
+  });
+
+  describe('calculateChecksum', () => {
+    it('should return consistent checksums for same input', () => {
+      const sql = 'CREATE TABLE test (id INTEGER);';
+      const hash1 = calculateChecksum(sql);
+      const hash2 = calculateChecksum(sql);
+
+      expect(hash1).toBe(hash2);
+    });
+
+    it('should return different checksums for different input', () => {
+      const hash1 = calculateChecksum('CREATE TABLE test1 (id INTEGER);');
+      const hash2 = calculateChecksum('CREATE TABLE test2 (id INTEGER);');
+
+      expect(hash1).not.toBe(hash2);
+    });
+
+    it('should return 8-character hex string', () => {
+      const hash = calculateChecksum('some sql');
+      expect(hash).toMatch(/^[0-9a-f]{8}$/);
+    });
+  });
+
+  describe('recordMigration / getAppliedMigrations', () => {
+    it('should record and retrieve migrations', () => {
+      ensureMigrationsTable(db);
+
+      recordMigration(db, '0001_initial', 'abc12345');
+      recordMigration(db, '0002_add_column', 'def67890');
+
+      const applied = getAppliedMigrations(db);
+
+      expect(applied).toHaveLength(2);
+      expect(applied[0].name).toBe('0001_initial');
+      expect(applied[0].checksum).toBe('abc12345');
+      expect(applied[1].name).toBe('0002_add_column');
+    });
+  });
+
+  describe('isMigrationApplied', () => {
+    it('should return true for applied migrations', () => {
+      ensureMigrationsTable(db);
+      recordMigration(db, '0001_initial', 'abc12345');
+
+      expect(isMigrationApplied(db, '0001_initial')).toBe(true);
+    });
+
+    it('should return false for unapplied migrations', () => {
+      ensureMigrationsTable(db);
+
+      expect(isMigrationApplied(db, '0001_initial')).toBe(false);
+    });
+  });
+
+  describe('loadMigrationFiles', () => {
+    it('should load embedded migrations', () => {
+      const migrations = loadMigrationFiles();
+
+      expect(migrations.length).toBeGreaterThanOrEqual(1);
+      expect(migrations[0].name).toBe('0001_initial');
+      expect(migrations[0].sql).toContain('CREATE TABLE');
+      expect(migrations[0].checksum).toMatch(/^[0-9a-f]{8}$/);
+    });
+  });
+
+  describe('runMigrations', () => {
+    it('should run initial migration successfully', () => {
+      const result = runMigrations(db);
+
+      // Should have applied the initial migration
+      expect(result.applied.length).toBeGreaterThanOrEqual(1);
+      expect(result.errors).toHaveLength(0);
+
+      // Tables should exist
+      const tables = db.prepare(`
+        SELECT name FROM sqlite_master WHERE type='table' AND name IN ('relay_files', 'agents', '__migrations')
+      `).all() as Array<{ name: string }>;
+
+      expect(tables.map(t => t.name)).toContain('relay_files');
+      expect(tables.map(t => t.name)).toContain('agents');
+    });
+
+    it('should skip already applied migrations', () => {
+      // Run migrations first time
+      const result1 = runMigrations(db);
+      const appliedCount = result1.applied.length;
+
+      // Run again
+      const result2 = runMigrations(db);
+
+      // Should skip all previously applied
+      expect(result2.skipped.length).toBe(appliedCount);
+      expect(result2.applied).toHaveLength(0);
+    });
+
+    it('should be idempotent', () => {
+      runMigrations(db);
+      runMigrations(db);
+      runMigrations(db);
+
+      // Should still have valid schema
+      const result = db.prepare(`SELECT COUNT(*) as count FROM relay_files`).get() as { count: number };
+      expect(result.count).toBe(0);
+    });
+  });
+
+  describe('getPendingMigrations', () => {
+    it('should return all migrations for fresh database', () => {
+      // Don't run migrations, just check pending
+      ensureMigrationsTable(db);
+      const pending = getPendingMigrations(db);
+
+      expect(pending.length).toBeGreaterThanOrEqual(1);
+      expect(pending[0].name).toBe('0001_initial');
+    });
+
+    it('should return empty array after all migrations applied', () => {
+      runMigrations(db);
+      const pending = getPendingMigrations(db);
+
+      expect(pending).toHaveLength(0);
+    });
+  });
+
+  describe('verifyMigrations', () => {
+    it('should return empty array when checksums match', () => {
+      runMigrations(db);
+      const mismatches = verifyMigrations(db);
+
+      expect(mismatches).toHaveLength(0);
+    });
+
+    // Note: Testing checksum mismatch would require modifying embedded migrations
+    // which is not practical in unit tests
+  });
+});

package/packages/daemon/src/migrations/index.ts

@@ -0,0 +1,286 @@
+/**
+ * SQLite Migration Runner for Relay Ledger
+ *
+ * Provides a lightweight migration system for the relay-ledger.db SQLite database.
+ * Tracks applied migrations in a __migrations table and runs them in order.
+ *
+ * Features:
+ * - Sequential migration execution by name
+ * - Idempotent (safe to run multiple times)
+ * - Tracks applied migrations with timestamps
+ * - Embedded SQL for portability (no file dependencies)
+ */
+
+import type Database from 'better-sqlite3';
+
+// ============================================================================
+// Types
+// ============================================================================
+
+export interface MigrationRecord {
+  id: number;
+  name: string;
+  appliedAt: number;
+  checksum: string;
+}
+
+export interface MigrationFile {
+  name: string;
+  sql: string;
+  checksum: string;
+}
+
+export interface MigrationResult {
+  applied: string[];
+  skipped: string[];
+  errors: Array<{ name: string; error: string }>;
+}
+
+// ============================================================================
+// Embedded Migrations
+// ============================================================================
+
+/**
+ * Embedded migrations - SQL is stored directly in code for portability.
+ * Add new migrations to this array in order.
+ */
+const EMBEDDED_MIGRATIONS: Array<{ name: string; sql: string }> = [
+  {
+    name: '0001_initial',
+    sql: `
+-- Relay Ledger - Initial Schema
+-- Migration 0001: Initial tables for relay file tracking
+
+-- Main relay files table
+CREATE TABLE IF NOT EXISTS relay_files (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  file_id TEXT NOT NULL UNIQUE,
+  source_path TEXT NOT NULL,
+  archive_path TEXT,
+  agent_name TEXT NOT NULL,
+  message_type TEXT NOT NULL,
+  status TEXT NOT NULL DEFAULT 'pending',
+  retries INTEGER NOT NULL DEFAULT 0,
+  max_retries INTEGER NOT NULL DEFAULT 3,
+  discovered_at INTEGER NOT NULL,
+  processed_at INTEGER,
+  archived_at INTEGER,
+  error TEXT,
+  content_hash TEXT,
+  file_size INTEGER NOT NULL DEFAULT 0,
+  file_mtime_ns INTEGER,
+  file_inode INTEGER,
+  CONSTRAINT valid_status CHECK (status IN ('pending', 'processing', 'delivered', 'failed', 'archived'))
+);
+
+CREATE INDEX IF NOT EXISTS idx_relay_files_status ON relay_files(status);
+CREATE INDEX IF NOT EXISTS idx_relay_files_agent ON relay_files(agent_name);
+CREATE INDEX IF NOT EXISTS idx_relay_files_discovered ON relay_files(discovered_at);
+CREATE INDEX IF NOT EXISTS idx_relay_files_source ON relay_files(source_path);
+
+-- Agents registry table
+CREATE TABLE IF NOT EXISTS agents (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  agent_name TEXT NOT NULL UNIQUE,
+  created_at INTEGER NOT NULL,
+  last_seen_at INTEGER NOT NULL,
+  status TEXT NOT NULL DEFAULT 'active',
+  metadata TEXT,
+  CONSTRAINT valid_agent_status CHECK (status IN ('active', 'inactive'))
+);
+
+CREATE INDEX IF NOT EXISTS idx_agents_status ON agents(status);
+CREATE INDEX IF NOT EXISTS idx_agents_last_seen ON agents(last_seen_at);
+
+-- Orchestrator state table (key-value store for crash recovery)
+CREATE TABLE IF NOT EXISTS orchestrator_state (
+  key TEXT PRIMARY KEY,
+  value TEXT NOT NULL,
+  updated_at INTEGER NOT NULL
+);
+
+-- Pending operations table (crash recovery atomicity)
+CREATE TABLE IF NOT EXISTS pending_operations (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  operation_type TEXT NOT NULL,
+  target_id TEXT NOT NULL,
+  payload TEXT,
+  created_at INTEGER NOT NULL,
+  attempts INTEGER NOT NULL DEFAULT 0,
+  last_attempt_at INTEGER,
+  error TEXT,
+  CONSTRAINT valid_operation_type CHECK (operation_type IN ('process', 'archive', 'cleanup'))
+);
+
+CREATE INDEX IF NOT EXISTS idx_pending_ops_type ON pending_operations(operation_type);
+CREATE INDEX IF NOT EXISTS idx_pending_ops_target ON pending_operations(target_id);
+    `,
+  },
+  {
+    name: '0002_symlink_paths',
+    sql: `
+-- Migration 0002: Add symlink path tracking for production workspace support
+-- Workspaces in cloud/production may be symlinked for isolation.
+-- We store both the original symlink path (for debugging) and canonical path (for operations).
+
+-- Add symlink_path column to track original path (may be symlink)
+ALTER TABLE relay_files ADD COLUMN symlink_path TEXT;
+
+-- Create index for symlink lookups
+CREATE INDEX IF NOT EXISTS idx_relay_files_symlink ON relay_files(symlink_path);
+    `,
+  },
+];
+
+// ============================================================================
+// Migration Runner
+// ============================================================================
+
+/**
+ * Create the migrations tracking table if it doesn't exist
+ */
+export function ensureMigrationsTable(db: Database.Database): void {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS __migrations (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      name TEXT NOT NULL UNIQUE,
+      applied_at INTEGER NOT NULL,
+      checksum TEXT NOT NULL
+    );
+  `);
+}
+
+/**
+ * Get all applied migrations
+ */
+export function getAppliedMigrations(db: Database.Database): MigrationRecord[] {
+  const rows = db.prepare(`
+    SELECT id, name, applied_at, checksum FROM __migrations ORDER BY id ASC
+  `).all() as Array<{ id: number; name: string; applied_at: number; checksum: string }>;
+
+  return rows.map(row => ({
+    id: row.id,
+    name: row.name,
+    appliedAt: row.applied_at,
+    checksum: row.checksum,
+  }));
+}
+
+/**
+ * Check if a specific migration has been applied
+ */
+export function isMigrationApplied(db: Database.Database, name: string): boolean {
+  const row = db.prepare(`SELECT 1 FROM __migrations WHERE name = ?`).get(name);
+  return row !== undefined;
+}
+
+/**
+ * Record a migration as applied
+ */
+export function recordMigration(db: Database.Database, name: string, checksum: string): void {
+  db.prepare(`
+    INSERT INTO __migrations (name, applied_at, checksum) VALUES (?, ?, ?)
+  `).run(name, Date.now(), checksum);
+}
+
+/**
+ * Calculate a simple checksum for migration content
+ */
+export function calculateChecksum(sql: string): string {
+  // Simple hash for migration verification
+  let hash = 0;
+  for (let i = 0; i < sql.length; i++) {
+    const char = sql.charCodeAt(i);
+    hash = ((hash << 5) - hash) + char;
+    hash = hash & hash; // Convert to 32-bit integer
+  }
+  return Math.abs(hash).toString(16).padStart(8, '0');
+}
+
+/**
+ * Load migrations from embedded definitions
+ */
+export function loadMigrationFiles(): MigrationFile[] {
+  return EMBEDDED_MIGRATIONS.map(m => ({
+    name: m.name,
+    sql: m.sql,
+    checksum: calculateChecksum(m.sql),
+  }));
+}
+
+/**
+ * Run all pending migrations
+ */
+export function runMigrations(db: Database.Database): MigrationResult {
+  const result: MigrationResult = {
+    applied: [],
+    skipped: [],
+    errors: [],
+  };
+
+  // Ensure migrations table exists
+  ensureMigrationsTable(db);
+
+  // Load all migrations
+  const migrations = loadMigrationFiles();
+
+  // Get applied migrations
+  const applied = new Set(getAppliedMigrations(db).map(m => m.name));
+
+  // Run pending migrations in order
+  for (const migration of migrations) {
+    if (applied.has(migration.name)) {
+      result.skipped.push(migration.name);
+      continue;
+    }
+
+    try {
+      // Run migration in a transaction
+      db.transaction(() => {
+        db.exec(migration.sql);
+        recordMigration(db, migration.name, migration.checksum);
+      })();
+
+      result.applied.push(migration.name);
+    } catch (err) {
+      const errorMessage = err instanceof Error ? err.message : String(err);
+      result.errors.push({ name: migration.name, error: errorMessage });
+      // Stop on first error
+      break;
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Get pending migrations that haven't been applied yet
+ */
+export function getPendingMigrations(db: Database.Database): MigrationFile[] {
+  ensureMigrationsTable(db);
+  const applied = new Set(getAppliedMigrations(db).map(m => m.name));
+  return loadMigrationFiles().filter(m => !applied.has(m.name));
+}
+
+/**
+ * Verify migration checksums match what was originally applied
+ */
+export function verifyMigrations(db: Database.Database): Array<{ name: string; expected: string; actual: string }> {
+  const applied = getAppliedMigrations(db);
+  const files = loadMigrationFiles();
+  const fileMap = new Map(files.map(f => [f.name, f]));
+  const mismatches: Array<{ name: string; expected: string; actual: string }> = [];
+
+  for (const record of applied) {
+    const file = fileMap.get(record.name);
+    if (file && file.checksum !== record.checksum) {
+      mismatches.push({
+        name: record.name,
+        expected: record.checksum,
+        actual: file.checksum,
+      });
+    }
+  }
+
+  return mismatches;
+}

package/packages/daemon/src/orchestrator.test.ts

@@ -0,0 +1,231 @@
+/**
+ * Orchestrator Health Monitoring Tests
+ */
+
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+
+// Mock child_process for process checking
+vi.mock('child_process', () => ({
+  execSync: vi.fn(() => ''),
+}));
+
+// Test the health state management without starting the full orchestrator
+describe('Orchestrator Health Monitoring', () => {
+  describe('AgentHealthState', () => {
+    it('should track agent health state structure', () => {
+      // Verify the interface structure by creating a mock
+      const health = {
+        key: 'workspace1:agent1',
+        workspaceId: 'workspace1',
+        agentName: 'agent1',
+        pid: 12345,
+        lastHeartbeatAt: new Date(),
+        lastSampleAt: new Date(),
+        lastRssBytes: 100 * 1024 * 1024, // 100MB
+        lastCpuPercent: 25.5,
+        releasing: false,
+        lastCpuAlertAt: undefined,
+      };
+
+      expect(health.key).toBe('workspace1:agent1');
+      expect(health.pid).toBe(12345);
+      expect(health.releasing).toBe(false);
+    });
+  });
+
+  describe('isProcessAlive', () => {
+    it('should return true for current process', () => {
+      // Current process should always be alive
+      let alive = false;
+      try {
+        process.kill(process.pid, 0);
+        alive = true;
+      } catch {
+        alive = false;
+      }
+      expect(alive).toBe(true);
+    });
+
+    it('should return false for non-existent PID', () => {
+      // PID 99999999 should not exist
+      let alive = false;
+      try {
+        process.kill(99999999, 0);
+        alive = true;
+      } catch {
+        alive = false;
+      }
+      expect(alive).toBe(false);
+    });
+  });
+
+  describe('CPU Alert Threshold', () => {
+    it('should use default CPU threshold of 300%', () => {
+      // Default is 300% (allows for multi-core usage)
+      const defaultThreshold = 300;
+      expect(defaultThreshold).toBe(300);
+    });
+
+    it('should parse CPU threshold from environment', () => {
+      const envValue = '150';
+      const parsed = parseFloat(envValue);
+      expect(parsed).toBe(150);
+      expect(Number.isFinite(parsed)).toBe(true);
+    });
+  });
+
+  describe('Health Registration Flow', () => {
+    it('should create proper health key', () => {
+      const workspaceId = 'ws-123';
+      const agentName = 'test-agent';
+      const key = `${workspaceId}:${agentName}`;
+      expect(key).toBe('ws-123:test-agent');
+    });
+
+    it('should track releasing state', () => {
+      // Simulating the flow:
+      // 1. Agent spawns -> releasing = false
+      // 2. stopAgent called -> releasing = true
+      // 3. Heartbeat check -> skips crash detection because releasing = true
+
+      const health = {
+        key: 'ws1:agent1',
+        workspaceId: 'ws1',
+        agentName: 'agent1',
+        pid: 1234,
+        releasing: false,
+      };
+
+      // Before stop
+      expect(health.releasing).toBe(false);
+
+      // During stop
+      health.releasing = true;
+      expect(health.releasing).toBe(true);
+
+      // Heartbeat check would skip crash announcement
+      const isAlive = false; // Process died
+      const shouldAnnounce = !isAlive && !health.releasing;
+      expect(shouldAnnounce).toBe(false);
+    });
+  });
+
+  describe('Resource Alert Cooldown', () => {
+    it('should respect cooldown period for CPU alerts', () => {
+      const RESOURCE_ALERT_COOLDOWN_MS = 60_000;
+      const lastAlertAt = Date.now() - 30_000; // 30 seconds ago
+      const now = Date.now();
+
+      const inCooldown = lastAlertAt && now - lastAlertAt < RESOURCE_ALERT_COOLDOWN_MS;
+      expect(inCooldown).toBe(true);
+    });
+
+    it('should allow alert after cooldown expires', () => {
+      const RESOURCE_ALERT_COOLDOWN_MS = 60_000;
+      const lastAlertAt = Date.now() - 90_000; // 90 seconds ago
+      const now = Date.now();
+
+      const inCooldown = lastAlertAt && now - lastAlertAt < RESOURCE_ALERT_COOLDOWN_MS;
+      expect(inCooldown).toBe(false);
+    });
+  });
+
+  describe('Crash Context', () => {
+    it('should categorize crash causes', () => {
+      const causes = ['oom', 'memory_leak', 'sudden_spike', 'unknown'] as const;
+
+      // Test OOM detection
+      const oomMemory = 2 * 1024 * 1024 * 1024; // 2GB
+      const oomThreshold = 1.5 * 1024 * 1024 * 1024; // 1.5GB
+      const isOom = oomMemory >= oomThreshold;
+      expect(isOom).toBe(true);
+
+      // Test memory leak detection
+      const growthRate = 15 * 1024 * 1024; // 15MB/min
+      const leakThreshold = 10 * 1024 * 1024; // 10MB/min
+      const isLeak = growthRate > leakThreshold;
+      expect(isLeak).toBe(true);
+
+      // Test sudden spike detection
+      const currentMemory = 500 * 1024 * 1024;
+      const previousMemory = 300 * 1024 * 1024;
+      const spike = currentMemory - previousMemory;
+      const isSpike = spike > 100 * 1024 * 1024; // 100MB threshold
+      expect(isSpike).toBe(true);
+    });
+  });
+});
+
+describe('Health Monitoring Integration', () => {
+  describe('Event Broadcasting', () => {
+    it('should structure crash event correctly', () => {
+      const event = {
+        type: 'agent:crashed',
+        workspaceId: 'ws-123',
+        data: {
+          name: 'test-agent',
+          pid: 12345,
+          crashContext: {
+            likelyCause: 'oom',
+            peakMemory: 1024 * 1024 * 1024,
+            averageMemory: 800 * 1024 * 1024,
+            memoryTrend: 'growing',
+            analysisNotes: ['Memory was at OOM-imminent level'],
+          },
+        },
+        timestamp: new Date(),
+      };
+
+      expect(event.type).toBe('agent:crashed');
+      expect(event.data.crashContext.likelyCause).toBe('oom');
+    });
+
+    it('should structure resource alert event correctly', () => {
+      const event = {
+        type: 'agent:resource-alert',
+        workspaceId: 'ws-123',
+        agentId: 'test-agent',
+        data: {
+          name: 'test-agent',
+          resourceType: 'memory',
+          currentValue: 600 * 1024 * 1024,
+          alertLevel: 'warning',
+          message: 'Agent "test-agent" memory usage is elevated',
+          recommendation: 'Keep monitoring, consider investigation if trend continues',
+        },
+        timestamp: new Date(),
+      };
+
+      expect(event.type).toBe('agent:resource-alert');
+      expect(event.data.resourceType).toBe('memory');
+      expect(event.data.alertLevel).toBe('warning');
+    });
+  });
+
+  describe('System Message Broadcasting', () => {
+    it('should format crash message correctly', () => {
+      const agentName = 'worker-1';
+      const pid = 12345;
+      const likelyCause = 'oom';
+      const analysisNotes = ['Memory was at OOM-imminent level', 'Peak memory: 1.5 GB'];
+
+      const message = likelyCause !== 'unknown'
+        ? `AGENT CRASHED: "${agentName}" has died unexpectedly (PID: ${pid}). Likely cause: ${likelyCause}. ${analysisNotes.slice(0, 2).join('. ')}`
+        : `AGENT CRASHED: "${agentName}" has died unexpectedly (PID: ${pid}).`;
+
+      expect(message).toContain('AGENT CRASHED');
+      expect(message).toContain(agentName);
+      expect(message).toContain('Likely cause: oom');
+    });
+
+    it('should format resource alert message correctly', () => {
+      const agentName = 'worker-1';
+      const cpuPercent = 350.5;
+
+      const message = `RESOURCE ALERT: "${agentName}" is running at ${cpuPercent.toFixed(1)}% CPU. Consider reducing workload.`;
+
+      expect(message).toContain('RESOURCE ALERT');
+      expect(message).toContain('350.5% CPU');
+    });
+  });
+});