agent-relay 1.6.0 → 2.0.0

package/packages/daemon/dist/relay-ledger.js

@@ -0,0 +1,532 @@
+/**
+ * Relay Ledger - SQLite-based tracking of relay file processing
+ *
+ * Tracks files discovered in agent outboxes through their lifecycle:
+ * pending -> processing -> delivered/failed -> archived
+ *
+ * Features:
+ * - Atomic claims with status WHERE clause
+ * - Crash recovery (reset processing -> pending on startup)
+ * - Archive path tracking for auditing
+ * - Configurable retention
+ * - Schema migrations for version upgrades
+ */
+import Database from 'better-sqlite3';
+import path from 'node:path';
+import fs from 'node:fs';
+import crypto from 'node:crypto';
+import { runMigrations, getPendingMigrations, verifyMigrations } from './migrations/index.js';
+// ============================================================================
+// Constants
+// ============================================================================
+const DEFAULT_MAX_RETRIES = 3;
+const DEFAULT_ARCHIVE_RETENTION_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
+const DEFAULT_BUSY_TIMEOUT = 5000;
+// Reserved agent names for special routing
+const RESERVED_AGENT_NAMES = new Set(['Lead', 'System', 'Broadcast', '*']);
+// ============================================================================
+// RelayLedger Class
+// ============================================================================
+export class RelayLedger {
+    db;
+    config;
+    // Prepared statements for performance
+    stmtInsert;
+    stmtClaim;
+    stmtUpdateStatus;
+    stmtMarkDelivered;
+    stmtMarkFailed;
+    stmtMarkArchived;
+    stmtGetPending;
+    stmtGetByPath;
+    stmtGetById;
+    stmtResetProcessing;
+    stmtCleanupArchived;
+    stmtGetStats;
+    constructor(config) {
+        this.config = {
+            dbPath: config.dbPath,
+            maxRetries: config.maxRetries ?? DEFAULT_MAX_RETRIES,
+            archiveRetentionMs: config.archiveRetentionMs ?? DEFAULT_ARCHIVE_RETENTION_MS,
+            busyTimeout: config.busyTimeout ?? DEFAULT_BUSY_TIMEOUT,
+        };
+        // Ensure directory exists
+        const dbDir = path.dirname(this.config.dbPath);
+        if (!fs.existsSync(dbDir)) {
+            fs.mkdirSync(dbDir, { recursive: true });
+        }
+        // Open database with recommended settings
+        this.db = new Database(this.config.dbPath);
+        this.db.pragma('journal_mode = WAL');
+        this.db.pragma(`busy_timeout = ${this.config.busyTimeout}`);
+        this.db.pragma('synchronous = NORMAL');
+        this.db.pragma('foreign_keys = ON');
+        this.initSchema();
+        this.prepareStatements();
+    }
+    /** Last migration result (for debugging) */
+    lastMigrationResult;
+    /**
+     * Initialize database schema using migrations
+     */
+    initSchema() {
+        // Run migrations (creates tables if they don't exist, or upgrades existing schema)
+        this.lastMigrationResult = runMigrations(this.db);
+        // Log migration results for debugging
+        if (this.lastMigrationResult.applied.length > 0) {
+            console.log(`[relay-ledger] Applied migrations: ${this.lastMigrationResult.applied.join(', ')}`);
+        }
+        if (this.lastMigrationResult.errors.length > 0) {
+            console.error(`[relay-ledger] Migration errors:`, this.lastMigrationResult.errors);
+        }
+    }
+    /**
+     * Get the last migration result
+     */
+    getMigrationResult() {
+        return this.lastMigrationResult;
+    }
+    /**
+     * Get pending migrations that haven't been applied
+     */
+    getPendingMigrations() {
+        return getPendingMigrations(this.db);
+    }
+    /**
+     * Verify migration checksums match what was originally applied
+     */
+    verifyMigrationIntegrity() {
+        return verifyMigrations(this.db);
+    }
+    /**
+     * Prepare frequently-used statements
+     */
+    prepareStatements() {
+        this.stmtInsert = this.db.prepare(`
+      INSERT INTO relay_files (file_id, source_path, symlink_path, agent_name, message_type, discovered_at, file_size, content_hash, file_mtime_ns, file_inode)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(file_id) DO NOTHING
+    `);
+        this.stmtClaim = this.db.prepare(`
+      UPDATE relay_files
+      SET status = 'processing', retries = retries + 1
+      WHERE file_id = ? AND status = 'pending' AND retries < max_retries
+      RETURNING *
+    `);
+        this.stmtUpdateStatus = this.db.prepare(`
+      UPDATE relay_files SET status = ? WHERE file_id = ?
+    `);
+        this.stmtMarkDelivered = this.db.prepare(`
+      UPDATE relay_files
+      SET status = 'delivered', processed_at = ?
+      WHERE file_id = ?
+    `);
+        this.stmtMarkFailed = this.db.prepare(`
+      UPDATE relay_files
+      SET status = 'failed', processed_at = ?, error = ?
+      WHERE file_id = ?
+    `);
+        this.stmtMarkArchived = this.db.prepare(`
+      UPDATE relay_files
+      SET status = 'archived', archive_path = ?, archived_at = ?
+      WHERE file_id = ?
+    `);
+        this.stmtGetPending = this.db.prepare(`
+      SELECT * FROM relay_files
+      WHERE status = 'pending' AND retries < max_retries
+      ORDER BY id ASC
+      LIMIT ?
+    `);
+        this.stmtGetByPath = this.db.prepare(`
+      SELECT * FROM relay_files WHERE source_path = ?
+    `);
+        this.stmtGetById = this.db.prepare(`
+      SELECT * FROM relay_files WHERE file_id = ?
+    `);
+        this.stmtResetProcessing = this.db.prepare(`
+      UPDATE relay_files
+      SET status = 'pending'
+      WHERE status = 'processing'
+    `);
+        this.stmtCleanupArchived = this.db.prepare(`
+      DELETE FROM relay_files
+      WHERE status = 'archived' AND archived_at < ?
+    `);
+        this.stmtGetStats = this.db.prepare(`
+      SELECT status, COUNT(*) as count FROM relay_files GROUP BY status
+    `);
+    }
+    /**
+     * Generate a unique file ID (12-char hex for ~281 trillion combinations)
+     */
+    generateFileId() {
+        return crypto.randomBytes(6).toString('hex');
+    }
+    /**
+     * Check if an agent name is reserved
+     */
+    isReservedAgentName(name) {
+        return RESERVED_AGENT_NAMES.has(name);
+    }
+    /**
+     * Register a newly discovered file
+     * @param sourcePath - Canonical path (symlinks resolved)
+     * @param agentName - Agent that owns the file
+     * @param messageType - Type of message (msg, spawn, release, etc.)
+     * @param fileSize - Size in bytes
+     * @param contentHash - Optional hash for deduplication
+     * @param fileMtimeNs - Optional modification time in nanoseconds
+     * @param fileInode - Optional inode number
+     * @param symlinkPath - Original path if it was a symlink (for debugging)
+     */
+    registerFile(sourcePath, agentName, messageType, fileSize, contentHash, fileMtimeNs, fileInode, symlinkPath) {
+        const fileId = this.generateFileId();
+        const now = Date.now();
+        this.stmtInsert.run(fileId, sourcePath, symlinkPath ?? null, agentName, messageType, now, fileSize, contentHash ?? null, fileMtimeNs ?? null, fileInode ?? null);
+        return fileId;
+    }
+    /**
+     * Check if a file is already registered
+     */
+    isFileRegistered(sourcePath) {
+        const row = this.stmtGetByPath.get(sourcePath);
+        return row !== undefined;
+    }
+    /**
+     * Atomically claim a file for processing
+     * Returns the record if claim succeeded, null otherwise
+     */
+    claimFile(fileId) {
+        const row = this.stmtClaim.get(fileId);
+        if (!row) {
+            // Check why claim failed
+            const existing = this.stmtGetById.get(fileId);
+            if (!existing) {
+                return { success: false, reason: 'File not found' };
+            }
+            if (existing.status !== 'pending') {
+                return { success: false, reason: `File status is ${existing.status}, not pending` };
+            }
+            if (existing.retries >= existing.max_retries) {
+                return { success: false, reason: `Max retries (${existing.max_retries}) exceeded` };
+            }
+            return { success: false, reason: 'Unknown claim failure' };
+        }
+        return {
+            success: true,
+            record: this.rowToRecord(row),
+        };
+    }
+    /**
+     * Mark a file as successfully delivered
+     */
+    markDelivered(fileId) {
+        this.stmtMarkDelivered.run(Date.now(), fileId);
+    }
+    /**
+     * Mark a file as failed (will retry if under max retries)
+     */
+    markFailed(fileId, error) {
+        const record = this.getById(fileId);
+        if (!record)
+            return;
+        if (record.retries >= record.maxRetries) {
+            // Permanent failure
+            this.stmtMarkFailed.run(Date.now(), error, fileId);
+        }
+        else {
+            // Reset to pending for retry
+            this.stmtUpdateStatus.run('pending', fileId);
+        }
+    }
+    /**
+     * Mark a file as archived (moved to archive location)
+     */
+    markArchived(fileId, archivePath) {
+        this.stmtMarkArchived.run(archivePath, Date.now(), fileId);
+    }
+    /**
+     * Get pending files ready for processing
+     */
+    getPendingFiles(limit = 100) {
+        const rows = this.stmtGetPending.all(limit);
+        return rows.map(row => this.rowToRecord(row));
+    }
+    /**
+     * Get a file record by ID
+     */
+    getById(fileId) {
+        const row = this.stmtGetById.get(fileId);
+        return row ? this.rowToRecord(row) : null;
+    }
+    /**
+     * Get a file record by source path
+     */
+    getByPath(sourcePath) {
+        const row = this.stmtGetByPath.get(sourcePath);
+        return row ? this.rowToRecord(row) : null;
+    }
+    /**
+     * Reset all 'processing' files to 'pending' (crash recovery)
+     */
+    resetProcessingFiles() {
+        const result = this.stmtResetProcessing.run();
+        return result.changes;
+    }
+    /**
+     * Mark files as failed if their source file no longer exists
+     */
+    reconcileWithFilesystem() {
+        let reset = 0;
+        let failed = 0;
+        // Get all processing files
+        const processingFiles = this.db
+            .prepare(`SELECT * FROM relay_files WHERE status = 'processing'`)
+            .all();
+        for (const row of processingFiles) {
+            if (!fs.existsSync(row.source_path)) {
+                // File was deleted while processing - mark as failed
+                this.stmtMarkFailed.run(Date.now(), 'Source file deleted during processing', row.file_id);
+                failed++;
+            }
+            else {
+                // Reset to pending
+                this.stmtUpdateStatus.run('pending', row.file_id);
+                reset++;
+            }
+        }
+        return { reset, failed };
+    }
+    /**
+     * Clean up old archived records
+     */
+    cleanupArchivedRecords() {
+        const cutoff = Date.now() - this.config.archiveRetentionMs;
+        const result = this.stmtCleanupArchived.run(cutoff);
+        return result.changes;
+    }
+    /**
+     * Get statistics about file processing
+     */
+    getStats() {
+        const rows = this.stmtGetStats.all();
+        const stats = {
+            pending: 0,
+            processing: 0,
+            delivered: 0,
+            failed: 0,
+            archived: 0,
+        };
+        for (const row of rows) {
+            stats[row.status] = row.count;
+        }
+        return stats;
+    }
+    /**
+     * Convert database row to RelayFileRecord
+     */
+    rowToRecord(row) {
+        return {
+            id: row.id,
+            fileId: row.file_id,
+            sourcePath: row.source_path,
+            symlinkPath: row.symlink_path,
+            archivePath: row.archive_path,
+            agentName: row.agent_name,
+            messageType: row.message_type,
+            status: row.status,
+            retries: row.retries,
+            maxRetries: row.max_retries,
+            discoveredAt: row.discovered_at,
+            processedAt: row.processed_at,
+            archivedAt: row.archived_at,
+            error: row.error,
+            contentHash: row.content_hash,
+            fileSize: row.file_size,
+            fileMtimeNs: row.file_mtime_ns,
+            fileInode: row.file_inode,
+        };
+    }
+    // ==========================================================================
+    // Agent Management
+    // ==========================================================================
+    /**
+     * Register or update an agent
+     */
+    registerAgent(agentName, metadata) {
+        const now = Date.now();
+        const metadataJson = metadata ? JSON.stringify(metadata) : null;
+        this.db.prepare(`
+      INSERT INTO agents (agent_name, created_at, last_seen_at, status, metadata)
+      VALUES (?, ?, ?, 'active', ?)
+      ON CONFLICT(agent_name) DO UPDATE SET
+        last_seen_at = excluded.last_seen_at,
+        status = 'active',
+        metadata = COALESCE(excluded.metadata, agents.metadata)
+    `).run(agentName, now, now, metadataJson);
+    }
+    /**
+     * Update agent last seen time
+     */
+    updateAgentLastSeen(agentName) {
+        this.db.prepare(`
+      UPDATE agents SET last_seen_at = ? WHERE agent_name = ?
+    `).run(Date.now(), agentName);
+    }
+    /**
+     * Mark agent as inactive
+     */
+    markAgentInactive(agentName) {
+        this.db.prepare(`
+      UPDATE agents SET status = 'inactive' WHERE agent_name = ?
+    `).run(agentName);
+    }
+    /**
+     * Get all active agents
+     */
+    getActiveAgents() {
+        const rows = this.db.prepare(`
+      SELECT * FROM agents WHERE status = 'active' ORDER BY last_seen_at DESC
+    `).all();
+        return rows.map(row => ({
+            id: row.id,
+            agentName: row.agent_name,
+            createdAt: row.created_at,
+            lastSeenAt: row.last_seen_at,
+            status: row.status,
+            metadata: row.metadata ? JSON.parse(row.metadata) : null,
+        }));
+    }
+    /**
+     * Get agent by name
+     */
+    getAgent(agentName) {
+        const row = this.db.prepare(`
+      SELECT * FROM agents WHERE agent_name = ?
+    `).get(agentName);
+        if (!row)
+            return null;
+        return {
+            id: row.id,
+            agentName: row.agent_name,
+            createdAt: row.created_at,
+            lastSeenAt: row.last_seen_at,
+            status: row.status,
+            metadata: row.metadata ? JSON.parse(row.metadata) : null,
+        };
+    }
+    // ==========================================================================
+    // Orchestrator State
+    // ==========================================================================
+    /**
+     * Save orchestrator state
+     */
+    saveState(key, value) {
+        this.db.prepare(`
+      INSERT INTO orchestrator_state (key, value, updated_at)
+      VALUES (?, ?, ?)
+      ON CONFLICT(key) DO UPDATE SET
+        value = excluded.value,
+        updated_at = excluded.updated_at
+    `).run(key, value, Date.now());
+    }
+    /**
+     * Get orchestrator state
+     */
+    getState(key) {
+        const row = this.db.prepare(`
+      SELECT value FROM orchestrator_state WHERE key = ?
+    `).get(key);
+        return row?.value ?? null;
+    }
+    /**
+     * Delete orchestrator state
+     */
+    deleteState(key) {
+        this.db.prepare(`
+      DELETE FROM orchestrator_state WHERE key = ?
+    `).run(key);
+    }
+    /**
+     * Get all orchestrator state
+     */
+    getAllState() {
+        const rows = this.db.prepare(`
+      SELECT * FROM orchestrator_state ORDER BY key
+    `).all();
+        return rows.map(row => ({
+            key: row.key,
+            value: row.value,
+            updatedAt: row.updated_at,
+        }));
+    }
+    // ==========================================================================
+    // Pending Operations (for crash recovery)
+    // ==========================================================================
+    /**
+     * Add a pending operation
+     */
+    addPendingOperation(operationType, targetId, payload) {
+        const result = this.db.prepare(`
+      INSERT INTO pending_operations (operation_type, target_id, payload, created_at)
+      VALUES (?, ?, ?, ?)
+    `).run(operationType, targetId, payload ?? null, Date.now());
+        return result.lastInsertRowid;
+    }
+    /**
+     * Complete a pending operation (remove it)
+     */
+    completePendingOperation(id) {
+        this.db.prepare(`
+      DELETE FROM pending_operations WHERE id = ?
+    `).run(id);
+    }
+    /**
+     * Fail a pending operation (increment attempts, record error)
+     */
+    failPendingOperation(id, error) {
+        this.db.prepare(`
+      UPDATE pending_operations
+      SET attempts = attempts + 1, last_attempt_at = ?, error = ?
+      WHERE id = ?
+    `).run(Date.now(), error, id);
+    }
+    /**
+     * Get pending operations for recovery
+     */
+    getPendingOperations(maxAttempts = 5) {
+        const rows = this.db.prepare(`
+      SELECT * FROM pending_operations
+      WHERE attempts < ?
+      ORDER BY created_at ASC
+    `).all(maxAttempts);
+        return rows.map(row => ({
+            id: row.id,
+            operationType: row.operation_type,
+            targetId: row.target_id,
+            payload: row.payload,
+            createdAt: row.created_at,
+            attempts: row.attempts,
+            lastAttemptAt: row.last_attempt_at,
+            error: row.error,
+        }));
+    }
+    /**
+     * Cleanup old failed operations
+     */
+    cleanupFailedOperations(maxAge = 24 * 60 * 60 * 1000) {
+        const cutoff = Date.now() - maxAge;
+        const result = this.db.prepare(`
+      DELETE FROM pending_operations
+      WHERE created_at < ? AND attempts >= 5
+    `).run(cutoff);
+        return result.changes;
+    }
+    /**
+     * Close the database connection
+     */
+    close() {
+        this.db.close();
+    }
+}
+//# sourceMappingURL=relay-ledger.js.map

package/packages/daemon/dist/relay-watchdog.d.ts

@@ -0,0 +1,125 @@
+/**
+ * Relay Watchdog - File-based relay message detection and processing
+ *
+ * Monitors agent outbox directories for new relay files and processes them:
+ * 1. Detects new files via fs.watch + periodic reconciliation
+ * 2. Validates files (size > 0, not symlink, settled)
+ * 3. Claims files atomically via ledger
+ * 4. Processes and archives files
+ *
+ * Features:
+ * - fsevents/inotify watchers with overflow fallback
+ * - Configurable settle time for file stability
+ * - Symlink rejection (security)
+ * - Orphaned .pending file cleanup
+ * - Crash recovery on startup
+ */
+import { EventEmitter } from 'node:events';
+import { type RelayFileRecord } from './relay-ledger.js';
+import { type RelayPaths } from '@agent-relay/config/relay-file-writer';
+export interface WatchdogConfig {
+    /** Base relay paths (auto-detected if not provided) */
+    relayPaths?: RelayPaths;
+    /** Ledger database path (default: ~/.agent-relay/meta/ledger.sqlite) */
+    ledgerPath?: string;
+    /** Settle time before processing file (default: 500ms) */
+    settleTimeMs?: number;
+    /** Timeout for malformed/incomplete files (default: 10000ms) */
+    malformedTimeoutMs?: number;
+    /** Interval for periodic reconciliation (default: 30000ms) */
+    reconcileIntervalMs?: number;
+    /** Maximum message size in bytes (default: 1MB) */
+    maxMessageSizeBytes?: number;
+    /** Maximum attachment size in bytes (default: 10MB) */
+    maxAttachmentSizeBytes?: number;
+    /** Cleanup interval for orphaned files (default: 60000ms) */
+    cleanupIntervalMs?: number;
+    /** Age threshold for orphaned .pending files (default: 30000ms) */
+    orphanedPendingAgeMs?: number;
+    /** Archive retention in milliseconds (default: 7 days) */
+    archiveRetentionMs?: number;
+    /** Enable debug logging */
+    debug?: boolean;
+}
+export interface DiscoveredFile {
+    path: string;
+    agentName: string;
+    messageType: string;
+    size: number;
+    mtime: number;
+    contentHash?: string;
+}
+export interface ProcessedFile {
+    fileId: string;
+    agentName: string;
+    messageType: string;
+    content: string;
+    headers: Record<string, string>;
+    body: string;
+}
+export interface WatchdogEvents {
+    'file:discovered': (file: DiscoveredFile) => void;
+    'file:processing': (record: RelayFileRecord) => void;
+    'file:delivered': (result: ProcessedFile) => void;
+    'file:failed': (record: RelayFileRecord, error: Error) => void;
+    'file:archived': (record: RelayFileRecord, archivePath: string) => void;
+    'watcher:overflow': (dir: string) => void;
+    'reconcile:complete': (stats: {
+        discovered: number;
+        failed: number;
+    }) => void;
+    error: (error: Error) => void;
+}
+export declare class RelayWatchdog extends EventEmitter {
+    private config;
+    private relayPaths;
+    private ledger;
+    private watchers;
+    private pendingFiles;
+    private reconcileTimer?;
+    private cleanupTimer?;
+    private running;
+    constructor(config?: WatchdogConfig);
+    /**
+     * Resolve symlinks in relay paths for cloud/production workspace support
+     * This ensures we work with canonical paths even when directories are symlinked
+     */
+    private resolveRelayPaths;
+    /**
+     * Start the watchdog
+     */
+    start(): Promise<void>;
+    /**
+     * Stop the watchdog
+     */
+    stop(): Promise<void>;
+    /**
+     * Get ledger statistics
+     */
+    getStats(): Record<string, number>;
+    /**
+     * Get pending file count
+     */
+    getPendingCount(): number;
+    private ensureDirectories;
+    private startRootWatcher;
+    private startAgentWatchers;
+    private checkAndWatchAgentDir;
+    private startAgentWatcher;
+    private handleFileEvent;
+    private processDiscoveredFile;
+    private validateFile;
+    private calculateFileHash;
+    private processFile;
+    private parseFileContent;
+    private archiveFile;
+    private fullReconciliation;
+    private cleanupOrphanedFiles;
+    private shouldIgnore;
+    private log;
+}
+export interface RelayWatchdog {
+    on<K extends keyof WatchdogEvents>(event: K, listener: WatchdogEvents[K]): this;
+    emit<K extends keyof WatchdogEvents>(event: K, ...args: Parameters<WatchdogEvents[K]>): boolean;
+}
+//# sourceMappingURL=relay-watchdog.d.ts.map