agent-relay 2.0.29 → 2.0.33

package/packages/daemon/src/delivery-tracker.ts

@@ -0,0 +1,145 @@
+import type { DeliverEnvelope, AckPayload, Envelope } from '@agent-relay/protocol/types';
+import type { StorageAdapter } from '@agent-relay/storage/adapter';
+import { routerLog } from '@agent-relay/utils/logger';
+
+export interface DeliveryReliabilityOptions {
+  /** How long to wait for an ACK before retrying (ms) */
+  ackTimeoutMs: number;
+  /** Maximum attempts (initial send counts as attempt 1) */
+  maxAttempts: number;
+  /** How long to keep retrying before dropping (ms) */
+  deliveryTtlMs: number;
+}
+
+export const DEFAULT_DELIVERY_OPTIONS: DeliveryReliabilityOptions = {
+  ackTimeoutMs: 5000,
+  maxAttempts: 5,
+  deliveryTtlMs: 60_000,
+};
+
+export interface DeliveryTrackerConnection {
+  id: string;
+  send(envelope: DeliverEnvelope): boolean;
+}
+
+interface PendingDelivery {
+  envelope: DeliverEnvelope;
+  connectionId: string;
+  attempts: number;
+  firstSentAt: number;
+  timer?: NodeJS.Timeout;
+}
+
+export class DeliveryTracker {
+  private pendingDeliveries: Map<string, PendingDelivery> = new Map();
+  private deliveryOptions: DeliveryReliabilityOptions;
+  private storage?: StorageAdapter;
+  private getConnection: (id: string) => DeliveryTrackerConnection | undefined;
+
+  constructor(options: {
+    storage?: StorageAdapter;
+    delivery?: Partial<DeliveryReliabilityOptions>;
+    getConnection: (id: string) => DeliveryTrackerConnection | undefined;
+  }) {
+    this.storage = options.storage;
+    this.deliveryOptions = { ...DEFAULT_DELIVERY_OPTIONS, ...options.delivery };
+    this.getConnection = options.getConnection;
+  }
+
+  get pendingCount(): number {
+    return this.pendingDeliveries.size;
+  }
+
+  track(target: DeliveryTrackerConnection, deliver: DeliverEnvelope): void {
+    const pending: PendingDelivery = {
+      envelope: deliver,
+      connectionId: target.id,
+      attempts: 1,
+      firstSentAt: Date.now(),
+    };
+
+    pending.timer = this.scheduleRetry(deliver.id);
+    this.pendingDeliveries.set(deliver.id, pending);
+  }
+
+  handleAck(connectionId: string, ackId: string): void {
+    const pending = this.pendingDeliveries.get(ackId);
+    if (!pending) return;
+
+    // Only accept ACKs from the same connection that received the deliver
+    if (pending.connectionId !== connectionId) return;
+
+    if (pending.timer) {
+      clearTimeout(pending.timer);
+    }
+    this.pendingDeliveries.delete(ackId);
+    const statusUpdate = this.storage?.updateMessageStatus?.(ackId, 'acked');
+    if (statusUpdate instanceof Promise) {
+      statusUpdate.catch(err => {
+        routerLog.error('Failed to record ACK status', { error: String(err) });
+      });
+    }
+    routerLog.debug(`ACK received for ${ackId}`);
+  }
+
+  clearPendingForConnection(connectionId: string): void {
+    for (const [id, pending] of this.pendingDeliveries.entries()) {
+      if (pending.connectionId === connectionId) {
+        if (pending.timer) clearTimeout(pending.timer);
+        this.pendingDeliveries.delete(id);
+      }
+    }
+  }
+
+  private scheduleRetry(deliverId: string): NodeJS.Timeout | undefined {
+    return setTimeout(() => {
+      const pending = this.pendingDeliveries.get(deliverId);
+      if (!pending) return;
+
+      const now = Date.now();
+      const elapsed = now - pending.firstSentAt;
+      if (elapsed > this.deliveryOptions.deliveryTtlMs) {
+        routerLog.warn(`Dropping ${deliverId} after TTL`, { ttlMs: this.deliveryOptions.deliveryTtlMs });
+        this.pendingDeliveries.delete(deliverId);
+        this.markFailed(deliverId);
+        return;
+      }
+
+      if (pending.attempts >= this.deliveryOptions.maxAttempts) {
+        routerLog.warn(`Dropping ${deliverId} after max attempts`, { maxAttempts: this.deliveryOptions.maxAttempts });
+        this.pendingDeliveries.delete(deliverId);
+        this.markFailed(deliverId);
+        return;
+      }
+
+      const target = this.getConnection(pending.connectionId);
+      if (!target) {
+        routerLog.warn(`Dropping ${deliverId} - connection unavailable`);
+        this.pendingDeliveries.delete(deliverId);
+        this.markFailed(deliverId);
+        return;
+      }
+
+      pending.attempts++;
+      const sent = target.send(pending.envelope);
+      if (!sent) {
+        routerLog.warn(`Retry failed for ${deliverId}`, { attempt: pending.attempts });
+      } else {
+        routerLog.debug(`Retried ${deliverId}`, { attempt: pending.attempts });
+      }
+
+      pending.timer = this.scheduleRetry(deliverId);
+    }, this.deliveryOptions.ackTimeoutMs);
+  }
+
+  private markFailed(deliverId: string): void {
+    const statusUpdate = this.storage?.updateMessageStatus?.(deliverId, 'failed');
+    if (statusUpdate instanceof Promise) {
+      statusUpdate.catch(err => {
+        routerLog.error(`Failed to update status for ${deliverId}`, { error: String(err) });
+      });
+    }
+  }
+}
+
+export type AckEnvelope = Envelope<AckPayload>;

package/packages/daemon/src/enhanced-features.ts

@@ -0,0 +1,390 @@
+/**
+ * Enhanced Features Integration Module
+ *
+ * Wires together the new performance and reliability features:
+ * - Precompiled regex patterns
+ * - Agent authentication with signing
+ * - Dead Letter Queue
+ * - Context compaction
+ * - Consensus mechanism
+ *
+ * This module provides a unified interface for integrating
+ * these features into the existing daemon and router.
+ */
+
+import type { Database as BetterSqlite3Database } from 'better-sqlite3';
+import type { Pool as PgPool } from 'pg';
+
+// Import new modules
+import {
+  getCompiledPatterns,
+  isInstructionalTextFast,
+  isPlaceholderTargetFast,
+  stripAnsiFast,
+  StaticPatterns,
+} from '@agent-relay/utils/precompiled-patterns';
+
+import {
+  AgentSigningManager,
+  loadSigningConfig,
+  attachSignature,
+  extractSignature,
+  type AgentSigningConfig,
+  type SignedMessage,
+} from './agent-signing.js';
+
+import {
+  SQLiteDLQAdapter,
+  PostgresDLQAdapter,
+  InMemoryDLQAdapter,
+  createDLQAdapter,
+  DEFAULT_DLQ_CONFIG,
+  type DLQStorageAdapter,
+  type DLQConfig,
+  type DeadLetter,
+  type DLQStats,
+} from '@agent-relay/storage/dlq-adapter';
+
+import {
+  ContextCompactor,
+  createContextCompactor,
+  estimateTokens,
+  estimateContextTokens,
+  formatTokenCount,
+  type Message,
+  type CompactionConfig,
+  type CompactionResult,
+} from '@agent-relay/memory';
+
+import type { VoteValue } from '@agent-relay/protocol';
+import {
+  ConsensusEngine,
+  createConsensusEngine,
+  formatProposalMessage,
+  parseVoteCommand,
+  formatResultMessage,
+  type Proposal,
+  type ConsensusResult,
+  type ConsensusConfig,
+} from './consensus.js';
+
+// =============================================================================
+// Types
+// =============================================================================
+
+export interface EnhancedFeaturesConfig {
+  /** Pattern matching configuration */
+  patterns?: {
+    relayPrefix?: string;
+    thinkingPrefix?: string;
+  };
+
+  /** Signing configuration (or path to config file) */
+  signing?: Partial<AgentSigningConfig> | string;
+
+  /** DLQ configuration */
+  dlq?: Partial<DLQConfig> & {
+    /** Storage type */
+    type?: 'sqlite' | 'postgres' | 'memory';
+    /** SQLite database (if type is sqlite) */
+    sqlite?: BetterSqlite3Database;
+    /** PostgreSQL pool (if type is postgres) */
+    postgres?: PgPool;
+  };
+
+  /** Context compaction configuration */
+  compaction?: Partial<CompactionConfig>;
+
+  /** Consensus configuration */
+  consensus?: Partial<ConsensusConfig>;
+}
+
+export interface EnhancedFeatures {
+  /** Precompiled pattern matching */
+  patterns: {
+    compiled: ReturnType<typeof getCompiledPatterns>;
+    isInstructionalText: typeof isInstructionalTextFast;
+    isPlaceholderTarget: typeof isPlaceholderTargetFast;
+    stripAnsi: typeof stripAnsiFast;
+    static: typeof StaticPatterns;
+  };
+
+  /** Agent signing manager */
+  signing: AgentSigningManager;
+
+  /** Dead Letter Queue */
+  dlq: DLQStorageAdapter;
+
+  /** Context compactor */
+  compaction: ContextCompactor;
+
+  /** Consensus engine */
+  consensus: ConsensusEngine;
+
+  /** Cleanup function */
+  cleanup: () => Promise<void>;
+}
+
+// =============================================================================
+// Factory Function
+// =============================================================================
+
+/**
+ * Initialize all enhanced features.
+ */
+export async function initEnhancedFeatures(
+  config: EnhancedFeaturesConfig = {}
+): Promise<EnhancedFeatures> {
+  // Initialize pattern matching
+  const patterns = {
+    compiled: getCompiledPatterns(
+      config.patterns?.relayPrefix ?? '->relay:',
+      config.patterns?.thinkingPrefix ?? '->thinking:'
+    ),
+    isInstructionalText: isInstructionalTextFast,
+    isPlaceholderTarget: isPlaceholderTargetFast,
+    stripAnsi: stripAnsiFast,
+    static: StaticPatterns,
+  };
+
+  // Initialize signing
+  const signingConfig = typeof config.signing === 'string'
+    ? loadSigningConfig(config.signing)
+    : { ...loadSigningConfig(), ...config.signing };
+  const signing = new AgentSigningManager(signingConfig);
+
+  // Initialize DLQ
+  let dlq: DLQStorageAdapter;
+  const dlqConfig = config.dlq ?? {};
+
+  if (dlqConfig.type === 'postgres' && dlqConfig.postgres) {
+    dlq = new PostgresDLQAdapter(dlqConfig.postgres);
+  } else if (dlqConfig.type === 'sqlite' && dlqConfig.sqlite) {
+    dlq = new SQLiteDLQAdapter(dlqConfig.sqlite);
+  } else if (dlqConfig.type === 'memory' || (!dlqConfig.sqlite && !dlqConfig.postgres)) {
+    dlq = new InMemoryDLQAdapter();
+  } else {
+    dlq = new InMemoryDLQAdapter();
+  }
+  await dlq.init();
+
+  // Initialize context compaction
+  const compaction = createContextCompactor(config.compaction);
+
+  // Initialize consensus
+  const consensus = createConsensusEngine(config.consensus);
+
+  // Cleanup function
+  const cleanup = async () => {
+    await dlq.close();
+    consensus.cleanup();
+  };
+
+  return {
+    patterns,
+    signing,
+    dlq,
+    compaction,
+    consensus,
+    cleanup,
+  };
+}
+
+// =============================================================================
+// Router Integration Helpers
+// =============================================================================
+
+/**
+ * Handle failed message delivery by adding to DLQ.
+ */
+export async function handleDeliveryFailure(
+  dlq: DLQStorageAdapter,
+  envelope: {
+    id: string;
+    from: string;
+    to: string;
+    topic?: string;
+    payload: {
+      kind: string;
+      body: string;
+      data?: Record<string, unknown>;
+      thread?: string;
+    };
+    ts: number;
+  },
+  reason: 'max_retries_exceeded' | 'ttl_expired' | 'connection_lost' | 'target_not_found',
+  attemptCount: number,
+  errorMessage?: string
+): Promise<DeadLetter> {
+  return dlq.add(
+    envelope.id,
+    {
+      from: envelope.from,
+      to: envelope.to,
+      topic: envelope.topic,
+      kind: envelope.payload.kind,
+      body: envelope.payload.body,
+      data: envelope.payload.data,
+      thread: envelope.payload.thread,
+      ts: envelope.ts,
+    },
+    reason,
+    attemptCount,
+    errorMessage
+  );
+}
+
+/**
+ * Sign an outgoing envelope if signing is enabled.
+ */
+export function signEnvelope<T extends Record<string, unknown>>(
+  signing: AgentSigningManager,
+  envelope: T,
+  agentName: string
+): T {
+  if (!signing.enabled) {
+    return envelope;
+  }
+
+  const content = JSON.stringify(envelope);
+  const signed = signing.sign(agentName, content);
+
+  if (!signed) {
+    return envelope;
+  }
+
+  return attachSignature(envelope, signed) as T;
+}
+
+/**
+ * Verify an incoming envelope signature.
+ */
+export function verifyEnvelope(
+  signing: AgentSigningManager,
+  envelope: Record<string, unknown>
+): { valid: boolean; error?: string } {
+  if (!signing.enabled) {
+    return { valid: true };
+  }
+
+  const signed = extractSignature(envelope);
+  if (!signed) {
+    const from = typeof envelope.from === 'string' ? envelope.from : 'unknown';
+    if (signing.requiresVerification(from)) {
+      return { valid: false, error: 'Missing signature' };
+    }
+    return { valid: true };
+  }
+
+  return signing.verify(signed);
+}
+
+// =============================================================================
+// Consensus Integration Helpers
+// =============================================================================
+
+/**
+ * Process a potential vote command from a relay message.
+ */
+export function processVoteMessage(
+  consensus: ConsensusEngine,
+  from: string,
+  body: string
+): { processed: boolean; result?: ReturnType<ConsensusEngine['vote']> } {
+  const vote = parseVoteCommand(body);
+  if (!vote) {
+    return { processed: false };
+  }
+
+  const result = consensus.vote(vote.proposalId, from, vote.value, vote.reason);
+  return { processed: true, result };
+}
+
+/**
+ * Create a proposal and format it for broadcast.
+ */
+export function createAndBroadcastProposal(
+  consensus: ConsensusEngine,
+  options: Parameters<ConsensusEngine['createProposal']>[0]
+): { proposal: Proposal; message: string } {
+  const proposal = consensus.createProposal(options);
+  const message = formatProposalMessage(proposal);
+  return { proposal, message };
+}
+
+// =============================================================================
+// Context Management Helpers
+// =============================================================================
+
+/**
+ * Check if context needs compaction and compact if necessary.
+ */
+export function manageContext(
+  compactor: ContextCompactor,
+  messages: Message[]
+): {
+  messages: Message[];
+  compacted: boolean;
+  result?: CompactionResult;
+  budget: ReturnType<ContextCompactor['getTokenBudget']>;
+} {
+  const budget = compactor.getTokenBudget(messages);
+
+  if (!compactor.needsCompaction(messages)) {
+    return { messages, compacted: false, budget };
+  }
+
+  const result = compactor.compact(messages);
+  return {
+    messages: result.messages,
+    compacted: true,
+    result,
+    budget: compactor.getTokenBudget(result.messages),
+  };
+}
+
+// =============================================================================
+// Re-exports for convenience
+// =============================================================================
+
+export {
+  // Signing
+  AgentSigningManager,
+  loadSigningConfig,
+  attachSignature,
+  extractSignature,
+  type AgentSigningConfig,
+  type SignedMessage,
+
+  // DLQ
+  SQLiteDLQAdapter,
+  PostgresDLQAdapter,
+  InMemoryDLQAdapter,
+  createDLQAdapter,
+  DEFAULT_DLQ_CONFIG,
+  type DLQStorageAdapter,
+  type DLQConfig,
+  type DeadLetter,
+  type DLQStats,
+
+  // Compaction
+  ContextCompactor,
+  createContextCompactor,
+  estimateTokens,
+  estimateContextTokens,
+  formatTokenCount,
+  type Message,
+  type CompactionConfig,
+  type CompactionResult,
+
+  // Consensus
+  ConsensusEngine,
+  createConsensusEngine,
+  formatProposalMessage,
+  parseVoteCommand,
+  formatResultMessage,
+  type Proposal,
+  type ConsensusResult,
+  type ConsensusConfig,
+  type VoteValue,
+};

package/packages/daemon/src/index.ts

@@ -0,0 +1,52 @@
+/**
+ * @relay/daemon
+ *
+ * Relay daemon server - agent coordination and message routing.
+ *
+ * This package provides the core daemon infrastructure for:
+ * - Per-project agent communication via RelayServer
+ * - Message routing between agents
+ * - Connection management and heartbeat
+ * - Agent registry for discovery
+ * - Multi-workspace orchestration (dashboard-first mode)
+ * - Enhanced features (consensus, signing, reliability)
+ */
+
+// Core daemon infrastructure (per-project)
+export * from './server.js';
+export * from './router.js';
+export * from './connection.js';
+export * from './agent-registry.js';
+export * from './registry.js';
+
+// Multi-workspace orchestrator (dashboard-first)
+export * from './types.js';
+export * from './orchestrator.js';
+export * from './workspace-manager.js';
+export * from './agent-manager.js';
+
+// Enhanced features (performance, reliability, coordination)
+export * from './enhanced-features.js';
+export * from './agent-signing.js';
+export * from './consensus.js';
+export * from './consensus-integration.js';
+
+// CLI authentication (OAuth flows for workspace CLIs)
+export {
+  startCLIAuth,
+  getAuthSession,
+  cancelAuthSession,
+  submitAuthCode,
+  completeAuthSession,
+  getSupportedProviders,
+  type CLIAuthConfig,
+  type PromptHandler,
+  type StartCLIAuthOptions,
+} from './cli-auth.js';
+
+// Relay file watchdog (file-based message detection and processing)
+export * from './relay-ledger.js';
+export * from './relay-watchdog.js';
+
+// Spawn manager (protocol-based agent spawning)
+export { SpawnManager, type SpawnManagerConfig } from './spawn-manager.js';

package/packages/daemon/src/migrations/0001_initial.sql

@@ -0,0 +1,72 @@
+-- Relay Ledger - Initial Schema
+-- Migration 0001: Initial tables for relay file tracking
+--
+-- Tables:
+-- - relay_files: Tracks relay message files through lifecycle
+-- - agents: Registry of known agents
+-- - orchestrator_state: Key-value store for orchestrator persistence
+-- - pending_operations: Crash recovery for incomplete operations
+
+-- Main relay files table
+CREATE TABLE IF NOT EXISTS relay_files (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  file_id TEXT NOT NULL UNIQUE,
+  source_path TEXT NOT NULL,
+  archive_path TEXT,
+  agent_name TEXT NOT NULL,
+  message_type TEXT NOT NULL,
+  status TEXT NOT NULL DEFAULT 'pending',
+  retries INTEGER NOT NULL DEFAULT 0,
+  max_retries INTEGER NOT NULL DEFAULT 3,
+  discovered_at INTEGER NOT NULL,
+  processed_at INTEGER,
+  archived_at INTEGER,
+  error TEXT,
+  content_hash TEXT,
+  file_size INTEGER NOT NULL DEFAULT 0,
+  file_mtime_ns INTEGER,
+  file_inode INTEGER,
+  CONSTRAINT valid_status CHECK (status IN ('pending', 'processing', 'delivered', 'failed', 'archived'))
+);
+
+CREATE INDEX IF NOT EXISTS idx_relay_files_status ON relay_files(status);
+CREATE INDEX IF NOT EXISTS idx_relay_files_agent ON relay_files(agent_name);
+CREATE INDEX IF NOT EXISTS idx_relay_files_discovered ON relay_files(discovered_at);
+CREATE INDEX IF NOT EXISTS idx_relay_files_source ON relay_files(source_path);
+
+-- Agents registry table
+CREATE TABLE IF NOT EXISTS agents (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  agent_name TEXT NOT NULL UNIQUE,
+  created_at INTEGER NOT NULL,
+  last_seen_at INTEGER NOT NULL,
+  status TEXT NOT NULL DEFAULT 'active',
+  metadata TEXT,
+  CONSTRAINT valid_agent_status CHECK (status IN ('active', 'inactive'))
+);
+
+CREATE INDEX IF NOT EXISTS idx_agents_status ON agents(status);
+CREATE INDEX IF NOT EXISTS idx_agents_last_seen ON agents(last_seen_at);
+
+-- Orchestrator state table (key-value store for crash recovery)
+CREATE TABLE IF NOT EXISTS orchestrator_state (
+  key TEXT PRIMARY KEY,
+  value TEXT NOT NULL,
+  updated_at INTEGER NOT NULL
+);
+
+-- Pending operations table (crash recovery atomicity)
+CREATE TABLE IF NOT EXISTS pending_operations (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  operation_type TEXT NOT NULL,
+  target_id TEXT NOT NULL,
+  payload TEXT,
+  created_at INTEGER NOT NULL,
+  attempts INTEGER NOT NULL DEFAULT 0,
+  last_attempt_at INTEGER,
+  error TEXT,
+  CONSTRAINT valid_operation_type CHECK (operation_type IN ('process', 'archive', 'cleanup'))
+);
+
+CREATE INDEX IF NOT EXISTS idx_pending_ops_type ON pending_operations(operation_type);
+CREATE INDEX IF NOT EXISTS idx_pending_ops_target ON pending_operations(target_id);