@cleocode/core 2026.3.42 → 2026.3.44

package/src/agents/registry.ts

@@ -0,0 +1,449 @@
+/**
+ * Agent instance registry -- runtime tracking of live agent processes.
+ *
+ * Provides CRUD operations for the `agent_instances` table:
+ * registration, deregistration, heartbeat, status updates, and queries.
+ *
+ * This is the DB-backed runtime registry for live agent *instances*.
+ * The file-based registry in `skills/agents/registry.ts` tracks installed
+ * agent *definitions* -- those are complementary, not competing, concepts.
+ *
+ * @module agents/registry
+ */
+
+import { randomBytes } from 'node:crypto';
+import { and, eq, inArray, lt, sql } from 'drizzle-orm';
+import { getDb } from '../store/sqlite.js';
+import {
+  type AgentErrorLogRow,
+  type AgentErrorType,
+  type AgentInstanceRow,
+  type AgentInstanceStatus,
+  type AgentType,
+  agentErrorLog,
+  agentInstances,
+} from './agent-schema.js';
+
+// ============================================================================
+// ID generation
+// ============================================================================
+
+/**
+ * Generate a unique agent instance ID.
+ * Format: `agt_{YYYYMMDDHHmmss}_{6hex}`
+ */
+export function generateAgentId(): string {
+  const now = new Date();
+  const ts = now.toISOString().replace(/[-:T]/g, '').substring(0, 14);
+  const hex = randomBytes(3).toString('hex');
+  return `agt_${ts}_${hex}`;
+}
+
+// ============================================================================
+// Registration
+// ============================================================================
+
+/** Options for registering a new agent instance. */
+export interface RegisterAgentOptions {
+  agentType: AgentType;
+  sessionId?: string;
+  taskId?: string;
+  parentAgentId?: string;
+  metadata?: Record<string, unknown>;
+}
+
+/**
+ * Register a new agent instance in the database.
+ * Sets initial status to 'starting' and records the first heartbeat.
+ */
+export async function registerAgent(
+  opts: RegisterAgentOptions,
+  cwd?: string,
+): Promise<AgentInstanceRow> {
+  const db = await getDb(cwd);
+  const id = generateAgentId();
+  const now = new Date().toISOString();
+
+  const row: typeof agentInstances.$inferInsert = {
+    id,
+    agentType: opts.agentType,
+    status: 'starting',
+    sessionId: opts.sessionId ?? null,
+    taskId: opts.taskId ?? null,
+    startedAt: now,
+    lastHeartbeat: now,
+    stoppedAt: null,
+    errorCount: 0,
+    totalTasksCompleted: 0,
+    capacity: '1.0',
+    metadataJson: opts.metadata ? JSON.stringify(opts.metadata) : '{}',
+    parentAgentId: opts.parentAgentId ?? null,
+  };
+
+  await db.insert(agentInstances).values(row);
+  return row as AgentInstanceRow;
+}
+
+/**
+ * Deregister (stop) an agent instance.
+ * Sets status to 'stopped' and records the stop timestamp.
+ */
+export async function deregisterAgent(id: string, cwd?: string): Promise<AgentInstanceRow | null> {
+  const db = await getDb(cwd);
+  const now = new Date().toISOString();
+
+  const existing = await db.select().from(agentInstances).where(eq(agentInstances.id, id)).get();
+  if (!existing) return null;
+
+  // Already stopped -- idempotent
+  if (existing.status === 'stopped') return existing;
+
+  await db
+    .update(agentInstances)
+    .set({ status: 'stopped', stoppedAt: now })
+    .where(eq(agentInstances.id, id));
+
+  return { ...existing, status: 'stopped', stoppedAt: now };
+}
+
+// ============================================================================
+// Heartbeat
+// ============================================================================
+
+/**
+ * Record a heartbeat for an agent instance.
+ * Updates `last_heartbeat` and returns the current status so the agent
+ * can detect if it has been externally marked for shutdown.
+ */
+export async function heartbeat(id: string, cwd?: string): Promise<AgentInstanceStatus | null> {
+  const db = await getDb(cwd);
+  const now = new Date().toISOString();
+
+  const existing = await db.select().from(agentInstances).where(eq(agentInstances.id, id)).get();
+  if (!existing) return null;
+
+  // Do not update heartbeat for terminal states
+  if (existing.status === 'stopped' || existing.status === 'crashed') {
+    return existing.status;
+  }
+
+  await db.update(agentInstances).set({ lastHeartbeat: now }).where(eq(agentInstances.id, id));
+
+  return existing.status;
+}
+
+// ============================================================================
+// Status management
+// ============================================================================
+
+/** Options for updating agent status. */
+export interface UpdateStatusOptions {
+  status: AgentInstanceStatus;
+  error?: string;
+  taskId?: string;
+}
+
+/**
+ * Update agent status with optional error tracking.
+ * When status is 'error' or 'crashed', increments the error count
+ * and logs the error to the agent_error_log table.
+ */
+export async function updateAgentStatus(
+  id: string,
+  opts: UpdateStatusOptions,
+  cwd?: string,
+): Promise<AgentInstanceRow | null> {
+  const db = await getDb(cwd);
+
+  const existing = await db.select().from(agentInstances).where(eq(agentInstances.id, id)).get();
+  if (!existing) return null;
+
+  const updates: Partial<typeof agentInstances.$inferInsert> = {
+    status: opts.status,
+  };
+
+  if (opts.taskId !== undefined) {
+    updates.taskId = opts.taskId;
+  }
+
+  // If transitioning to active, update heartbeat
+  if (opts.status === 'active') {
+    updates.lastHeartbeat = new Date().toISOString();
+  }
+
+  // Track errors
+  if (opts.status === 'error' || opts.status === 'crashed') {
+    updates.errorCount = existing.errorCount + 1;
+
+    if (opts.error) {
+      const errorType = classifyError(new Error(opts.error));
+      await db.insert(agentErrorLog).values({
+        agentId: id,
+        errorType,
+        message: opts.error,
+        occurredAt: new Date().toISOString(),
+      });
+    }
+  }
+
+  // If stopped, set stoppedAt
+  if (opts.status === 'stopped') {
+    updates.stoppedAt = new Date().toISOString();
+  }
+
+  await db.update(agentInstances).set(updates).where(eq(agentInstances.id, id));
+
+  return { ...existing, ...updates } as AgentInstanceRow;
+}
+
+/**
+ * Increment the completed task count for an agent.
+ */
+export async function incrementTasksCompleted(id: string, cwd?: string): Promise<void> {
+  const db = await getDb(cwd);
+  await db
+    .update(agentInstances)
+    .set({ totalTasksCompleted: sql`${agentInstances.totalTasksCompleted} + 1` })
+    .where(eq(agentInstances.id, id));
+}
+
+// ============================================================================
+// Queries
+// ============================================================================
+
+/** Filters for listing agent instances. */
+export interface ListAgentFilters {
+  status?: AgentInstanceStatus | AgentInstanceStatus[];
+  agentType?: AgentType | AgentType[];
+  sessionId?: string;
+  parentAgentId?: string;
+}
+
+/**
+ * List agent instances with optional filters.
+ */
+export async function listAgentInstances(
+  filters?: ListAgentFilters,
+  cwd?: string,
+): Promise<AgentInstanceRow[]> {
+  const db = await getDb(cwd);
+
+  const conditions = [];
+
+  if (filters?.status) {
+    const statuses = Array.isArray(filters.status) ? filters.status : [filters.status];
+    conditions.push(inArray(agentInstances.status, statuses));
+  }
+
+  if (filters?.agentType) {
+    const types = Array.isArray(filters.agentType) ? filters.agentType : [filters.agentType];
+    conditions.push(inArray(agentInstances.agentType, types));
+  }
+
+  if (filters?.sessionId) {
+    conditions.push(eq(agentInstances.sessionId, filters.sessionId));
+  }
+
+  if (filters?.parentAgentId) {
+    conditions.push(eq(agentInstances.parentAgentId, filters.parentAgentId));
+  }
+
+  if (conditions.length === 0) {
+    return db.select().from(agentInstances).all();
+  }
+
+  return db
+    .select()
+    .from(agentInstances)
+    .where(and(...conditions))
+    .all();
+}
+
+/**
+ * Get a single agent instance by ID.
+ */
+export async function getAgentInstance(id: string, cwd?: string): Promise<AgentInstanceRow | null> {
+  const db = await getDb(cwd);
+  const row = await db.select().from(agentInstances).where(eq(agentInstances.id, id)).get();
+  return row ?? null;
+}
+
+// ============================================================================
+// Error classification
+// ============================================================================
+
+/** Patterns indicating retriable errors. */
+const RETRIABLE_PATTERNS = [
+  /timeout/i,
+  /ECONNREFUSED/,
+  /ECONNRESET/,
+  /EPIPE/,
+  /ETIMEDOUT/,
+  /rate.?limit/i,
+  /429/,
+  /503/,
+  /502/,
+  /SQLITE_BUSY/i,
+  /database is locked/i,
+  /temporarily unavailable/i,
+  /too many requests/i,
+  /network/i,
+  /socket hang up/i,
+];
+
+/** Patterns indicating permanent (non-retriable) errors. */
+const PERMANENT_PATTERNS = [
+  /permission denied/i,
+  /EACCES/,
+  /authentication/i,
+  /unauthorized/i,
+  /401/,
+  /403/,
+  /404/,
+  /not found/i,
+  /invalid.*token/i,
+  /SQLITE_CONSTRAINT/i,
+  /syntax error/i,
+  /type error/i,
+  /reference error/i,
+];
+
+/**
+ * Classify an error as retriable, permanent, or unknown.
+ *
+ * Retriable errors are transient conditions (network, rate limits, locks)
+ * where a retry may succeed. Permanent errors are structural (auth,
+ * not found, constraint violations) where retrying is pointless.
+ */
+export function classifyError(error: unknown): AgentErrorType {
+  const message = error instanceof Error ? error.message : String(error);
+
+  for (const pattern of RETRIABLE_PATTERNS) {
+    if (pattern.test(message)) return 'retriable';
+  }
+
+  for (const pattern of PERMANENT_PATTERNS) {
+    if (pattern.test(message)) return 'permanent';
+  }
+
+  return 'unknown';
+}
+
+/**
+ * Get the error history for a specific agent.
+ */
+export async function getAgentErrorHistory(
+  agentId: string,
+  cwd?: string,
+): Promise<AgentErrorLogRow[]> {
+  const db = await getDb(cwd);
+  return db.select().from(agentErrorLog).where(eq(agentErrorLog.agentId, agentId)).all();
+}
+
+// ============================================================================
+// Health Monitoring
+// ============================================================================
+
+/**
+ * Check agent health by finding instances whose last heartbeat exceeds the threshold.
+ * Default threshold: 30000ms (30 seconds) as specified by the BRAIN spec.
+ *
+ * Returns agents that appear to have crashed (stale heartbeat).
+ */
+export async function checkAgentHealth(
+  thresholdMs: number = 30_000,
+  cwd?: string,
+): Promise<AgentInstanceRow[]> {
+  const db = await getDb(cwd);
+  const cutoff = new Date(Date.now() - thresholdMs).toISOString();
+
+  return db
+    .select()
+    .from(agentInstances)
+    .where(
+      and(
+        inArray(agentInstances.status, ['active', 'idle', 'starting']),
+        lt(agentInstances.lastHeartbeat, cutoff),
+      ),
+    )
+    .all();
+}
+
+/**
+ * Mark an agent instance as crashed.
+ * Increments error count and sets status to 'crashed'.
+ */
+export async function markCrashed(
+  id: string,
+  reason?: string,
+  cwd?: string,
+): Promise<AgentInstanceRow | null> {
+  return updateAgentStatus(
+    id,
+    { status: 'crashed', error: reason ?? 'Heartbeat timeout — agent presumed crashed' },
+    cwd,
+  );
+}
+
+/** Agent health report summary. */
+export interface AgentHealthReport {
+  total: number;
+  active: number;
+  idle: number;
+  starting: number;
+  error: number;
+  crashed: number;
+  stopped: number;
+  totalErrors: number;
+  staleAgents: AgentInstanceRow[];
+}
+
+/**
+ * Generate a health report summarizing all agent instances.
+ * Includes counts by status and identifies stale agents.
+ */
+export async function getHealthReport(
+  thresholdMs: number = 30_000,
+  cwd?: string,
+): Promise<AgentHealthReport> {
+  const allAgents = await listAgentInstances(undefined, cwd);
+  const staleAgents = await checkAgentHealth(thresholdMs, cwd);
+
+  const report: AgentHealthReport = {
+    total: allAgents.length,
+    active: 0,
+    idle: 0,
+    starting: 0,
+    error: 0,
+    crashed: 0,
+    stopped: 0,
+    totalErrors: 0,
+    staleAgents,
+  };
+
+  for (const agent of allAgents) {
+    switch (agent.status) {
+      case 'active':
+        report.active++;
+        break;
+      case 'idle':
+        report.idle++;
+        break;
+      case 'starting':
+        report.starting++;
+        break;
+      case 'error':
+        report.error++;
+        break;
+      case 'crashed':
+        report.crashed++;
+        break;
+      case 'stopped':
+        report.stopped++;
+        break;
+    }
+    report.totalErrors += agent.errorCount;
+  }
+
+  return report;
+}

package/src/agents/retry.ts

@@ -0,0 +1,255 @@
+/**
+ * Self-healing and retry logic for the Agent dimension.
+ *
+ * Provides configurable retry policies with exponential backoff + jitter,
+ * and recovery mechanisms for crashed agents. Error classification
+ * determines whether a retry is appropriate.
+ *
+ * @module agents/retry
+ */
+
+import {
+  checkAgentHealth,
+  classifyError,
+  listAgentInstances,
+  updateAgentStatus,
+} from './registry.js';
+
+// ============================================================================
+// Retry Policy
+// ============================================================================
+
+/** Configuration for retry behavior. */
+export interface RetryPolicy {
+  /** Maximum number of retry attempts. Default: 3. */
+  maxRetries: number;
+  /** Base delay in milliseconds before first retry. Default: 1000. */
+  baseDelayMs: number;
+  /** Maximum delay in milliseconds between retries. Default: 30000. */
+  maxDelayMs: number;
+  /** Multiplier for exponential backoff. Default: 2. */
+  backoffMultiplier: number;
+  /** Whether to add random jitter to delays. Default: true. */
+  jitter: boolean;
+  /** Whether to retry on 'unknown' error classification. Default: true. */
+  retryOnUnknown: boolean;
+}
+
+/** Default retry policy matching the BRAIN specification. */
+export const DEFAULT_RETRY_POLICY: Readonly<RetryPolicy> = Object.freeze({
+  maxRetries: 3,
+  baseDelayMs: 1_000,
+  maxDelayMs: 30_000,
+  backoffMultiplier: 2,
+  jitter: true,
+  retryOnUnknown: true,
+});
+
+/**
+ * Create a retry policy by merging overrides with the default policy.
+ */
+export function createRetryPolicy(overrides?: Partial<RetryPolicy>): RetryPolicy {
+  return { ...DEFAULT_RETRY_POLICY, ...overrides };
+}
+
+/**
+ * Calculate the delay for a given retry attempt using exponential backoff.
+ *
+ * Formula: min(baseDelay * multiplier^attempt, maxDelay) + jitter
+ * Jitter adds 0-25% randomness to prevent thundering herd.
+ */
+export function calculateDelay(attempt: number, policy: RetryPolicy): number {
+  const exponentialDelay = policy.baseDelayMs * policy.backoffMultiplier ** attempt;
+  const clampedDelay = Math.min(exponentialDelay, policy.maxDelayMs);
+
+  if (!policy.jitter) return clampedDelay;
+
+  // Add 0-25% jitter
+  const jitterRange = clampedDelay * 0.25;
+  const jitterValue = Math.random() * jitterRange;
+  return Math.floor(clampedDelay + jitterValue);
+}
+
+/**
+ * Determine whether an error should be retried based on its classification
+ * and the retry policy.
+ */
+export function shouldRetry(error: unknown, attempt: number, policy: RetryPolicy): boolean {
+  if (attempt >= policy.maxRetries) return false;
+
+  const classification = classifyError(error);
+
+  if (classification === 'permanent') return false;
+  if (classification === 'retriable') return true;
+
+  // 'unknown' classification -- policy determines behavior
+  return policy.retryOnUnknown;
+}
+
+// ============================================================================
+// Retry wrapper
+// ============================================================================
+
+/** Result of a retried operation. */
+export interface RetryResult<T> {
+  success: boolean;
+  value?: T;
+  error?: Error;
+  attempts: number;
+  totalDelayMs: number;
+}
+
+/**
+ * Wrap an async function with retry logic using configurable exponential backoff.
+ *
+ * The function will be retried according to the policy when retriable errors
+ * occur. Permanent errors cause immediate failure. Unknown errors respect
+ * the `retryOnUnknown` policy setting.
+ *
+ * @param fn - The async function to execute with retries
+ * @param policy - Retry policy (uses DEFAULT_RETRY_POLICY if not provided)
+ * @returns The result of the operation with retry metadata
+ */
+export async function withRetry<T>(
+  fn: () => Promise<T>,
+  policy?: Partial<RetryPolicy>,
+): Promise<RetryResult<T>> {
+  const effectivePolicy = createRetryPolicy(policy);
+  let lastError: Error | undefined;
+  let totalDelayMs = 0;
+
+  for (let attempt = 0; attempt <= effectivePolicy.maxRetries; attempt++) {
+    try {
+      const value = await fn();
+      return {
+        success: true,
+        value,
+        attempts: attempt + 1,
+        totalDelayMs,
+      };
+    } catch (err) {
+      lastError = err instanceof Error ? err : new Error(String(err));
+
+      if (!shouldRetry(err, attempt, effectivePolicy)) {
+        return {
+          success: false,
+          error: lastError,
+          attempts: attempt + 1,
+          totalDelayMs,
+        };
+      }
+
+      // Calculate and apply delay before next attempt
+      if (attempt < effectivePolicy.maxRetries) {
+        const delay = calculateDelay(attempt, effectivePolicy);
+        totalDelayMs += delay;
+        await sleep(delay);
+      }
+    }
+  }
+
+  return {
+    success: false,
+    error: lastError ?? new Error('Retry exhausted without error'),
+    attempts: effectivePolicy.maxRetries + 1,
+    totalDelayMs,
+  };
+}
+
+// ============================================================================
+// Recovery
+// ============================================================================
+
+/** Result of a recovery attempt for a single agent. */
+export interface AgentRecoveryResult {
+  agentId: string;
+  recovered: boolean;
+  action: 'restarted' | 'abandoned' | 'skipped';
+  reason: string;
+}
+
+/**
+ * Attempt to recover crashed agents.
+ *
+ * Finds all agents with status 'crashed' and determines if they can be
+ * restarted based on their error history. Agents whose last error was
+ * classified as 'permanent' are abandoned. Agents with retriable errors
+ * are reset to 'starting' for the orchestration layer to re-assign.
+ *
+ * @param thresholdMs - Heartbeat threshold for crash detection (default: 30000)
+ * @param cwd - Working directory
+ * @returns Recovery results for each crashed agent
+ */
+export async function recoverCrashedAgents(
+  thresholdMs: number = 30_000,
+  cwd?: string,
+): Promise<AgentRecoveryResult[]> {
+  const results: AgentRecoveryResult[] = [];
+
+  // Step 1: detect stale agents and mark them crashed
+  const staleAgents = await checkAgentHealth(thresholdMs, cwd);
+  for (const agent of staleAgents) {
+    await updateAgentStatus(
+      agent.id,
+      { status: 'crashed', error: 'Heartbeat timeout detected during recovery sweep' },
+      cwd,
+    );
+  }
+
+  // Step 2: process all crashed agents
+  const crashedAgents = await listAgentInstances({ status: 'crashed' }, cwd);
+
+  for (const agent of crashedAgents) {
+    // Check error history to determine recovery action
+    const { getAgentErrorHistory } = await import('./registry.js');
+    const errors = await getAgentErrorHistory(agent.id, cwd);
+    const lastError = errors.length > 0 ? errors[errors.length - 1] : null;
+
+    // Too many errors -- abandon
+    if (agent.errorCount >= 5) {
+      await updateAgentStatus(agent.id, { status: 'stopped' }, cwd);
+      results.push({
+        agentId: agent.id,
+        recovered: false,
+        action: 'abandoned',
+        reason: `Error count (${agent.errorCount}) exceeds threshold (5)`,
+      });
+      continue;
+    }
+
+    // Last error was permanent -- abandon
+    if (lastError?.errorType === 'permanent') {
+      await updateAgentStatus(agent.id, { status: 'stopped' }, cwd);
+      results.push({
+        agentId: agent.id,
+        recovered: false,
+        action: 'abandoned',
+        reason: `Permanent error: ${lastError.message}`,
+      });
+      continue;
+    }
+
+    // Retriable or unknown -- attempt restart by resetting to 'starting'
+    await updateAgentStatus(agent.id, { status: 'starting' }, cwd);
+    results.push({
+      agentId: agent.id,
+      recovered: true,
+      action: 'restarted',
+      reason: 'Agent reset to starting for re-assignment',
+    });
+  }
+
+  return results;
+}
+
+// ============================================================================
+// Utilities
+// ============================================================================
+
+/**
+ * Sleep for a given number of milliseconds.
+ * Extracted for testability.
+ */
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}