@cleocode/core 2026.3.43 → 2026.3.45

package/src/agents/__tests__/retry.test.ts

@@ -0,0 +1,289 @@
+/**
+ * Tests for retry logic, exponential backoff, and self-healing recovery.
+ *
+ * @module agents/__tests__/retry.test
+ */
+
+import { mkdir, mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { registerAgent, updateAgentStatus } from '../registry.js';
+import {
+  calculateDelay,
+  createRetryPolicy,
+  DEFAULT_RETRY_POLICY,
+  recoverCrashedAgents,
+  shouldRetry,
+  withRetry,
+} from '../retry.js';
+
+// ==========================================================================
+// Retry Policy
+// ==========================================================================
+
+describe('Retry Policy', () => {
+  describe('createRetryPolicy', () => {
+    it('returns default policy when no overrides', () => {
+      const policy = createRetryPolicy();
+      expect(policy.maxRetries).toBe(3);
+      expect(policy.baseDelayMs).toBe(1_000);
+      expect(policy.maxDelayMs).toBe(30_000);
+      expect(policy.backoffMultiplier).toBe(2);
+      expect(policy.jitter).toBe(true);
+      expect(policy.retryOnUnknown).toBe(true);
+    });
+
+    it('merges partial overrides with defaults', () => {
+      const policy = createRetryPolicy({ maxRetries: 5, baseDelayMs: 500 });
+      expect(policy.maxRetries).toBe(5);
+      expect(policy.baseDelayMs).toBe(500);
+      expect(policy.maxDelayMs).toBe(30_000); // unchanged default
+    });
+
+    it('DEFAULT_RETRY_POLICY is frozen', () => {
+      expect(Object.isFrozen(DEFAULT_RETRY_POLICY)).toBe(true);
+    });
+  });
+
+  describe('calculateDelay', () => {
+    it('applies exponential backoff', () => {
+      const policy = createRetryPolicy({ jitter: false, baseDelayMs: 100, backoffMultiplier: 2 });
+
+      expect(calculateDelay(0, policy)).toBe(100); // 100 * 2^0
+      expect(calculateDelay(1, policy)).toBe(200); // 100 * 2^1
+      expect(calculateDelay(2, policy)).toBe(400); // 100 * 2^2
+      expect(calculateDelay(3, policy)).toBe(800); // 100 * 2^3
+    });
+
+    it('caps at maxDelay', () => {
+      const policy = createRetryPolicy({
+        jitter: false,
+        baseDelayMs: 100,
+        maxDelayMs: 500,
+        backoffMultiplier: 10,
+      });
+
+      expect(calculateDelay(0, policy)).toBe(100);
+      expect(calculateDelay(1, policy)).toBe(500); // capped
+      expect(calculateDelay(2, policy)).toBe(500); // capped
+    });
+
+    it('adds jitter when enabled', () => {
+      const policy = createRetryPolicy({
+        jitter: true,
+        baseDelayMs: 1000,
+        backoffMultiplier: 1,
+      });
+
+      // With jitter, delay should be >= base but <= base * 1.25
+      const delays = Array.from({ length: 20 }, () => calculateDelay(0, policy));
+      const allInRange = delays.every((d) => d >= 1000 && d <= 1250);
+      expect(allInRange).toBe(true);
+
+      // With 20 samples, at least some should differ (jitter is random)
+      const uniqueDelays = new Set(delays);
+      expect(uniqueDelays.size).toBeGreaterThan(1);
+    });
+  });
+
+  describe('shouldRetry', () => {
+    it('allows retry for retriable errors within limit', () => {
+      const policy = createRetryPolicy({ maxRetries: 3 });
+      expect(shouldRetry(new Error('ECONNREFUSED'), 0, policy)).toBe(true);
+      expect(shouldRetry(new Error('timeout'), 1, policy)).toBe(true);
+      expect(shouldRetry(new Error('503 Service Unavailable'), 2, policy)).toBe(true);
+    });
+
+    it('denies retry when attempt exceeds maxRetries', () => {
+      const policy = createRetryPolicy({ maxRetries: 3 });
+      expect(shouldRetry(new Error('ECONNREFUSED'), 3, policy)).toBe(false);
+      expect(shouldRetry(new Error('ECONNREFUSED'), 4, policy)).toBe(false);
+    });
+
+    it('denies retry for permanent errors', () => {
+      const policy = createRetryPolicy({ maxRetries: 10 });
+      expect(shouldRetry(new Error('Permission denied'), 0, policy)).toBe(false);
+      expect(shouldRetry(new Error('401 Unauthorized'), 0, policy)).toBe(false);
+    });
+
+    it('respects retryOnUnknown policy', () => {
+      const retryUnknown = createRetryPolicy({ retryOnUnknown: true });
+      const noRetryUnknown = createRetryPolicy({ retryOnUnknown: false });
+
+      const unknownError = new Error('Something weird');
+      expect(shouldRetry(unknownError, 0, retryUnknown)).toBe(true);
+      expect(shouldRetry(unknownError, 0, noRetryUnknown)).toBe(false);
+    });
+  });
+});
+
+// ==========================================================================
+// withRetry wrapper
+// ==========================================================================
+
+describe('withRetry', () => {
+  it('succeeds on first attempt', async () => {
+    let callCount = 0;
+    const result = await withRetry(async () => {
+      callCount++;
+      return 'success';
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.value).toBe('success');
+    expect(result.attempts).toBe(1);
+    expect(callCount).toBe(1);
+  });
+
+  it('retries on retriable error and eventually succeeds', async () => {
+    let callCount = 0;
+    const result = await withRetry(
+      async () => {
+        callCount++;
+        if (callCount < 3) throw new Error('ECONNREFUSED');
+        return 'recovered';
+      },
+      { baseDelayMs: 1, maxDelayMs: 5, jitter: false },
+    );
+
+    expect(result.success).toBe(true);
+    expect(result.value).toBe('recovered');
+    expect(result.attempts).toBe(3);
+    expect(callCount).toBe(3);
+  });
+
+  it('fails immediately on permanent error', async () => {
+    let callCount = 0;
+    const result = await withRetry(
+      async () => {
+        callCount++;
+        throw new Error('Permission denied');
+      },
+      { baseDelayMs: 1, jitter: false },
+    );
+
+    expect(result.success).toBe(false);
+    expect(result.error?.message).toBe('Permission denied');
+    expect(result.attempts).toBe(1);
+    expect(callCount).toBe(1);
+  });
+
+  it('exhausts retries and fails', async () => {
+    let callCount = 0;
+    const result = await withRetry(
+      async () => {
+        callCount++;
+        throw new Error('ECONNREFUSED');
+      },
+      { maxRetries: 2, baseDelayMs: 1, maxDelayMs: 5, jitter: false },
+    );
+
+    expect(result.success).toBe(false);
+    expect(result.attempts).toBe(3); // 1 initial + 2 retries
+    expect(callCount).toBe(3);
+  });
+
+  it('tracks total delay time', async () => {
+    let callCount = 0;
+    const result = await withRetry(
+      async () => {
+        callCount++;
+        if (callCount <= 2) throw new Error('timeout');
+        return 'ok';
+      },
+      { baseDelayMs: 10, backoffMultiplier: 1, jitter: false },
+    );
+
+    expect(result.success).toBe(true);
+    expect(result.totalDelayMs).toBeGreaterThanOrEqual(20); // 10 + 10
+  });
+
+  it('uses custom retry policy', async () => {
+    let callCount = 0;
+    const result = await withRetry(
+      async () => {
+        callCount++;
+        throw new Error('rate limit');
+      },
+      { maxRetries: 1, baseDelayMs: 1, jitter: false },
+    );
+
+    expect(result.success).toBe(false);
+    expect(result.attempts).toBe(2); // 1 initial + 1 retry
+    expect(callCount).toBe(2);
+  });
+});
+
+// ==========================================================================
+// Self-Healing Recovery
+// ==========================================================================
+
+describe('recoverCrashedAgents', () => {
+  let tempDir: string;
+
+  beforeEach(async () => {
+    tempDir = await mkdtemp(join(tmpdir(), 'cleo-recover-test-'));
+    await mkdir(join(tempDir, '.cleo'), { recursive: true });
+    await mkdir(join(tempDir, '.cleo', 'backups', 'operational'), { recursive: true });
+  });
+
+  afterEach(async () => {
+    try {
+      const { closeAllDatabases } = await import('../../store/sqlite.js');
+      await closeAllDatabases();
+    } catch {
+      /* module may not be loaded */
+    }
+    await Promise.race([
+      rm(tempDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 300 }).catch(() => {}),
+      new Promise<void>((resolve) => setTimeout(resolve, 8_000)),
+    ]);
+  });
+
+  it('recovers crashed agents with retriable errors', async () => {
+    const agent = await registerAgent({ agentType: 'executor' }, tempDir);
+    await updateAgentStatus(agent.id, { status: 'crashed', error: 'ECONNREFUSED' }, tempDir);
+
+    const results = await recoverCrashedAgents(30_000, tempDir);
+
+    expect(results.length).toBe(1);
+    expect(results[0]!.recovered).toBe(true);
+    expect(results[0]!.action).toBe('restarted');
+  });
+
+  it('abandons agents with permanent errors', async () => {
+    const agent = await registerAgent({ agentType: 'executor' }, tempDir);
+    await updateAgentStatus(agent.id, { status: 'crashed', error: 'Permission denied' }, tempDir);
+
+    const results = await recoverCrashedAgents(30_000, tempDir);
+
+    expect(results.length).toBe(1);
+    expect(results[0]!.recovered).toBe(false);
+    expect(results[0]!.action).toBe('abandoned');
+  });
+
+  it('abandons agents exceeding error threshold', async () => {
+    const agent = await registerAgent({ agentType: 'executor' }, tempDir);
+
+    // Simulate 5+ errors
+    for (let i = 0; i < 5; i++) {
+      await updateAgentStatus(agent.id, { status: 'error', error: `Error ${i}` }, tempDir);
+    }
+    await updateAgentStatus(agent.id, { status: 'crashed' }, tempDir);
+
+    const results = await recoverCrashedAgents(30_000, tempDir);
+
+    expect(results.length).toBe(1);
+    expect(results[0]!.recovered).toBe(false);
+    expect(results[0]!.action).toBe('abandoned');
+    expect(results[0]!.reason).toContain('exceeds threshold');
+  });
+
+  it('returns empty results when no crashed agents', async () => {
+    await registerAgent({ agentType: 'executor' }, tempDir);
+
+    const results = await recoverCrashedAgents(60_000, tempDir);
+    expect(results.length).toBe(0);
+  });
+});

package/src/agents/agent-schema.ts

@@ -0,0 +1,107 @@
+/**
+ * Drizzle ORM schema for the CLEO Agent dimension.
+ *
+ * Defines the `agent_instances` table that tracks live agent processes,
+ * their health (heartbeat protocol), capacity, and error history.
+ *
+ * This is the DB-backed runtime registry -- distinct from the file-based
+ * skill agent registry in `skills/agents/registry.ts` which tracks
+ * installed agent *definitions*. This table tracks running agent *instances*.
+ *
+ * @module agents/agent-schema
+ */
+
+import { sql } from 'drizzle-orm';
+import { index, integer, sqliteTable, text } from 'drizzle-orm/sqlite-core';
+
+// ============================================================================
+// Canonical enum constants
+// ============================================================================
+
+/** Agent instance status values matching DB CHECK constraint. */
+export const AGENT_INSTANCE_STATUSES = [
+  'starting',
+  'active',
+  'idle',
+  'error',
+  'crashed',
+  'stopped',
+] as const;
+
+/** Agent type values for classification. */
+export const AGENT_TYPES = [
+  'orchestrator',
+  'executor',
+  'researcher',
+  'architect',
+  'validator',
+  'documentor',
+  'custom',
+] as const;
+
+// ============================================================================
+// Agent Instances Table
+// ============================================================================
+
+export const agentInstances = sqliteTable(
+  'agent_instances',
+  {
+    id: text('id').primaryKey(),
+    agentType: text('agent_type', { enum: AGENT_TYPES }).notNull(),
+    status: text('status', { enum: AGENT_INSTANCE_STATUSES }).notNull().default('starting'),
+    sessionId: text('session_id'),
+    taskId: text('task_id'),
+    startedAt: text('started_at').notNull().default(sql`(datetime('now'))`),
+    lastHeartbeat: text('last_heartbeat').notNull().default(sql`(datetime('now'))`),
+    stoppedAt: text('stopped_at'),
+    errorCount: integer('error_count').notNull().default(0),
+    totalTasksCompleted: integer('total_tasks_completed').notNull().default(0),
+    capacity: text('capacity').notNull().default('1.0'),
+    metadataJson: text('metadata_json').default('{}'),
+    parentAgentId: text('parent_agent_id'),
+  },
+  (table) => [
+    index('idx_agent_instances_status').on(table.status),
+    index('idx_agent_instances_agent_type').on(table.agentType),
+    index('idx_agent_instances_session_id').on(table.sessionId),
+    index('idx_agent_instances_task_id').on(table.taskId),
+    index('idx_agent_instances_parent_agent_id').on(table.parentAgentId),
+    index('idx_agent_instances_last_heartbeat').on(table.lastHeartbeat),
+  ],
+);
+
+// ============================================================================
+// Agent Error Log Table
+// ============================================================================
+
+export const agentErrorLog = sqliteTable(
+  'agent_error_log',
+  {
+    id: integer('id').primaryKey({ autoIncrement: true }),
+    agentId: text('agent_id').notNull(),
+    errorType: text('error_type', {
+      enum: ['retriable', 'permanent', 'unknown'],
+    }).notNull(),
+    message: text('message').notNull(),
+    stack: text('stack'),
+    occurredAt: text('occurred_at').notNull().default(sql`(datetime('now'))`),
+    resolved: integer('resolved', { mode: 'boolean' }).notNull().default(false),
+  },
+  (table) => [
+    index('idx_agent_error_log_agent_id').on(table.agentId),
+    index('idx_agent_error_log_error_type').on(table.errorType),
+    index('idx_agent_error_log_occurred_at').on(table.occurredAt),
+  ],
+);
+
+// ============================================================================
+// Type exports
+// ============================================================================
+
+export type AgentInstanceRow = typeof agentInstances.$inferSelect;
+export type NewAgentInstanceRow = typeof agentInstances.$inferInsert;
+export type AgentErrorLogRow = typeof agentErrorLog.$inferSelect;
+export type NewAgentErrorLogRow = typeof agentErrorLog.$inferInsert;
+export type AgentInstanceStatus = (typeof AGENT_INSTANCE_STATUSES)[number];
+export type AgentType = (typeof AGENT_TYPES)[number];
+export type AgentErrorType = 'retriable' | 'permanent' | 'unknown';

package/src/agents/capacity.ts

@@ -0,0 +1,151 @@
+/**
+ * Capacity tracking and load balancing for the Agent dimension.
+ *
+ * Tracks per-agent capacity (0.0-1.0) and provides queries for
+ * capacity-aware work distribution.
+ *
+ * @module agents/capacity
+ */
+
+import { eq } from 'drizzle-orm';
+import { getDb } from '../store/sqlite.js';
+import { type AgentInstanceRow, type AgentType, agentInstances } from './agent-schema.js';
+import { listAgentInstances } from './registry.js';
+
+// ============================================================================
+// Capacity Updates
+// ============================================================================
+
+/**
+ * Update the capacity value for an agent instance.
+ *
+ * @param id - Agent instance ID
+ * @param capacity - New capacity value (0.0 to 1.0)
+ * @param cwd - Working directory
+ * @returns Updated agent row, or null if not found
+ */
+export async function updateCapacity(
+  id: string,
+  capacity: number,
+  cwd?: string,
+): Promise<AgentInstanceRow | null> {
+  if (capacity < 0 || capacity > 1) {
+    throw new Error(`Capacity must be between 0.0 and 1.0, got ${capacity}`);
+  }
+
+  const db = await getDb(cwd);
+  const existing = await db.select().from(agentInstances).where(eq(agentInstances.id, id)).get();
+
+  if (!existing) return null;
+
+  const capacityStr = capacity.toFixed(4);
+  await db.update(agentInstances).set({ capacity: capacityStr }).where(eq(agentInstances.id, id));
+
+  return { ...existing, capacity: capacityStr };
+}
+
+// ============================================================================
+// Capacity Queries
+// ============================================================================
+
+/**
+ * Get the total available capacity across all active agents.
+ *
+ * Only considers agents in 'active' or 'idle' status.
+ * Returns the sum of all capacity values.
+ */
+export async function getAvailableCapacity(cwd?: string): Promise<number> {
+  const agents = await listAgentInstances({ status: ['active', 'idle'] }, cwd);
+  return agents.reduce((sum, agent) => sum + parseCapacity(agent.capacity), 0);
+}
+
+/**
+ * Find the agent with the most available capacity.
+ *
+ * @param agentType - Optional type filter
+ * @param cwd - Working directory
+ * @returns Agent with highest capacity, or null if no active agents
+ */
+export async function findLeastLoadedAgent(
+  agentType?: AgentType,
+  cwd?: string,
+): Promise<AgentInstanceRow | null> {
+  const filters: import('./registry.js').ListAgentFilters = agentType
+    ? { status: ['active', 'idle'] as ('active' | 'idle')[], agentType }
+    : { status: ['active', 'idle'] as ('active' | 'idle')[] };
+
+  const agents = await listAgentInstances(filters, cwd);
+
+  if (agents.length === 0) return null;
+
+  let best = agents[0]!;
+  let bestCapacity = parseCapacity(best.capacity);
+
+  for (let i = 1; i < agents.length; i++) {
+    const cap = parseCapacity(agents[i]!.capacity);
+    if (cap > bestCapacity) {
+      best = agents[i]!;
+      bestCapacity = cap;
+    }
+  }
+
+  return best;
+}
+
+/**
+ * Check if the system is overloaded (total capacity below threshold).
+ *
+ * @param threshold - Minimum acceptable capacity (default: 0.1)
+ * @param cwd - Working directory
+ * @returns true if total available capacity is below the threshold
+ */
+export async function isOverloaded(threshold: number = 0.1, cwd?: string): Promise<boolean> {
+  const capacity = await getAvailableCapacity(cwd);
+  return capacity < threshold;
+}
+
+/** Capacity summary for reporting. */
+export interface CapacitySummary {
+  totalCapacity: number;
+  activeAgentCount: number;
+  averageCapacity: number;
+  overloaded: boolean;
+  threshold: number;
+}
+
+/**
+ * Get a capacity summary across the entire agent pool.
+ *
+ * @param threshold - Overload threshold (default: 0.1)
+ * @param cwd - Working directory
+ */
+export async function getCapacitySummary(
+  threshold: number = 0.1,
+  cwd?: string,
+): Promise<CapacitySummary> {
+  const agents = await listAgentInstances({ status: ['active', 'idle'] }, cwd);
+  const totalCapacity = agents.reduce((sum, a) => sum + parseCapacity(a.capacity), 0);
+  const activeAgentCount = agents.length;
+
+  return {
+    totalCapacity,
+    activeAgentCount,
+    averageCapacity: activeAgentCount > 0 ? totalCapacity / activeAgentCount : 0,
+    overloaded: totalCapacity < threshold,
+    threshold,
+  };
+}
+
+// ============================================================================
+// Utilities
+// ============================================================================
+
+/**
+ * Parse a capacity string to a number.
+ * The DB stores capacity as TEXT to avoid floating-point representation issues.
+ */
+function parseCapacity(value: string | null | undefined): number {
+  if (!value) return 0;
+  const parsed = parseFloat(value);
+  return Number.isNaN(parsed) ? 0 : Math.max(0, Math.min(1, parsed));
+}

package/src/agents/index.ts

@@ -0,0 +1,68 @@
+/**
+ * Agent dimension -- runtime tracking, health monitoring, self-healing, and capacity.
+ *
+ * This module provides the complete Agent dimension for the BRAIN specification:
+ *
+ * - **Registry**: CRUD for agent instances (register, deregister, heartbeat, queries)
+ * - **Health**: Crash detection via heartbeat protocol, health reports
+ * - **Self-Healing**: Retry with exponential backoff, crashed agent recovery
+ * - **Capacity**: Load awareness, least-loaded agent selection, overload detection
+ *
+ * @module agents
+ */
+
+// Schema & types
+export {
+  AGENT_INSTANCE_STATUSES,
+  AGENT_TYPES,
+  type AgentErrorLogRow,
+  type AgentErrorType,
+  type AgentInstanceRow,
+  type AgentInstanceStatus,
+  type AgentType,
+  agentErrorLog,
+  agentInstances,
+  type NewAgentErrorLogRow,
+  type NewAgentInstanceRow,
+} from './agent-schema.js';
+// Capacity tracking
+export {
+  type CapacitySummary,
+  findLeastLoadedAgent,
+  getAvailableCapacity,
+  getCapacitySummary,
+  isOverloaded,
+  updateCapacity,
+} from './capacity.js';
+// Registry (CRUD, heartbeat, health, errors)
+export {
+  type AgentHealthReport,
+  checkAgentHealth,
+  classifyError,
+  deregisterAgent,
+  generateAgentId,
+  getAgentErrorHistory,
+  getAgentInstance,
+  getHealthReport,
+  heartbeat,
+  incrementTasksCompleted,
+  type ListAgentFilters,
+  listAgentInstances,
+  markCrashed,
+  type RegisterAgentOptions,
+  registerAgent,
+  type UpdateStatusOptions,
+  updateAgentStatus,
+} from './registry.js';
+// Retry & self-healing
+export {
+  type AgentRecoveryResult,
+  calculateDelay,
+  createRetryPolicy,
+  DEFAULT_RETRY_POLICY,
+  type RetryPolicy,
+  type RetryResult,
+  recoverCrashedAgents,
+  shouldRetry,
+  withRetry,
+} from './retry.js';