cognitive-core 0.0.2 → 0.1.2

package/src/runtime/types.ts

@@ -0,0 +1,198 @@
+import type { Task, Trajectory } from '../types/index.js';
+import type { MemoryQueryResultV2 } from '../memory/system.js';
+import type { ComputeRequirements } from './compute-provider.js';
+
+/**
+ * Message in an agent conversation
+ */
+export interface AgentMessage {
+  role: 'user' | 'assistant' | 'system' | 'tool_use' | 'tool_result';
+  content: string;
+  toolName?: string;
+  toolInput?: unknown;
+  toolOutput?: unknown;
+  timestamp: Date;
+}
+
+/**
+ * Tool call made by an agent
+ */
+export interface ToolCall {
+  id: string;
+  name: string;
+  input: unknown;
+  output?: unknown;
+  error?: string;
+  startTime: Date;
+  endTime?: Date;
+}
+
+/**
+ * Agent execution state
+ */
+export type AgentState = 'idle' | 'running' | 'paused' | 'completed' | 'failed';
+
+/**
+ * Agent session - represents a running or completed agent execution
+ */
+export interface AgentSession {
+  id: string;
+  agentType: string;
+  task: Task;
+  state: AgentState;
+  messages: AgentMessage[];
+  toolCalls: ToolCall[];
+  startTime: Date;
+  endTime?: Date;
+  result?: unknown;
+  error?: string;
+  metadata: Record<string, unknown>;
+}
+
+/**
+ * Configuration for spawning an agent
+ */
+export interface AgentSpawnConfig {
+  /** Type of agent to spawn (e.g., 'claude-code', 'custom', etc.) */
+  agentType: string;
+  /** Task for the agent to work on */
+  task: Task;
+  /** Optional knowledge to inject into the agent's context */
+  injectedKnowledge?: MemoryQueryResultV2;
+  /** Custom system prompt additions */
+  systemPromptAdditions?: string;
+  /** Environment variables for the agent */
+  env?: Record<string, string>;
+  /** Working directory */
+  cwd?: string;
+  /** Timeout in milliseconds */
+  timeout?: number;
+  /** Whether to capture detailed tool calls */
+  captureToolCalls?: boolean;
+  /** Callback for real-time observation */
+  onMessage?: (message: AgentMessage) => void;
+  onToolCall?: (toolCall: ToolCall) => void;
+  /** Additional backend-specific options */
+  backendOptions?: Record<string, unknown>;
+  /** Compute requirements for provider-based backend resolution */
+  computeRequirements?: ComputeRequirements;
+}
+
+/**
+ * Result of agent execution
+ */
+export interface AgentResult {
+  session: AgentSession;
+  trajectory: Trajectory;
+  success: boolean;
+  metrics: {
+    totalTime: number;
+    toolCallCount: number;
+    messageCount: number;
+    tokenEstimate?: number;
+  };
+}
+
+/**
+ * Agent backend protocol - implement this for different agent systems
+ */
+export interface AgentBackend {
+  /** Unique name for this backend */
+  readonly name: string;
+
+  /** Supported agent types */
+  readonly supportedTypes: string[];
+
+  /** Check if backend is available/configured */
+  isAvailable(): Promise<boolean>;
+
+  /** Spawn and run an agent */
+  spawn(config: AgentSpawnConfig): Promise<AgentSession>;
+
+  /** Get status of a running session */
+  getSession(sessionId: string): Promise<AgentSession | undefined>;
+
+  /** Send a message to a running session */
+  sendMessage?(sessionId: string, message: string): Promise<void>;
+
+  /** Pause a running session */
+  pause?(sessionId: string): Promise<void>;
+
+  /** Resume a paused session */
+  resume?(sessionId: string): Promise<void>;
+
+  /** Terminate a session */
+  terminate?(sessionId: string): Promise<void>;
+
+  /** List active sessions */
+  listSessions?(): Promise<AgentSession[]>;
+}
+
+/**
+ * Knowledge injector - formats knowledge for injection into agents
+ */
+export interface KnowledgeInjector {
+  /** Format knowledge for a specific agent type */
+  format(
+    knowledge: MemoryQueryResultV2,
+    agentType: string,
+    options?: { maxTokens?: number; format?: 'markdown' | 'json' | 'xml' }
+  ): string;
+}
+
+/**
+ * Trajectory extractor - extracts trajectories from agent sessions
+ */
+export interface TrajectoryExtractor {
+  /** Extract a trajectory from an agent session */
+  extract(session: AgentSession, task: Task): Trajectory;
+}
+
+/**
+ * Agent observer callbacks
+ */
+export interface AgentObserverCallbacks {
+  onSessionStart?: (session: AgentSession) => void;
+  onMessage?: (session: AgentSession, message: AgentMessage) => void;
+  onToolCall?: (session: AgentSession, toolCall: ToolCall) => void;
+  onSessionEnd?: (session: AgentSession, trajectory: Trajectory) => void;
+  onError?: (session: AgentSession, error: Error) => void;
+}
+
+/**
+ * Options passed to an AgentDelegate when executing a task
+ */
+export interface AgentDelegateOptions {
+  /** Working directory for execution */
+  cwd: string;
+  /** System context (formatted knowledge + skill context) */
+  systemContext?: string;
+  /** Timeout in milliseconds */
+  timeoutMs?: number;
+}
+
+/**
+ * Result returned by an AgentDelegate after execution
+ */
+export interface AgentDelegateResult {
+  /** Whether the execution succeeded */
+  success: boolean;
+  /** Text output from the agent */
+  output: string;
+  /** Optional structured output */
+  structured?: unknown;
+  /** Optional trajectory — when provided, used directly for full learning fidelity */
+  trajectory?: Trajectory;
+}
+
+/**
+ * AgentDelegate — lets callers provide their own agent instances.
+ *
+ * When set on AgentManager, bypasses backend resolution and compute provider.
+ * The caller owns all lifecycle, tracking, and billing. cognitive-core only
+ * sends a prompt + context and receives a result.
+ */
+export interface AgentDelegate {
+  /** Execute a task with the given prompt and options */
+  execute(prompt: string, options: AgentDelegateOptions): Promise<AgentDelegateResult>;
+}

package/src/search/evaluator.ts

@@ -0,0 +1,499 @@
+/**
+ * Solution Evaluator
+ *
+ * Evaluates solution quality using a hybrid approach:
+ * 1. First tries task verification (if available)
+ * 2. Falls back to ACP agent evaluation
+ * 3. Falls back to heuristic evaluation as last resort
+ */
+
+import type { Trajectory } from '../types/trajectory.js';
+import type { Task } from '../types/task.js';
+import type { AgentManager } from '../runtime/manager.js';
+import type { AgenticTaskRunner } from '../workspace/runner.js';
+import { solutionEvaluationTemplate } from '../workspace/templates/solution-evaluation.js';
+import {
+  type EvaluationResult,
+  createEvaluationResult,
+  scoreToQuality,
+} from './refinement-types.js';
+
+/**
+ * Verification function signature for task-specific verification
+ */
+export interface VerificationFunction {
+  (trajectory: Trajectory, task: Task): Promise<VerificationResult>;
+}
+
+/**
+ * Result from a verification function
+ */
+export interface VerificationResult {
+  /** Whether the solution passed verification */
+  passed: boolean;
+  /** Confidence in the verification result (0-1) */
+  confidence: number;
+  /** Specific issues found */
+  issues?: Array<{
+    type: 'incomplete' | 'incorrect' | 'error';
+    description: string;
+    severity?: 'critical' | 'major' | 'minor';
+  }>;
+  /** Additional details */
+  details?: string;
+}
+
+/**
+ * Configuration for the SolutionEvaluator
+ */
+export interface EvaluatorConfig {
+  /** Agent type to use for evaluation (default: 'evaluator') */
+  evaluatorAgentType?: string;
+  /** Minimum confidence for verification to be accepted */
+  verificationConfidenceThreshold?: number;
+  /** Whether to always run agent evaluation even if verification passes */
+  alwaysUseAgent?: boolean;
+  /** Timeout for agent evaluation in ms */
+  agentTimeout?: number;
+}
+
+/**
+ * Default evaluator configuration
+ */
+const DEFAULT_CONFIG: Required<EvaluatorConfig> = {
+  evaluatorAgentType: 'evaluator',
+  verificationConfidenceThreshold: 0.8,
+  alwaysUseAgent: false,
+  agentTimeout: 60000,
+};
+
+/**
+ * SolutionEvaluator - Evaluates trajectory quality using hybrid approach
+ */
+export class SolutionEvaluator {
+  private agentManager: AgentManager | null;
+  private taskRunner: AgenticTaskRunner | null = null;
+  private config: Required<EvaluatorConfig>;
+  private verifiers: Map<string, VerificationFunction> = new Map();
+
+  constructor(
+    agentManager: AgentManager | null,
+    config: EvaluatorConfig = {}
+  ) {
+    this.agentManager = agentManager;
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+
+  /**
+   * Set the task runner for workspace-based evaluation.
+   * When set, evaluation delegates to solutionEvaluationTemplate.
+   */
+  setTaskRunner(runner: AgenticTaskRunner): void {
+    this.taskRunner = runner;
+  }
+
+  /**
+   * Register a verification function for a task domain
+   */
+  registerVerifier(domain: string, verifier: VerificationFunction): void {
+    this.verifiers.set(domain, verifier);
+  }
+
+  /**
+   * Evaluate a trajectory's quality
+   */
+  async evaluate(
+    trajectory: Trajectory,
+    task: Task
+  ): Promise<EvaluationResult> {
+    // 1. Try verification first (if available and applicable)
+    const verificationResult = await this.tryVerification(trajectory, task);
+    if (verificationResult) {
+      const { result, evalResult } = verificationResult;
+
+      // If verification is confident enough and doesn't always require agent
+      if (
+        result.confidence >= this.config.verificationConfidenceThreshold &&
+        !this.config.alwaysUseAgent
+      ) {
+        return evalResult;
+      }
+    }
+
+    // 2. Try workspace-based evaluation (preferred)
+    if (this.taskRunner) {
+      try {
+        const result = await this.taskRunner.run(solutionEvaluationTemplate, {
+          trajectory,
+          task,
+        });
+        return result.output;
+      } catch (error) {
+        console.warn(
+          'Workspace-based evaluation failed, falling back:',
+          error instanceof Error ? error.message : String(error)
+        );
+      }
+    }
+
+    // 3. Try ad-hoc ACP agent evaluation
+    if (this.agentManager) {
+      try {
+        const agentResult = await this.evaluateWithAgent(trajectory, task);
+        return agentResult;
+      } catch (error) {
+        const errorMsg = error instanceof Error ? error.message : String(error);
+        console.warn(`Agent evaluation failed, falling back to heuristic: ${errorMsg}`);
+      }
+    }
+
+    // 4. Fall back to heuristic evaluation
+    return this.evaluateHeuristic(trajectory, task);
+  }
+
+  /**
+   * Try verification if a verifier is available for the task domain
+   */
+  private async tryVerification(
+    trajectory: Trajectory,
+    task: Task
+  ): Promise<{ result: VerificationResult; evalResult: EvaluationResult } | null> {
+    // Check for domain-specific verifier
+    const verifier = task.domain ? this.verifiers.get(task.domain) : undefined;
+    if (!verifier) {
+      return null;
+    }
+
+    try {
+      const result = await verifier(trajectory, task);
+
+      const score = result.passed
+        ? Math.max(0.7, result.confidence)
+        : Math.min(0.5, 1 - result.confidence);
+
+      const evalResult = createEvaluationResult({
+        quality: scoreToQuality(score),
+        score,
+        acceptable: result.passed && result.confidence >= this.config.verificationConfidenceThreshold,
+        issues: result.issues?.map((issue) => ({
+          type: issue.type,
+          description: issue.description,
+          severity: issue.severity ?? 'major',
+        })) ?? [],
+        method: 'verification',
+        rawResponse: result.details,
+      });
+
+      return { result, evalResult };
+    } catch (error) {
+      // Verification failed, return null to try other methods
+      console.warn(
+        'Verification failed:',
+        error instanceof Error ? error.message : String(error)
+      );
+      return null;
+    }
+  }
+
+  /**
+   * Evaluate using an ACP agent
+   */
+  private async evaluateWithAgent(
+    trajectory: Trajectory,
+    task: Task
+  ): Promise<EvaluationResult> {
+    if (!this.agentManager) {
+      throw new Error('AgentManager not available for agent evaluation');
+    }
+
+    // Create evaluation task for the agent
+    const evaluationTask: Task = {
+      id: `eval-${trajectory.id}`,
+      description: this.buildEvaluationPrompt(trajectory, task),
+      domain: 'evaluation',
+      context: {},
+      createdAt: new Date(),
+      metadata: {
+        originalTaskId: task.id,
+        trajectoryId: trajectory.id,
+      },
+    };
+
+    // Spawn evaluation agent
+    const result = await this.agentManager.spawn({
+      agentType: this.config.evaluatorAgentType,
+      task: evaluationTask,
+      timeout: this.config.agentTimeout,
+      backendOptions: {
+        // Evaluation agents should be concise
+        maxTokens: 2000,
+      },
+    });
+
+    // Parse agent response into EvaluationResult
+    return this.parseAgentEvaluation(result.session.result, result.trajectory);
+  }
+
+  /**
+   * Build evaluation prompt for the agent
+   */
+  private buildEvaluationPrompt(trajectory: Trajectory, task: Task): string {
+    const steps = trajectory.steps
+      .map((step, i) => {
+        let stepStr = `Step ${i + 1}:`;
+        if (step.thought) stepStr += `\n  Thought: ${step.thought}`;
+        stepStr += `\n  Action: ${step.action}`;
+        if (step.observation) stepStr += `\n  Observation: ${step.observation}`;
+        return stepStr;
+      })
+      .join('\n\n');
+
+    const outcomeStr = trajectory.outcome.success
+      ? `SUCCESS: ${trajectory.outcome.solution ?? 'Task completed'}`
+      : `FAILURE: ${trajectory.outcome.errorInfo ?? 'Unknown error'}`;
+
+    return `Evaluate the quality of this solution attempt.
+
+## Original Task
+${task.description}
+
+## Solution Steps
+${steps}
+
+## Outcome
+${outcomeStr}
+
+## Instructions
+Analyze the solution and provide:
+1. Overall quality assessment (excellent/good/needs_work/poor)
+2. A numeric score from 0.0 to 1.0
+3. Whether this solution is acceptable
+4. List any specific issues found
+5. Suggestions for improvement
+
+Respond in JSON format:
+{
+  "quality": "excellent|good|needs_work|poor",
+  "score": 0.85,
+  "acceptable": true,
+  "issues": [
+    {"type": "incomplete|incorrect|inefficient|off_topic|error", "description": "...", "severity": "critical|major|minor"}
+  ],
+  "suggestions": ["..."]
+}`;
+  }
+
+  /**
+   * Parse agent evaluation response into EvaluationResult
+   */
+  private parseAgentEvaluation(
+    result: unknown,
+    _evalTrajectory: Trajectory
+  ): EvaluationResult {
+    // Try to parse as JSON
+    if (typeof result === 'string') {
+      try {
+        // Extract JSON from response (agent may include extra text)
+        const jsonMatch = result.match(/\{[\s\S]*\}/);
+        if (jsonMatch) {
+          const parsed = JSON.parse(jsonMatch[0]) as {
+            quality?: string;
+            score?: number;
+            acceptable?: boolean;
+            issues?: Array<{
+              type?: string;
+              description?: string;
+              severity?: string;
+            }>;
+            suggestions?: string[];
+          };
+
+          return createEvaluationResult({
+            quality: this.parseQuality(parsed.quality),
+            score: Math.min(1, Math.max(0, parsed.score ?? 0.5)),
+            acceptable: parsed.acceptable ?? false,
+            issues: (parsed.issues ?? []).map((issue) => ({
+              type: this.parseIssueType(issue.type),
+              description: issue.description ?? 'Unknown issue',
+              severity: this.parseSeverity(issue.severity),
+              suggestion: undefined,
+            })),
+            method: 'agent',
+            rawResponse: result,
+          });
+        }
+      } catch {
+        // Fall through to heuristic parsing
+      }
+    }
+
+    // If we can't parse, return a conservative result
+    return createEvaluationResult({
+      quality: 'needs_work',
+      score: 0.5,
+      acceptable: false,
+      issues: [
+        {
+          type: 'error',
+          description: 'Could not parse agent evaluation response',
+          severity: 'major',
+        },
+      ],
+      method: 'agent',
+      rawResponse: typeof result === 'string' ? result : JSON.stringify(result),
+    });
+  }
+
+  /**
+   * Heuristic evaluation based on trajectory properties
+   */
+  private evaluateHeuristic(
+    trajectory: Trajectory,
+    _task: Task
+  ): EvaluationResult {
+    const issues: Array<{
+      type: 'incomplete' | 'incorrect' | 'inefficient' | 'off_topic' | 'error';
+      description: string;
+      severity: 'critical' | 'major' | 'minor';
+    }> = [];
+
+    let score = 0.5; // Start neutral
+
+    // Check outcome
+    if (trajectory.outcome.success) {
+      score += 0.3;
+    } else {
+      score -= 0.2;
+      issues.push({
+        type: 'error',
+        description: trajectory.outcome.errorInfo ?? 'Task did not complete successfully',
+        severity: 'critical',
+      });
+    }
+
+    // Check for steps taken
+    if (trajectory.steps.length === 0) {
+      score -= 0.2;
+      issues.push({
+        type: 'incomplete',
+        description: 'No steps were taken to solve the task',
+        severity: 'critical',
+      });
+    } else {
+      // Penalize very long trajectories (may indicate inefficiency)
+      if (trajectory.steps.length > 20) {
+        score -= 0.1;
+        issues.push({
+          type: 'inefficient',
+          description: `Solution took ${trajectory.steps.length} steps, which may indicate inefficiency`,
+          severity: 'minor',
+        });
+      }
+    }
+
+    // Check for errors in steps
+    const errorSteps = trajectory.steps.filter(
+      (step) => step.observation?.toLowerCase().includes('error')
+    );
+    if (errorSteps.length > 0) {
+      score -= 0.1 * Math.min(3, errorSteps.length);
+      issues.push({
+        type: 'error',
+        description: `${errorSteps.length} step(s) encountered errors`,
+        severity: errorSteps.length > 2 ? 'major' : 'minor',
+      });
+    }
+
+    // Check key steps (high attribution)
+    const keySteps = trajectory.steps.filter(
+      (step) => (step.attributionScore ?? 0) >= 0.15
+    );
+    if (keySteps.length > 0 && trajectory.outcome.success) {
+      score += 0.1;
+    }
+
+    // Normalize score to [0, 1]
+    score = Math.max(0, Math.min(1, score));
+
+    return createEvaluationResult({
+      quality: scoreToQuality(score),
+      score,
+      acceptable: trajectory.outcome.success && score >= 0.6,
+      issues,
+      method: 'heuristic',
+    });
+  }
+
+  /**
+   * Parse quality string to QualityLevel
+   */
+  private parseQuality(
+    quality?: string
+  ): 'excellent' | 'good' | 'needs_work' | 'poor' {
+    const normalized = quality?.toLowerCase();
+    switch (normalized) {
+      case 'excellent':
+        return 'excellent';
+      case 'good':
+        return 'good';
+      case 'needs_work':
+      case 'needs work':
+        return 'needs_work';
+      case 'poor':
+        return 'poor';
+      default:
+        return 'needs_work';
+    }
+  }
+
+  /**
+   * Parse issue type string
+   */
+  private parseIssueType(
+    type?: string
+  ): 'incomplete' | 'incorrect' | 'inefficient' | 'off_topic' | 'error' {
+    const normalized = type?.toLowerCase();
+    switch (normalized) {
+      case 'incomplete':
+        return 'incomplete';
+      case 'incorrect':
+        return 'incorrect';
+      case 'inefficient':
+        return 'inefficient';
+      case 'off_topic':
+      case 'off-topic':
+        return 'off_topic';
+      case 'error':
+        return 'error';
+      default:
+        return 'error';
+    }
+  }
+
+  /**
+   * Parse severity string
+   */
+  private parseSeverity(severity?: string): 'critical' | 'major' | 'minor' {
+    const normalized = severity?.toLowerCase();
+    switch (normalized) {
+      case 'critical':
+        return 'critical';
+      case 'major':
+        return 'major';
+      case 'minor':
+        return 'minor';
+      default:
+        return 'major';
+    }
+  }
+}
+
+/**
+ * Create a solution evaluator
+ */
+export function createSolutionEvaluator(
+  agentManager: AgentManager | null,
+  config?: EvaluatorConfig
+): SolutionEvaluator {
+  return new SolutionEvaluator(agentManager, config);
+}