cognitive-core 0.0.1 → 0.1.0

package/src/search/evaluator.ts

@@ -0,0 +1,474 @@
+/**
+ * Solution Evaluator
+ *
+ * Evaluates solution quality using a hybrid approach:
+ * 1. First tries task verification (if available)
+ * 2. Falls back to ACP agent evaluation
+ * 3. Falls back to heuristic evaluation as last resort
+ */
+
+import type { Trajectory } from '../types/trajectory.js';
+import type { Task } from '../types/task.js';
+import type { AgentManager } from '../runtime/manager.js';
+import {
+  type EvaluationResult,
+  createEvaluationResult,
+  scoreToQuality,
+} from './refinement-types.js';
+
+/**
+ * Verification function signature for task-specific verification
+ */
+export interface VerificationFunction {
+  (trajectory: Trajectory, task: Task): Promise<VerificationResult>;
+}
+
+/**
+ * Result from a verification function
+ */
+export interface VerificationResult {
+  /** Whether the solution passed verification */
+  passed: boolean;
+  /** Confidence in the verification result (0-1) */
+  confidence: number;
+  /** Specific issues found */
+  issues?: Array<{
+    type: 'incomplete' | 'incorrect' | 'error';
+    description: string;
+    severity?: 'critical' | 'major' | 'minor';
+  }>;
+  /** Additional details */
+  details?: string;
+}
+
+/**
+ * Configuration for the SolutionEvaluator
+ */
+export interface EvaluatorConfig {
+  /** Agent type to use for evaluation (default: 'evaluator') */
+  evaluatorAgentType?: string;
+  /** Minimum confidence for verification to be accepted */
+  verificationConfidenceThreshold?: number;
+  /** Whether to always run agent evaluation even if verification passes */
+  alwaysUseAgent?: boolean;
+  /** Timeout for agent evaluation in ms */
+  agentTimeout?: number;
+}
+
+/**
+ * Default evaluator configuration
+ */
+const DEFAULT_CONFIG: Required<EvaluatorConfig> = {
+  evaluatorAgentType: 'evaluator',
+  verificationConfidenceThreshold: 0.8,
+  alwaysUseAgent: false,
+  agentTimeout: 60000,
+};
+
+/**
+ * SolutionEvaluator - Evaluates trajectory quality using hybrid approach
+ */
+export class SolutionEvaluator {
+  private agentManager: AgentManager | null;
+  private config: Required<EvaluatorConfig>;
+  private verifiers: Map<string, VerificationFunction> = new Map();
+
+  constructor(
+    agentManager: AgentManager | null,
+    config: EvaluatorConfig = {}
+  ) {
+    this.agentManager = agentManager;
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+
+  /**
+   * Register a verification function for a task domain
+   */
+  registerVerifier(domain: string, verifier: VerificationFunction): void {
+    this.verifiers.set(domain, verifier);
+  }
+
+  /**
+   * Evaluate a trajectory's quality
+   */
+  async evaluate(
+    trajectory: Trajectory,
+    task: Task
+  ): Promise<EvaluationResult> {
+    // 1. Try verification first (if available and applicable)
+    const verificationResult = await this.tryVerification(trajectory, task);
+    if (verificationResult) {
+      const { result, evalResult } = verificationResult;
+
+      // If verification is confident enough and doesn't always require agent
+      if (
+        result.confidence >= this.config.verificationConfidenceThreshold &&
+        !this.config.alwaysUseAgent
+      ) {
+        return evalResult;
+      }
+    }
+
+    // 2. Try ACP agent evaluation
+    if (this.agentManager) {
+      try {
+        const agentResult = await this.evaluateWithAgent(trajectory, task);
+        return agentResult;
+      } catch (error) {
+        // Fall through to heuristic if agent fails
+        const errorMsg = error instanceof Error ? error.message : String(error);
+        // Log warning but continue to heuristic
+        console.warn(`Agent evaluation failed, falling back to heuristic: ${errorMsg}`);
+      }
+    }
+
+    // 3. Fall back to heuristic evaluation
+    return this.evaluateHeuristic(trajectory, task);
+  }
+
+  /**
+   * Try verification if a verifier is available for the task domain
+   */
+  private async tryVerification(
+    trajectory: Trajectory,
+    task: Task
+  ): Promise<{ result: VerificationResult; evalResult: EvaluationResult } | null> {
+    // Check for domain-specific verifier
+    const verifier = task.domain ? this.verifiers.get(task.domain) : undefined;
+    if (!verifier) {
+      return null;
+    }
+
+    try {
+      const result = await verifier(trajectory, task);
+
+      const score = result.passed
+        ? Math.max(0.7, result.confidence)
+        : Math.min(0.5, 1 - result.confidence);
+
+      const evalResult = createEvaluationResult({
+        quality: scoreToQuality(score),
+        score,
+        acceptable: result.passed && result.confidence >= this.config.verificationConfidenceThreshold,
+        issues: result.issues?.map((issue) => ({
+          type: issue.type,
+          description: issue.description,
+          severity: issue.severity ?? 'major',
+        })) ?? [],
+        method: 'verification',
+        rawResponse: result.details,
+      });
+
+      return { result, evalResult };
+    } catch (error) {
+      // Verification failed, return null to try other methods
+      console.warn(
+        'Verification failed:',
+        error instanceof Error ? error.message : String(error)
+      );
+      return null;
+    }
+  }
+
+  /**
+   * Evaluate using an ACP agent
+   */
+  private async evaluateWithAgent(
+    trajectory: Trajectory,
+    task: Task
+  ): Promise<EvaluationResult> {
+    if (!this.agentManager) {
+      throw new Error('AgentManager not available for agent evaluation');
+    }
+
+    // Create evaluation task for the agent
+    const evaluationTask: Task = {
+      id: `eval-${trajectory.id}`,
+      description: this.buildEvaluationPrompt(trajectory, task),
+      domain: 'evaluation',
+      context: {},
+      createdAt: new Date(),
+      metadata: {
+        originalTaskId: task.id,
+        trajectoryId: trajectory.id,
+      },
+    };
+
+    // Spawn evaluation agent
+    const result = await this.agentManager.spawn({
+      agentType: this.config.evaluatorAgentType,
+      task: evaluationTask,
+      timeout: this.config.agentTimeout,
+      backendOptions: {
+        // Evaluation agents should be concise
+        maxTokens: 2000,
+      },
+    });
+
+    // Parse agent response into EvaluationResult
+    return this.parseAgentEvaluation(result.session.result, result.trajectory);
+  }
+
+  /**
+   * Build evaluation prompt for the agent
+   */
+  private buildEvaluationPrompt(trajectory: Trajectory, task: Task): string {
+    const steps = trajectory.steps
+      .map((step, i) => {
+        let stepStr = `Step ${i + 1}:`;
+        if (step.thought) stepStr += `\n  Thought: ${step.thought}`;
+        stepStr += `\n  Action: ${step.action}`;
+        if (step.observation) stepStr += `\n  Observation: ${step.observation}`;
+        return stepStr;
+      })
+      .join('\n\n');
+
+    const outcomeStr = trajectory.outcome.success
+      ? `SUCCESS: ${trajectory.outcome.solution ?? 'Task completed'}`
+      : `FAILURE: ${trajectory.outcome.errorInfo ?? 'Unknown error'}`;
+
+    return `Evaluate the quality of this solution attempt.
+
+## Original Task
+${task.description}
+
+## Solution Steps
+${steps}
+
+## Outcome
+${outcomeStr}
+
+## Instructions
+Analyze the solution and provide:
+1. Overall quality assessment (excellent/good/needs_work/poor)
+2. A numeric score from 0.0 to 1.0
+3. Whether this solution is acceptable
+4. List any specific issues found
+5. Suggestions for improvement
+
+Respond in JSON format:
+{
+  "quality": "excellent|good|needs_work|poor",
+  "score": 0.85,
+  "acceptable": true,
+  "issues": [
+    {"type": "incomplete|incorrect|inefficient|off_topic|error", "description": "...", "severity": "critical|major|minor"}
+  ],
+  "suggestions": ["..."]
+}`;
+  }
+
+  /**
+   * Parse agent evaluation response into EvaluationResult
+   */
+  private parseAgentEvaluation(
+    result: unknown,
+    _evalTrajectory: Trajectory
+  ): EvaluationResult {
+    // Try to parse as JSON
+    if (typeof result === 'string') {
+      try {
+        // Extract JSON from response (agent may include extra text)
+        const jsonMatch = result.match(/\{[\s\S]*\}/);
+        if (jsonMatch) {
+          const parsed = JSON.parse(jsonMatch[0]) as {
+            quality?: string;
+            score?: number;
+            acceptable?: boolean;
+            issues?: Array<{
+              type?: string;
+              description?: string;
+              severity?: string;
+            }>;
+            suggestions?: string[];
+          };
+
+          return createEvaluationResult({
+            quality: this.parseQuality(parsed.quality),
+            score: Math.min(1, Math.max(0, parsed.score ?? 0.5)),
+            acceptable: parsed.acceptable ?? false,
+            issues: (parsed.issues ?? []).map((issue) => ({
+              type: this.parseIssueType(issue.type),
+              description: issue.description ?? 'Unknown issue',
+              severity: this.parseSeverity(issue.severity),
+              suggestion: undefined,
+            })),
+            method: 'agent',
+            rawResponse: result,
+          });
+        }
+      } catch {
+        // Fall through to heuristic parsing
+      }
+    }
+
+    // If we can't parse, return a conservative result
+    return createEvaluationResult({
+      quality: 'needs_work',
+      score: 0.5,
+      acceptable: false,
+      issues: [
+        {
+          type: 'error',
+          description: 'Could not parse agent evaluation response',
+          severity: 'major',
+        },
+      ],
+      method: 'agent',
+      rawResponse: typeof result === 'string' ? result : JSON.stringify(result),
+    });
+  }
+
+  /**
+   * Heuristic evaluation based on trajectory properties
+   */
+  private evaluateHeuristic(
+    trajectory: Trajectory,
+    _task: Task
+  ): EvaluationResult {
+    const issues: Array<{
+      type: 'incomplete' | 'incorrect' | 'inefficient' | 'off_topic' | 'error';
+      description: string;
+      severity: 'critical' | 'major' | 'minor';
+    }> = [];
+
+    let score = 0.5; // Start neutral
+
+    // Check outcome
+    if (trajectory.outcome.success) {
+      score += 0.3;
+    } else {
+      score -= 0.2;
+      issues.push({
+        type: 'error',
+        description: trajectory.outcome.errorInfo ?? 'Task did not complete successfully',
+        severity: 'critical',
+      });
+    }
+
+    // Check for steps taken
+    if (trajectory.steps.length === 0) {
+      score -= 0.2;
+      issues.push({
+        type: 'incomplete',
+        description: 'No steps were taken to solve the task',
+        severity: 'critical',
+      });
+    } else {
+      // Penalize very long trajectories (may indicate inefficiency)
+      if (trajectory.steps.length > 20) {
+        score -= 0.1;
+        issues.push({
+          type: 'inefficient',
+          description: `Solution took ${trajectory.steps.length} steps, which may indicate inefficiency`,
+          severity: 'minor',
+        });
+      }
+    }
+
+    // Check for errors in steps
+    const errorSteps = trajectory.steps.filter(
+      (step) => step.observation?.toLowerCase().includes('error')
+    );
+    if (errorSteps.length > 0) {
+      score -= 0.1 * Math.min(3, errorSteps.length);
+      issues.push({
+        type: 'error',
+        description: `${errorSteps.length} step(s) encountered errors`,
+        severity: errorSteps.length > 2 ? 'major' : 'minor',
+      });
+    }
+
+    // Check key steps (high attribution)
+    const keySteps = trajectory.steps.filter(
+      (step) => (step.attributionScore ?? 0) >= 0.15
+    );
+    if (keySteps.length > 0 && trajectory.outcome.success) {
+      score += 0.1;
+    }
+
+    // Normalize score to [0, 1]
+    score = Math.max(0, Math.min(1, score));
+
+    return createEvaluationResult({
+      quality: scoreToQuality(score),
+      score,
+      acceptable: trajectory.outcome.success && score >= 0.6,
+      issues,
+      method: 'heuristic',
+    });
+  }
+
+  /**
+   * Parse quality string to QualityLevel
+   */
+  private parseQuality(
+    quality?: string
+  ): 'excellent' | 'good' | 'needs_work' | 'poor' {
+    const normalized = quality?.toLowerCase();
+    switch (normalized) {
+      case 'excellent':
+        return 'excellent';
+      case 'good':
+        return 'good';
+      case 'needs_work':
+      case 'needs work':
+        return 'needs_work';
+      case 'poor':
+        return 'poor';
+      default:
+        return 'needs_work';
+    }
+  }
+
+  /**
+   * Parse issue type string
+   */
+  private parseIssueType(
+    type?: string
+  ): 'incomplete' | 'incorrect' | 'inefficient' | 'off_topic' | 'error' {
+    const normalized = type?.toLowerCase();
+    switch (normalized) {
+      case 'incomplete':
+        return 'incomplete';
+      case 'incorrect':
+        return 'incorrect';
+      case 'inefficient':
+        return 'inefficient';
+      case 'off_topic':
+      case 'off-topic':
+        return 'off_topic';
+      case 'error':
+        return 'error';
+      default:
+        return 'error';
+    }
+  }
+
+  /**
+   * Parse severity string
+   */
+  private parseSeverity(severity?: string): 'critical' | 'major' | 'minor' {
+    const normalized = severity?.toLowerCase();
+    switch (normalized) {
+      case 'critical':
+        return 'critical';
+      case 'major':
+        return 'major';
+      case 'minor':
+        return 'minor';
+      default:
+        return 'major';
+    }
+  }
+}
+
+/**
+ * Create a solution evaluator
+ */
+export function createSolutionEvaluator(
+  agentManager: AgentManager | null,
+  config?: EvaluatorConfig
+): SolutionEvaluator {
+  return new SolutionEvaluator(agentManager, config);
+}

package/src/search/index.ts

@@ -0,0 +1,59 @@
+export {
+  TaskRouter,
+  createRouter,
+  type RoutingDecision,
+} from './router.js';
+
+export {
+  DirectSolver,
+  createSolver,
+  type SolverConfig,
+  type SolverResult,
+} from './solver.js';
+
+// Refinement types
+export {
+  type EvaluationResult,
+  type EvaluationIssue,
+  type EvaluationMethod,
+  type QualityLevel,
+  type FailureContext,
+  type RefinementResult,
+  type IssueType,
+  type IssueSeverity,
+  createEvaluationResult,
+  scoreToQuality,
+  isQualityAcceptable,
+  EvaluationResultSchema,
+  EvaluationIssueSchema,
+  QualityLevelSchema,
+  IssueSeveritySchema,
+  IssueTypeSchema,
+  EvaluationMethodSchema,
+} from './refinement-types.js';
+
+// Solution evaluator
+export {
+  SolutionEvaluator,
+  createSolutionEvaluator,
+  type EvaluatorConfig,
+  type VerificationFunction,
+  type VerificationResult,
+} from './evaluator.js';
+
+// Refinement loop
+export {
+  RefinementLoop,
+  createRefinementLoop,
+  type RefinementLoopConfig,
+} from './refinement-loop.js';
+
+// Verification runner
+export {
+  VerificationRunner,
+  createVerificationRunner,
+  TestRunners,
+  type CommandVerificationConfig,
+  type CommandResult,
+  type VerificationIssue,
+} from './verification-runner.js';