cognitive-core 0.2.4 → 0.3.1

package/src/runtime/types.ts

@@ -1,198 +0,0 @@
-import type { Task, Trajectory } from '../types/index.js';
-import type { MemoryQueryResultV2 } from '../memory/system.js';
-import type { ComputeRequirements } from './compute-provider.js';
-
-/**
- * Message in an agent conversation
- */
-export interface AgentMessage {
-  role: 'user' | 'assistant' | 'system' | 'tool_use' | 'tool_result';
-  content: string;
-  toolName?: string;
-  toolInput?: unknown;
-  toolOutput?: unknown;
-  timestamp: Date;
-}
-
-/**
- * Tool call made by an agent
- */
-export interface ToolCall {
-  id: string;
-  name: string;
-  input: unknown;
-  output?: unknown;
-  error?: string;
-  startTime: Date;
-  endTime?: Date;
-}
-
-/**
- * Agent execution state
- */
-export type AgentState = 'idle' | 'running' | 'paused' | 'completed' | 'failed';
-
-/**
- * Agent session - represents a running or completed agent execution
- */
-export interface AgentSession {
-  id: string;
-  agentType: string;
-  task: Task;
-  state: AgentState;
-  messages: AgentMessage[];
-  toolCalls: ToolCall[];
-  startTime: Date;
-  endTime?: Date;
-  result?: unknown;
-  error?: string;
-  metadata: Record<string, unknown>;
-}
-
-/**
- * Configuration for spawning an agent
- */
-export interface AgentSpawnConfig {
-  /** Type of agent to spawn (e.g., 'claude-code', 'custom', etc.) */
-  agentType: string;
-  /** Task for the agent to work on */
-  task: Task;
-  /** Optional knowledge to inject into the agent's context */
-  injectedKnowledge?: MemoryQueryResultV2;
-  /** Custom system prompt additions */
-  systemPromptAdditions?: string;
-  /** Environment variables for the agent */
-  env?: Record<string, string>;
-  /** Working directory */
-  cwd?: string;
-  /** Timeout in milliseconds */
-  timeout?: number;
-  /** Whether to capture detailed tool calls */
-  captureToolCalls?: boolean;
-  /** Callback for real-time observation */
-  onMessage?: (message: AgentMessage) => void;
-  onToolCall?: (toolCall: ToolCall) => void;
-  /** Additional backend-specific options */
-  backendOptions?: Record<string, unknown>;
-  /** Compute requirements for provider-based backend resolution */
-  computeRequirements?: ComputeRequirements;
-}
-
-/**
- * Result of agent execution
- */
-export interface AgentResult {
-  session: AgentSession;
-  trajectory: Trajectory;
-  success: boolean;
-  metrics: {
-    totalTime: number;
-    toolCallCount: number;
-    messageCount: number;
-    tokenEstimate?: number;
-  };
-}
-
-/**
- * Agent backend protocol - implement this for different agent systems
- */
-export interface AgentBackend {
-  /** Unique name for this backend */
-  readonly name: string;
-
-  /** Supported agent types */
-  readonly supportedTypes: string[];
-
-  /** Check if backend is available/configured */
-  isAvailable(): Promise<boolean>;
-
-  /** Spawn and run an agent */
-  spawn(config: AgentSpawnConfig): Promise<AgentSession>;
-
-  /** Get status of a running session */
-  getSession(sessionId: string): Promise<AgentSession | undefined>;
-
-  /** Send a message to a running session */
-  sendMessage?(sessionId: string, message: string): Promise<void>;
-
-  /** Pause a running session */
-  pause?(sessionId: string): Promise<void>;
-
-  /** Resume a paused session */
-  resume?(sessionId: string): Promise<void>;
-
-  /** Terminate a session */
-  terminate?(sessionId: string): Promise<void>;
-
-  /** List active sessions */
-  listSessions?(): Promise<AgentSession[]>;
-}
-
-/**
- * Knowledge injector - formats knowledge for injection into agents
- */
-export interface KnowledgeInjector {
-  /** Format knowledge for a specific agent type */
-  format(
-    knowledge: MemoryQueryResultV2,
-    agentType: string,
-    options?: { maxTokens?: number; format?: 'markdown' | 'json' | 'xml' }
-  ): string;
-}
-
-/**
- * Trajectory extractor - extracts trajectories from agent sessions
- */
-export interface TrajectoryExtractor {
-  /** Extract a trajectory from an agent session */
-  extract(session: AgentSession, task: Task): Trajectory;
-}
-
-/**
- * Agent observer callbacks
- */
-export interface AgentObserverCallbacks {
-  onSessionStart?: (session: AgentSession) => void;
-  onMessage?: (session: AgentSession, message: AgentMessage) => void;
-  onToolCall?: (session: AgentSession, toolCall: ToolCall) => void;
-  onSessionEnd?: (session: AgentSession, trajectory: Trajectory) => void;
-  onError?: (session: AgentSession, error: Error) => void;
-}
-
-/**
- * Options passed to an AgentDelegate when executing a task
- */
-export interface AgentDelegateOptions {
-  /** Working directory for execution */
-  cwd: string;
-  /** System context (formatted knowledge + skill context) */
-  systemContext?: string;
-  /** Timeout in milliseconds */
-  timeoutMs?: number;
-}
-
-/**
- * Result returned by an AgentDelegate after execution
- */
-export interface AgentDelegateResult {
-  /** Whether the execution succeeded */
-  success: boolean;
-  /** Text output from the agent */
-  output: string;
-  /** Optional structured output */
-  structured?: unknown;
-  /** Optional trajectory — when provided, used directly for full learning fidelity */
-  trajectory?: Trajectory;
-}
-
-/**
- * AgentDelegate — lets callers provide their own agent instances.
- *
- * When set on AgentManager, bypasses backend resolution and compute provider.
- * The caller owns all lifecycle, tracking, and billing. cognitive-core only
- * sends a prompt + context and receives a result.
- */
-export interface AgentDelegate {
-  /** Execute a task with the given prompt and options */
-  execute(prompt: string, options: AgentDelegateOptions): Promise<AgentDelegateResult>;
-}

package/src/search/evaluator.ts

@@ -1,499 +0,0 @@
-/**
- * Solution Evaluator
- *
- * Evaluates solution quality using a hybrid approach:
- * 1. First tries task verification (if available)
- * 2. Falls back to ACP agent evaluation
- * 3. Falls back to heuristic evaluation as last resort
- */
-
-import type { Trajectory } from '../types/trajectory.js';
-import type { Task } from '../types/task.js';
-import type { AgentManager } from '../runtime/manager.js';
-import type { AgenticTaskRunner } from '../workspace/runner.js';
-import { solutionEvaluationTemplate } from '../workspace/templates/solution-evaluation.js';
-import {
-  type EvaluationResult,
-  createEvaluationResult,
-  scoreToQuality,
-} from './refinement-types.js';
-
-/**
- * Verification function signature for task-specific verification
- */
-export interface VerificationFunction {
-  (trajectory: Trajectory, task: Task): Promise<VerificationResult>;
-}
-
-/**
- * Result from a verification function
- */
-export interface VerificationResult {
-  /** Whether the solution passed verification */
-  passed: boolean;
-  /** Confidence in the verification result (0-1) */
-  confidence: number;
-  /** Specific issues found */
-  issues?: Array<{
-    type: 'incomplete' | 'incorrect' | 'error';
-    description: string;
-    severity?: 'critical' | 'major' | 'minor';
-  }>;
-  /** Additional details */
-  details?: string;
-}
-
-/**
- * Configuration for the SolutionEvaluator
- */
-export interface EvaluatorConfig {
-  /** Agent type to use for evaluation (default: 'evaluator') */
-  evaluatorAgentType?: string;
-  /** Minimum confidence for verification to be accepted */
-  verificationConfidenceThreshold?: number;
-  /** Whether to always run agent evaluation even if verification passes */
-  alwaysUseAgent?: boolean;
-  /** Timeout for agent evaluation in ms */
-  agentTimeout?: number;
-}
-
-/**
- * Default evaluator configuration
- */
-const DEFAULT_CONFIG: Required<EvaluatorConfig> = {
-  evaluatorAgentType: 'evaluator',
-  verificationConfidenceThreshold: 0.8,
-  alwaysUseAgent: false,
-  agentTimeout: 60000,
-};
-
-/**
- * SolutionEvaluator - Evaluates trajectory quality using hybrid approach
- */
-export class SolutionEvaluator {
-  private agentManager: AgentManager | null;
-  private taskRunner: AgenticTaskRunner | null = null;
-  private config: Required<EvaluatorConfig>;
-  private verifiers: Map<string, VerificationFunction> = new Map();
-
-  constructor(
-    agentManager: AgentManager | null,
-    config: EvaluatorConfig = {}
-  ) {
-    this.agentManager = agentManager;
-    this.config = { ...DEFAULT_CONFIG, ...config };
-  }
-
-  /**
-   * Set the task runner for workspace-based evaluation.
-   * When set, evaluation delegates to solutionEvaluationTemplate.
-   */
-  setTaskRunner(runner: AgenticTaskRunner): void {
-    this.taskRunner = runner;
-  }
-
-  /**
-   * Register a verification function for a task domain
-   */
-  registerVerifier(domain: string, verifier: VerificationFunction): void {
-    this.verifiers.set(domain, verifier);
-  }
-
-  /**
-   * Evaluate a trajectory's quality
-   */
-  async evaluate(
-    trajectory: Trajectory,
-    task: Task
-  ): Promise<EvaluationResult> {
-    // 1. Try verification first (if available and applicable)
-    const verificationResult = await this.tryVerification(trajectory, task);
-    if (verificationResult) {
-      const { result, evalResult } = verificationResult;
-
-      // If verification is confident enough and doesn't always require agent
-      if (
-        result.confidence >= this.config.verificationConfidenceThreshold &&
-        !this.config.alwaysUseAgent
-      ) {
-        return evalResult;
-      }
-    }
-
-    // 2. Try workspace-based evaluation (preferred)
-    if (this.taskRunner) {
-      try {
-        const result = await this.taskRunner.run(solutionEvaluationTemplate, {
-          trajectory,
-          task,
-        });
-        return result.output;
-      } catch (error) {
-        console.warn(
-          'Workspace-based evaluation failed, falling back:',
-          error instanceof Error ? error.message : String(error)
-        );
-      }
-    }
-
-    // 3. Try ad-hoc ACP agent evaluation
-    if (this.agentManager) {
-      try {
-        const agentResult = await this.evaluateWithAgent(trajectory, task);
-        return agentResult;
-      } catch (error) {
-        const errorMsg = error instanceof Error ? error.message : String(error);
-        console.warn(`Agent evaluation failed, falling back to heuristic: ${errorMsg}`);
-      }
-    }
-
-    // 4. Fall back to heuristic evaluation
-    return this.evaluateHeuristic(trajectory, task);
-  }
-
-  /**
-   * Try verification if a verifier is available for the task domain
-   */
-  private async tryVerification(
-    trajectory: Trajectory,
-    task: Task
-  ): Promise<{ result: VerificationResult; evalResult: EvaluationResult } | null> {
-    // Check for domain-specific verifier
-    const verifier = task.domain ? this.verifiers.get(task.domain) : undefined;
-    if (!verifier) {
-      return null;
-    }
-
-    try {
-      const result = await verifier(trajectory, task);
-
-      const score = result.passed
-        ? Math.max(0.7, result.confidence)
-        : Math.min(0.5, 1 - result.confidence);
-
-      const evalResult = createEvaluationResult({
-        quality: scoreToQuality(score),
-        score,
-        acceptable: result.passed && result.confidence >= this.config.verificationConfidenceThreshold,
-        issues: result.issues?.map((issue) => ({
-          type: issue.type,
-          description: issue.description,
-          severity: issue.severity ?? 'major',
-        })) ?? [],
-        method: 'verification',
-        rawResponse: result.details,
-      });
-
-      return { result, evalResult };
-    } catch (error) {
-      // Verification failed, return null to try other methods
-      console.warn(
-        'Verification failed:',
-        error instanceof Error ? error.message : String(error)
-      );
-      return null;
-    }
-  }
-
-  /**
-   * Evaluate using an ACP agent
-   */
-  private async evaluateWithAgent(
-    trajectory: Trajectory,
-    task: Task
-  ): Promise<EvaluationResult> {
-    if (!this.agentManager) {
-      throw new Error('AgentManager not available for agent evaluation');
-    }
-
-    // Create evaluation task for the agent
-    const evaluationTask: Task = {
-      id: `eval-${trajectory.id}`,
-      description: this.buildEvaluationPrompt(trajectory, task),
-      domain: 'evaluation',
-      context: {},
-      createdAt: new Date(),
-      metadata: {
-        originalTaskId: task.id,
-        trajectoryId: trajectory.id,
-      },
-    };
-
-    // Spawn evaluation agent
-    const result = await this.agentManager.spawn({
-      agentType: this.config.evaluatorAgentType,
-      task: evaluationTask,
-      timeout: this.config.agentTimeout,
-      backendOptions: {
-        // Evaluation agents should be concise
-        maxTokens: 2000,
-      },
-    });
-
-    // Parse agent response into EvaluationResult
-    return this.parseAgentEvaluation(result.session.result, result.trajectory);
-  }
-
-  /**
-   * Build evaluation prompt for the agent
-   */
-  private buildEvaluationPrompt(trajectory: Trajectory, task: Task): string {
-    const steps = trajectory.steps
-      .map((step, i) => {
-        let stepStr = `Step ${i + 1}:`;
-        if (step.thought) stepStr += `\n  Thought: ${step.thought}`;
-        stepStr += `\n  Action: ${step.action}`;
-        if (step.observation) stepStr += `\n  Observation: ${step.observation}`;
-        return stepStr;
-      })
-      .join('\n\n');
-
-    const outcomeStr = trajectory.outcome.success
-      ? `SUCCESS: ${trajectory.outcome.solution ?? 'Task completed'}`
-      : `FAILURE: ${trajectory.outcome.errorInfo ?? 'Unknown error'}`;
-
-    return `Evaluate the quality of this solution attempt.
-
-## Original Task
-${task.description}
-
-## Solution Steps
-${steps}
-
-## Outcome
-${outcomeStr}
-
-## Instructions
-Analyze the solution and provide:
-1. Overall quality assessment (excellent/good/needs_work/poor)
-2. A numeric score from 0.0 to 1.0
-3. Whether this solution is acceptable
-4. List any specific issues found
-5. Suggestions for improvement
-
-Respond in JSON format:
-{
-  "quality": "excellent|good|needs_work|poor",
-  "score": 0.85,
-  "acceptable": true,
-  "issues": [
-    {"type": "incomplete|incorrect|inefficient|off_topic|error", "description": "...", "severity": "critical|major|minor"}
-  ],
-  "suggestions": ["..."]
-}`;
-  }
-
-  /**
-   * Parse agent evaluation response into EvaluationResult
-   */
-  private parseAgentEvaluation(
-    result: unknown,
-    _evalTrajectory: Trajectory
-  ): EvaluationResult {
-    // Try to parse as JSON
-    if (typeof result === 'string') {
-      try {
-        // Extract JSON from response (agent may include extra text)
-        const jsonMatch = result.match(/\{[\s\S]*\}/);
-        if (jsonMatch) {
-          const parsed = JSON.parse(jsonMatch[0]) as {
-            quality?: string;
-            score?: number;
-            acceptable?: boolean;
-            issues?: Array<{
-              type?: string;
-              description?: string;
-              severity?: string;
-            }>;
-            suggestions?: string[];
-          };
-
-          return createEvaluationResult({
-            quality: this.parseQuality(parsed.quality),
-            score: Math.min(1, Math.max(0, parsed.score ?? 0.5)),
-            acceptable: parsed.acceptable ?? false,
-            issues: (parsed.issues ?? []).map((issue) => ({
-              type: this.parseIssueType(issue.type),
-              description: issue.description ?? 'Unknown issue',
-              severity: this.parseSeverity(issue.severity),
-              suggestion: undefined,
-            })),
-            method: 'agent',
-            rawResponse: result,
-          });
-        }
-      } catch {
-        // Fall through to heuristic parsing
-      }
-    }
-
-    // If we can't parse, return a conservative result
-    return createEvaluationResult({
-      quality: 'needs_work',
-      score: 0.5,
-      acceptable: false,
-      issues: [
-        {
-          type: 'error',
-          description: 'Could not parse agent evaluation response',
-          severity: 'major',
-        },
-      ],
-      method: 'agent',
-      rawResponse: typeof result === 'string' ? result : JSON.stringify(result),
-    });
-  }
-
-  /**
-   * Heuristic evaluation based on trajectory properties
-   */
-  private evaluateHeuristic(
-    trajectory: Trajectory,
-    _task: Task
-  ): EvaluationResult {
-    const issues: Array<{
-      type: 'incomplete' | 'incorrect' | 'inefficient' | 'off_topic' | 'error';
-      description: string;
-      severity: 'critical' | 'major' | 'minor';
-    }> = [];
-
-    let score = 0.5; // Start neutral
-
-    // Check outcome
-    if (trajectory.outcome.success) {
-      score += 0.3;
-    } else {
-      score -= 0.2;
-      issues.push({
-        type: 'error',
-        description: trajectory.outcome.errorInfo ?? 'Task did not complete successfully',
-        severity: 'critical',
-      });
-    }
-
-    // Check for steps taken
-    if (trajectory.steps.length === 0) {
-      score -= 0.2;
-      issues.push({
-        type: 'incomplete',
-        description: 'No steps were taken to solve the task',
-        severity: 'critical',
-      });
-    } else {
-      // Penalize very long trajectories (may indicate inefficiency)
-      if (trajectory.steps.length > 20) {
-        score -= 0.1;
-        issues.push({
-          type: 'inefficient',
-          description: `Solution took ${trajectory.steps.length} steps, which may indicate inefficiency`,
-          severity: 'minor',
-        });
-      }
-    }
-
-    // Check for errors in steps
-    const errorSteps = trajectory.steps.filter(
-      (step) => step.observation?.toLowerCase().includes('error')
-    );
-    if (errorSteps.length > 0) {
-      score -= 0.1 * Math.min(3, errorSteps.length);
-      issues.push({
-        type: 'error',
-        description: `${errorSteps.length} step(s) encountered errors`,
-        severity: errorSteps.length > 2 ? 'major' : 'minor',
-      });
-    }
-
-    // Check key steps (high attribution)
-    const keySteps = trajectory.steps.filter(
-      (step) => (step.attributionScore ?? 0) >= 0.15
-    );
-    if (keySteps.length > 0 && trajectory.outcome.success) {
-      score += 0.1;
-    }
-
-    // Normalize score to [0, 1]
-    score = Math.max(0, Math.min(1, score));
-
-    return createEvaluationResult({
-      quality: scoreToQuality(score),
-      score,
-      acceptable: trajectory.outcome.success && score >= 0.6,
-      issues,
-      method: 'heuristic',
-    });
-  }
-
-  /**
-   * Parse quality string to QualityLevel
-   */
-  private parseQuality(
-    quality?: string
-  ): 'excellent' | 'good' | 'needs_work' | 'poor' {
-    const normalized = quality?.toLowerCase();
-    switch (normalized) {
-      case 'excellent':
-        return 'excellent';
-      case 'good':
-        return 'good';
-      case 'needs_work':
-      case 'needs work':
-        return 'needs_work';
-      case 'poor':
-        return 'poor';
-      default:
-        return 'needs_work';
-    }
-  }
-
-  /**
-   * Parse issue type string
-   */
-  private parseIssueType(
-    type?: string
-  ): 'incomplete' | 'incorrect' | 'inefficient' | 'off_topic' | 'error' {
-    const normalized = type?.toLowerCase();
-    switch (normalized) {
-      case 'incomplete':
-        return 'incomplete';
-      case 'incorrect':
-        return 'incorrect';
-      case 'inefficient':
-        return 'inefficient';
-      case 'off_topic':
-      case 'off-topic':
-        return 'off_topic';
-      case 'error':
-        return 'error';
-      default:
-        return 'error';
-    }
-  }
-
-  /**
-   * Parse severity string
-   */
-  private parseSeverity(severity?: string): 'critical' | 'major' | 'minor' {
-    const normalized = severity?.toLowerCase();
-    switch (normalized) {
-      case 'critical':
-        return 'critical';
-      case 'major':
-        return 'major';
-      case 'minor':
-        return 'minor';
-      default:
-        return 'major';
-    }
-  }
-}
-
-/**
- * Create a solution evaluator
- */
-export function createSolutionEvaluator(
-  agentManager: AgentManager | null,
-  config?: EvaluatorConfig
-): SolutionEvaluator {
-  return new SolutionEvaluator(agentManager, config);
-}