cognitive-core 0.2.5 → 0.3.1

package/src/search/verification-runner.ts

@@ -1,570 +0,0 @@
-/**
- * Verification Runner
- *
- * Provides common verification patterns for evaluating solutions:
- * - Command execution (bash/shell)
- * - Test runner integration
- * - Code linting/type checking
- * - Custom verification functions
- *
- * Used with SolutionEvaluator to verify trajectory outcomes.
- */
-
-import type { Trajectory } from '../types/trajectory.js';
-import type { Task } from '../types/task.js';
-import type { VerificationFunction, VerificationResult } from './evaluator.js';
-import { execSync, spawn } from 'child_process';
-
-/**
- * Configuration for a verification command
- */
-export interface CommandVerificationConfig {
-  /** Command to run (can include placeholders like {{solution}}, {{taskId}}) */
-  command: string;
-  /** Working directory for the command */
-  cwd?: string;
-  /** Environment variables to set */
-  env?: Record<string, string>;
-  /** Timeout in milliseconds (default: 30000) */
-  timeout?: number;
-  /** Whether exit code 0 indicates success (default: true) */
-  successOnZeroExit?: boolean;
-  /** Pattern to search for in output to indicate success */
-  successPattern?: RegExp;
-  /** Pattern to search for in output to indicate failure */
-  failurePattern?: RegExp;
-  /** Function to extract issues from output */
-  issueExtractor?: (output: string, exitCode: number) => VerificationIssue[];
-}
-
-/**
- * Issue found during verification
- */
-export interface VerificationIssue {
-  type: 'incomplete' | 'incorrect' | 'error';
-  description: string;
-  severity?: 'critical' | 'major' | 'minor';
-  line?: number;
-  file?: string;
-}
-
-/**
- * Result of running a command
- */
-export interface CommandResult {
-  stdout: string;
-  stderr: string;
-  exitCode: number;
-  timedOut: boolean;
-}
-
-/**
- * Common test runner configurations
- */
-export const TestRunners = {
-  /** Node.js test runners */
-  vitest: (testPath?: string): CommandVerificationConfig => ({
-    command: `npx vitest run ${testPath ?? ''} --reporter=json`,
-    timeout: 60000,
-    successOnZeroExit: true,
-    issueExtractor: extractVitestIssues,
-  }),
-
-  jest: (testPath?: string): CommandVerificationConfig => ({
-    command: `npx jest ${testPath ?? ''} --json`,
-    timeout: 60000,
-    successOnZeroExit: true,
-    issueExtractor: extractJestIssues,
-  }),
-
-  mocha: (testPath?: string): CommandVerificationConfig => ({
-    command: `npx mocha ${testPath ?? ''} --reporter json`,
-    timeout: 60000,
-    successOnZeroExit: true,
-  }),
-
-  /** Python test runners */
-  pytest: (testPath?: string): CommandVerificationConfig => ({
-    command: `python -m pytest ${testPath ?? ''} --tb=short -q`,
-    timeout: 60000,
-    successOnZeroExit: true,
-    failurePattern: /FAILED|ERROR/,
-    issueExtractor: extractPytestIssues,
-  }),
-
-  /** Go test runner */
-  goTest: (testPath?: string): CommandVerificationConfig => ({
-    command: `go test ${testPath ?? './...'} -v`,
-    timeout: 60000,
-    successOnZeroExit: true,
-    failurePattern: /--- FAIL:|FAIL\s+\w+/,
-  }),
-
-  /** Rust test runner */
-  cargoTest: (testPath?: string): CommandVerificationConfig => ({
-    command: `cargo test ${testPath ?? ''} -- --nocapture`,
-    timeout: 120000,
-    successOnZeroExit: true,
-    failurePattern: /test .+ \.\.\. FAILED/,
-  }),
-
-  /** TypeScript type checking */
-  typescript: (): CommandVerificationConfig => ({
-    command: 'npx tsc --noEmit',
-    timeout: 60000,
-    successOnZeroExit: true,
-    issueExtractor: extractTypescriptIssues,
-  }),
-
-  /** ESLint */
-  eslint: (path?: string): CommandVerificationConfig => ({
-    command: `npx eslint ${path ?? '.'} --format json`,
-    timeout: 30000,
-    successOnZeroExit: true,
-    issueExtractor: extractEslintIssues,
-  }),
-};
-
-/**
- * VerificationRunner - Executes verification commands and parses results
- */
-export class VerificationRunner {
-  private defaultCwd: string;
-  private defaultTimeout: number;
-  private defaultEnv: Record<string, string>;
-
-  constructor(options?: {
-    cwd?: string;
-    timeout?: number;
-    env?: Record<string, string>;
-  }) {
-    this.defaultCwd = options?.cwd ?? process.cwd();
-    this.defaultTimeout = options?.timeout ?? 30000;
-    this.defaultEnv = options?.env ?? {};
-  }
-
-  /**
-   * Create a verification function from a command configuration
-   */
-  createVerifier(config: CommandVerificationConfig): VerificationFunction {
-    return async (trajectory: Trajectory, task: Task): Promise<VerificationResult> => {
-      // Interpolate command with trajectory/task values
-      const command = this.interpolateCommand(config.command, trajectory, task);
-
-      // Run the command
-      const result = await this.runCommand(command, {
-        cwd: config.cwd ?? this.defaultCwd,
-        env: { ...this.defaultEnv, ...config.env },
-        timeout: config.timeout ?? this.defaultTimeout,
-      });
-
-      // Determine success
-      let passed = false;
-      const combinedOutput = result.stdout + '\n' + result.stderr;
-
-      if (result.timedOut) {
-        passed = false;
-      } else if (config.successOnZeroExit !== false && result.exitCode === 0) {
-        // Check success pattern if provided
-        if (config.successPattern) {
-          passed = config.successPattern.test(combinedOutput);
-        } else {
-          passed = true;
-        }
-      } else if (config.failurePattern) {
-        passed = !config.failurePattern.test(combinedOutput);
-      }
-
-      // Extract issues
-      let issues: VerificationIssue[] = [];
-      if (config.issueExtractor) {
-        issues = config.issueExtractor(combinedOutput, result.exitCode);
-      } else if (!passed) {
-        // Default issue extraction
-        issues = this.extractDefaultIssues(combinedOutput, result.exitCode);
-      }
-
-      // Calculate confidence based on clarity of result
-      const confidence = result.timedOut
-        ? 0.5
-        : (config.successPattern || config.failurePattern || config.issueExtractor)
-          ? 0.9
-          : 0.7;
-
-      return {
-        passed,
-        confidence,
-        issues: issues.map((issue) => ({
-          type: issue.type,
-          description: issue.description,
-          severity: issue.severity,
-        })),
-        details: result.timedOut
-          ? 'Verification timed out'
-          : `Exit code: ${result.exitCode}\n${combinedOutput.slice(0, 2000)}`,
-      };
-    };
-  }
-
-  /**
-   * Run a command and capture output
-   */
-  async runCommand(
-    command: string,
-    options: {
-      cwd?: string;
-      env?: Record<string, string>;
-      timeout?: number;
-    }
-  ): Promise<CommandResult> {
-    return new Promise((resolve) => {
-      const cwd = options.cwd ?? this.defaultCwd;
-      const timeout = options.timeout ?? this.defaultTimeout;
-      const env = { ...process.env, ...this.defaultEnv, ...options.env };
-
-      let stdout = '';
-      let stderr = '';
-      let timedOut = false;
-
-      const child = spawn(command, {
-        shell: true,
-        cwd,
-        env,
-      });
-
-      const timer = setTimeout(() => {
-        timedOut = true;
-        child.kill('SIGTERM');
-        // Force kill after 5 seconds if still running
-        setTimeout(() => child.kill('SIGKILL'), 5000);
-      }, timeout);
-
-      child.stdout.on('data', (data) => {
-        stdout += data.toString();
-      });
-
-      child.stderr.on('data', (data) => {
-        stderr += data.toString();
-      });
-
-      child.on('close', (code) => {
-        clearTimeout(timer);
-        resolve({
-          stdout,
-          stderr,
-          exitCode: code ?? (timedOut ? 124 : 1),
-          timedOut,
-        });
-      });
-
-      child.on('error', (error) => {
-        clearTimeout(timer);
-        resolve({
-          stdout,
-          stderr: stderr + '\n' + error.message,
-          exitCode: 1,
-          timedOut: false,
-        });
-      });
-    });
-  }
-
-  /**
-   * Run a command synchronously (for simple checks)
-   */
-  runCommandSync(
-    command: string,
-    options?: {
-      cwd?: string;
-      env?: Record<string, string>;
-      timeout?: number;
-    }
-  ): CommandResult {
-    const cwd = options?.cwd ?? this.defaultCwd;
-    const timeout = options?.timeout ?? this.defaultTimeout;
-    const env = { ...process.env, ...this.defaultEnv, ...options?.env };
-
-    try {
-      const output = execSync(command, {
-        cwd,
-        env,
-        timeout,
-        encoding: 'utf-8',
-        stdio: ['pipe', 'pipe', 'pipe'],
-      });
-
-      return {
-        stdout: output,
-        stderr: '',
-        exitCode: 0,
-        timedOut: false,
-      };
-    } catch (error: any) {
-      const timedOut = error.killed && error.signal === 'SIGTERM';
-      return {
-        stdout: error.stdout?.toString() ?? '',
-        stderr: error.stderr?.toString() ?? error.message,
-        exitCode: error.status ?? 1,
-        timedOut,
-      };
-    }
-  }
-
-  /**
-   * Interpolate command with trajectory/task values
-   */
-  private interpolateCommand(
-    command: string,
-    trajectory: Trajectory,
-    task: Task
-  ): string {
-    return command
-      .replace(/\{\{taskId\}\}/g, task.id)
-      .replace(/\{\{domain\}\}/g, task.domain ?? '')
-      .replace(/\{\{trajectoryId\}\}/g, trajectory.id)
-      .replace(
-        /\{\{solution\}\}/g,
-        typeof trajectory.outcome.solution === 'string'
-          ? trajectory.outcome.solution
-          : ''
-      );
-  }
-
-  /**
-   * Extract default issues from output
-   */
-  private extractDefaultIssues(
-    output: string,
-    exitCode: number
-  ): VerificationIssue[] {
-    const issues: VerificationIssue[] = [];
-
-    if (exitCode !== 0) {
-      // Look for common error patterns
-      const errorMatch = output.match(/(?:error|Error|ERROR)[:\s]+(.+?)(?:\n|$)/);
-      const failMatch = output.match(/(?:fail|Fail|FAIL)[:\s]+(.+?)(?:\n|$)/);
-
-      if (errorMatch) {
-        issues.push({
-          type: 'error',
-          description: errorMatch[1].slice(0, 200),
-          severity: 'critical',
-        });
-      } else if (failMatch) {
-        issues.push({
-          type: 'incorrect',
-          description: failMatch[1].slice(0, 200),
-          severity: 'major',
-        });
-      } else {
-        issues.push({
-          type: 'error',
-          description: `Command failed with exit code ${exitCode}`,
-          severity: 'major',
-        });
-      }
-    }
-
-    return issues;
-  }
-
-  /**
-   * Create a composite verifier that runs multiple verifications
-   */
-  createCompositeVerifier(
-    configs: CommandVerificationConfig[]
-  ): VerificationFunction {
-    const verifiers = configs.map((config) => this.createVerifier(config));
-
-    return async (trajectory: Trajectory, task: Task): Promise<VerificationResult> => {
-      const results: VerificationResult[] = [];
-
-      for (const verifier of verifiers) {
-        const result = await verifier(trajectory, task);
-        results.push(result);
-
-        // Fail fast on critical failure
-        if (!result.passed && result.issues?.some((i) => i.severity === 'critical')) {
-          break;
-        }
-      }
-
-      // Aggregate results
-      const allPassed = results.every((r) => r.passed);
-      const avgConfidence =
-        results.reduce((sum, r) => sum + r.confidence, 0) / results.length;
-      const allIssues = results.flatMap((r) => r.issues ?? []);
-      const details = results.map((r) => r.details).filter(Boolean).join('\n---\n');
-
-      return {
-        passed: allPassed,
-        confidence: avgConfidence,
-        issues: allIssues,
-        details,
-      };
-    };
-  }
-}
-
-/**
- * Create a verification runner
- */
-export function createVerificationRunner(options?: {
-  cwd?: string;
-  timeout?: number;
-  env?: Record<string, string>;
-}): VerificationRunner {
-  return new VerificationRunner(options);
-}
-
-// Issue extractors for common test runners
-
-function extractVitestIssues(output: string, _exitCode: number): VerificationIssue[] {
-  const issues: VerificationIssue[] = [];
-
-  try {
-    // Try to parse JSON output
-    const jsonMatch = output.match(/\{[\s\S]*"testResults"[\s\S]*\}/);
-    if (jsonMatch) {
-      const data = JSON.parse(jsonMatch[0]);
-      for (const testResult of data.testResults ?? []) {
-        for (const assertion of testResult.assertionResults ?? []) {
-          if (assertion.status === 'failed') {
-            issues.push({
-              type: 'incorrect',
-              description: `${assertion.fullName}: ${assertion.failureMessages?.[0]?.slice(0, 200) ?? 'Failed'}`,
-              severity: 'major',
-              file: testResult.name,
-            });
-          }
-        }
-      }
-    }
-  } catch {
-    // Fall back to pattern matching
-    const failureMatches = output.matchAll(/FAIL\s+(.+?)\n.*?(\d+)\s+failed/g);
-    for (const match of failureMatches) {
-      issues.push({
-        type: 'incorrect',
-        description: `Test suite failed: ${match[1]}`,
-        severity: 'major',
-        file: match[1],
-      });
-    }
-  }
-
-  return issues;
-}
-
-function extractJestIssues(output: string, _exitCode: number): VerificationIssue[] {
-  const issues: VerificationIssue[] = [];
-
-  try {
-    const jsonMatch = output.match(/\{[\s\S]*"testResults"[\s\S]*\}/);
-    if (jsonMatch) {
-      const data = JSON.parse(jsonMatch[0]);
-      for (const testResult of data.testResults ?? []) {
-        for (const assertion of testResult.assertionResults ?? []) {
-          if (assertion.status === 'failed') {
-            issues.push({
-              type: 'incorrect',
-              description: `${assertion.fullName}: ${assertion.failureMessages?.[0]?.slice(0, 200) ?? 'Failed'}`,
-              severity: 'major',
-              file: testResult.name,
-            });
-          }
-        }
-      }
-    }
-  } catch {
-    const failureMatches = output.matchAll(/● (.+)/g);
-    for (const match of failureMatches) {
-      issues.push({
-        type: 'incorrect',
-        description: match[1].slice(0, 200),
-        severity: 'major',
-      });
-    }
-  }
-
-  return issues;
-}
-
-function extractPytestIssues(output: string, _exitCode: number): VerificationIssue[] {
-  const issues: VerificationIssue[] = [];
-
-  // Match pytest failure lines
-  const failureMatches = output.matchAll(/FAILED\s+(.+?)(?:::|$)/g);
-  for (const match of failureMatches) {
-    issues.push({
-      type: 'incorrect',
-      description: `Test failed: ${match[1]}`,
-      severity: 'major',
-    });
-  }
-
-  // Match pytest errors
-  const errorMatches = output.matchAll(/ERROR\s+(.+?)(?:\n|$)/g);
-  for (const match of errorMatches) {
-    issues.push({
-      type: 'error',
-      description: match[1].slice(0, 200),
-      severity: 'critical',
-    });
-  }
-
-  return issues;
-}
-
-function extractTypescriptIssues(output: string, _exitCode: number): VerificationIssue[] {
-  const issues: VerificationIssue[] = [];
-
-  // Match TypeScript error format: file(line,col): error TS####: message
-  const errorMatches = output.matchAll(/(.+?)\((\d+),\d+\):\s+error\s+TS\d+:\s+(.+?)(?:\n|$)/g);
-  for (const match of errorMatches) {
-    issues.push({
-      type: 'error',
-      description: match[3].slice(0, 200),
-      severity: 'critical',
-      file: match[1],
-      line: parseInt(match[2], 10),
-    });
-  }
-
-  return issues;
-}
-
-function extractEslintIssues(output: string, _exitCode: number): VerificationIssue[] {
-  const issues: VerificationIssue[] = [];
-
-  try {
-    const data = JSON.parse(output);
-    for (const file of data) {
-      for (const message of file.messages ?? []) {
-        const severity: 'critical' | 'major' | 'minor' =
-          message.severity === 2 ? 'major' : 'minor';
-
-        issues.push({
-          type: message.fatal ? 'error' : 'incorrect',
-          description: `${message.ruleId}: ${message.message}`,
-          severity,
-          file: file.filePath,
-          line: message.line,
-        });
-      }
-    }
-  } catch {
-    // ESLint output wasn't JSON
-    const errorMatches = output.matchAll(/(\d+):\d+\s+error\s+(.+?)\s+(\S+)/g);
-    for (const match of errorMatches) {
-      issues.push({
-        type: 'error',
-        description: `${match[3]}: ${match[2]}`,
-        severity: 'major',
-        line: parseInt(match[1], 10),
-      });
-    }
-  }
-
-  return issues;
-}