cognitive-core 0.0.2 → 0.1.2

package/src/search/verification-runner.ts

@@ -0,0 +1,570 @@
+/**
+ * Verification Runner
+ *
+ * Provides common verification patterns for evaluating solutions:
+ * - Command execution (bash/shell)
+ * - Test runner integration
+ * - Code linting/type checking
+ * - Custom verification functions
+ *
+ * Used with SolutionEvaluator to verify trajectory outcomes.
+ */
+
+import type { Trajectory } from '../types/trajectory.js';
+import type { Task } from '../types/task.js';
+import type { VerificationFunction, VerificationResult } from './evaluator.js';
+import { execSync, spawn } from 'child_process';
+
+/**
+ * Configuration for a verification command
+ */
+export interface CommandVerificationConfig {
+  /** Command to run (can include placeholders like {{solution}}, {{taskId}}) */
+  command: string;
+  /** Working directory for the command */
+  cwd?: string;
+  /** Environment variables to set */
+  env?: Record<string, string>;
+  /** Timeout in milliseconds (default: 30000) */
+  timeout?: number;
+  /** Whether exit code 0 indicates success (default: true) */
+  successOnZeroExit?: boolean;
+  /** Pattern to search for in output to indicate success */
+  successPattern?: RegExp;
+  /** Pattern to search for in output to indicate failure */
+  failurePattern?: RegExp;
+  /** Function to extract issues from output */
+  issueExtractor?: (output: string, exitCode: number) => VerificationIssue[];
+}
+
+/**
+ * Issue found during verification
+ */
+export interface VerificationIssue {
+  type: 'incomplete' | 'incorrect' | 'error';
+  description: string;
+  severity?: 'critical' | 'major' | 'minor';
+  line?: number;
+  file?: string;
+}
+
+/**
+ * Result of running a command
+ */
+export interface CommandResult {
+  stdout: string;
+  stderr: string;
+  exitCode: number;
+  timedOut: boolean;
+}
+
+/**
+ * Common test runner configurations
+ */
+export const TestRunners = {
+  /** Node.js test runners */
+  vitest: (testPath?: string): CommandVerificationConfig => ({
+    command: `npx vitest run ${testPath ?? ''} --reporter=json`,
+    timeout: 60000,
+    successOnZeroExit: true,
+    issueExtractor: extractVitestIssues,
+  }),
+
+  jest: (testPath?: string): CommandVerificationConfig => ({
+    command: `npx jest ${testPath ?? ''} --json`,
+    timeout: 60000,
+    successOnZeroExit: true,
+    issueExtractor: extractJestIssues,
+  }),
+
+  mocha: (testPath?: string): CommandVerificationConfig => ({
+    command: `npx mocha ${testPath ?? ''} --reporter json`,
+    timeout: 60000,
+    successOnZeroExit: true,
+  }),
+
+  /** Python test runners */
+  pytest: (testPath?: string): CommandVerificationConfig => ({
+    command: `python -m pytest ${testPath ?? ''} --tb=short -q`,
+    timeout: 60000,
+    successOnZeroExit: true,
+    failurePattern: /FAILED|ERROR/,
+    issueExtractor: extractPytestIssues,
+  }),
+
+  /** Go test runner */
+  goTest: (testPath?: string): CommandVerificationConfig => ({
+    command: `go test ${testPath ?? './...'} -v`,
+    timeout: 60000,
+    successOnZeroExit: true,
+    failurePattern: /--- FAIL:|FAIL\s+\w+/,
+  }),
+
+  /** Rust test runner */
+  cargoTest: (testPath?: string): CommandVerificationConfig => ({
+    command: `cargo test ${testPath ?? ''} -- --nocapture`,
+    timeout: 120000,
+    successOnZeroExit: true,
+    failurePattern: /test .+ \.\.\. FAILED/,
+  }),
+
+  /** TypeScript type checking */
+  typescript: (): CommandVerificationConfig => ({
+    command: 'npx tsc --noEmit',
+    timeout: 60000,
+    successOnZeroExit: true,
+    issueExtractor: extractTypescriptIssues,
+  }),
+
+  /** ESLint */
+  eslint: (path?: string): CommandVerificationConfig => ({
+    command: `npx eslint ${path ?? '.'} --format json`,
+    timeout: 30000,
+    successOnZeroExit: true,
+    issueExtractor: extractEslintIssues,
+  }),
+};
+
+/**
+ * VerificationRunner - Executes verification commands and parses results
+ */
+export class VerificationRunner {
+  private defaultCwd: string;
+  private defaultTimeout: number;
+  private defaultEnv: Record<string, string>;
+
+  constructor(options?: {
+    cwd?: string;
+    timeout?: number;
+    env?: Record<string, string>;
+  }) {
+    this.defaultCwd = options?.cwd ?? process.cwd();
+    this.defaultTimeout = options?.timeout ?? 30000;
+    this.defaultEnv = options?.env ?? {};
+  }
+
+  /**
+   * Create a verification function from a command configuration
+   */
+  createVerifier(config: CommandVerificationConfig): VerificationFunction {
+    return async (trajectory: Trajectory, task: Task): Promise<VerificationResult> => {
+      // Interpolate command with trajectory/task values
+      const command = this.interpolateCommand(config.command, trajectory, task);
+
+      // Run the command
+      const result = await this.runCommand(command, {
+        cwd: config.cwd ?? this.defaultCwd,
+        env: { ...this.defaultEnv, ...config.env },
+        timeout: config.timeout ?? this.defaultTimeout,
+      });
+
+      // Determine success
+      let passed = false;
+      const combinedOutput = result.stdout + '\n' + result.stderr;
+
+      if (result.timedOut) {
+        passed = false;
+      } else if (config.successOnZeroExit !== false && result.exitCode === 0) {
+        // Check success pattern if provided
+        if (config.successPattern) {
+          passed = config.successPattern.test(combinedOutput);
+        } else {
+          passed = true;
+        }
+      } else if (config.failurePattern) {
+        passed = !config.failurePattern.test(combinedOutput);
+      }
+
+      // Extract issues
+      let issues: VerificationIssue[] = [];
+      if (config.issueExtractor) {
+        issues = config.issueExtractor(combinedOutput, result.exitCode);
+      } else if (!passed) {
+        // Default issue extraction
+        issues = this.extractDefaultIssues(combinedOutput, result.exitCode);
+      }
+
+      // Calculate confidence based on clarity of result
+      const confidence = result.timedOut
+        ? 0.5
+        : (config.successPattern || config.failurePattern || config.issueExtractor)
+          ? 0.9
+          : 0.7;
+
+      return {
+        passed,
+        confidence,
+        issues: issues.map((issue) => ({
+          type: issue.type,
+          description: issue.description,
+          severity: issue.severity,
+        })),
+        details: result.timedOut
+          ? 'Verification timed out'
+          : `Exit code: ${result.exitCode}\n${combinedOutput.slice(0, 2000)}`,
+      };
+    };
+  }
+
+  /**
+   * Run a command and capture output
+   */
+  async runCommand(
+    command: string,
+    options: {
+      cwd?: string;
+      env?: Record<string, string>;
+      timeout?: number;
+    }
+  ): Promise<CommandResult> {
+    return new Promise((resolve) => {
+      const cwd = options.cwd ?? this.defaultCwd;
+      const timeout = options.timeout ?? this.defaultTimeout;
+      const env = { ...process.env, ...this.defaultEnv, ...options.env };
+
+      let stdout = '';
+      let stderr = '';
+      let timedOut = false;
+
+      const child = spawn(command, {
+        shell: true,
+        cwd,
+        env,
+      });
+
+      const timer = setTimeout(() => {
+        timedOut = true;
+        child.kill('SIGTERM');
+        // Force kill after 5 seconds if still running
+        setTimeout(() => child.kill('SIGKILL'), 5000);
+      }, timeout);
+
+      child.stdout.on('data', (data) => {
+        stdout += data.toString();
+      });
+
+      child.stderr.on('data', (data) => {
+        stderr += data.toString();
+      });
+
+      child.on('close', (code) => {
+        clearTimeout(timer);
+        resolve({
+          stdout,
+          stderr,
+          exitCode: code ?? (timedOut ? 124 : 1),
+          timedOut,
+        });
+      });
+
+      child.on('error', (error) => {
+        clearTimeout(timer);
+        resolve({
+          stdout,
+          stderr: stderr + '\n' + error.message,
+          exitCode: 1,
+          timedOut: false,
+        });
+      });
+    });
+  }
+
+  /**
+   * Run a command synchronously (for simple checks)
+   */
+  runCommandSync(
+    command: string,
+    options?: {
+      cwd?: string;
+      env?: Record<string, string>;
+      timeout?: number;
+    }
+  ): CommandResult {
+    const cwd = options?.cwd ?? this.defaultCwd;
+    const timeout = options?.timeout ?? this.defaultTimeout;
+    const env = { ...process.env, ...this.defaultEnv, ...options?.env };
+
+    try {
+      const output = execSync(command, {
+        cwd,
+        env,
+        timeout,
+        encoding: 'utf-8',
+        stdio: ['pipe', 'pipe', 'pipe'],
+      });
+
+      return {
+        stdout: output,
+        stderr: '',
+        exitCode: 0,
+        timedOut: false,
+      };
+    } catch (error: any) {
+      const timedOut = error.killed && error.signal === 'SIGTERM';
+      return {
+        stdout: error.stdout?.toString() ?? '',
+        stderr: error.stderr?.toString() ?? error.message,
+        exitCode: error.status ?? 1,
+        timedOut,
+      };
+    }
+  }
+
+  /**
+   * Interpolate command with trajectory/task values
+   */
+  private interpolateCommand(
+    command: string,
+    trajectory: Trajectory,
+    task: Task
+  ): string {
+    return command
+      .replace(/\{\{taskId\}\}/g, task.id)
+      .replace(/\{\{domain\}\}/g, task.domain ?? '')
+      .replace(/\{\{trajectoryId\}\}/g, trajectory.id)
+      .replace(
+        /\{\{solution\}\}/g,
+        typeof trajectory.outcome.solution === 'string'
+          ? trajectory.outcome.solution
+          : ''
+      );
+  }
+
+  /**
+   * Extract default issues from output
+   */
+  private extractDefaultIssues(
+    output: string,
+    exitCode: number
+  ): VerificationIssue[] {
+    const issues: VerificationIssue[] = [];
+
+    if (exitCode !== 0) {
+      // Look for common error patterns
+      const errorMatch = output.match(/(?:error|Error|ERROR)[:\s]+(.+?)(?:\n|$)/);
+      const failMatch = output.match(/(?:fail|Fail|FAIL)[:\s]+(.+?)(?:\n|$)/);
+
+      if (errorMatch) {
+        issues.push({
+          type: 'error',
+          description: errorMatch[1].slice(0, 200),
+          severity: 'critical',
+        });
+      } else if (failMatch) {
+        issues.push({
+          type: 'incorrect',
+          description: failMatch[1].slice(0, 200),
+          severity: 'major',
+        });
+      } else {
+        issues.push({
+          type: 'error',
+          description: `Command failed with exit code ${exitCode}`,
+          severity: 'major',
+        });
+      }
+    }
+
+    return issues;
+  }
+
+  /**
+   * Create a composite verifier that runs multiple verifications
+   */
+  createCompositeVerifier(
+    configs: CommandVerificationConfig[]
+  ): VerificationFunction {
+    const verifiers = configs.map((config) => this.createVerifier(config));
+
+    return async (trajectory: Trajectory, task: Task): Promise<VerificationResult> => {
+      const results: VerificationResult[] = [];
+
+      for (const verifier of verifiers) {
+        const result = await verifier(trajectory, task);
+        results.push(result);
+
+        // Fail fast on critical failure
+        if (!result.passed && result.issues?.some((i) => i.severity === 'critical')) {
+          break;
+        }
+      }
+
+      // Aggregate results
+      const allPassed = results.every((r) => r.passed);
+      const avgConfidence =
+        results.reduce((sum, r) => sum + r.confidence, 0) / results.length;
+      const allIssues = results.flatMap((r) => r.issues ?? []);
+      const details = results.map((r) => r.details).filter(Boolean).join('\n---\n');
+
+      return {
+        passed: allPassed,
+        confidence: avgConfidence,
+        issues: allIssues,
+        details,
+      };
+    };
+  }
+}
+
+/**
+ * Create a verification runner
+ */
+export function createVerificationRunner(options?: {
+  cwd?: string;
+  timeout?: number;
+  env?: Record<string, string>;
+}): VerificationRunner {
+  return new VerificationRunner(options);
+}
+
+// Issue extractors for common test runners
+
+function extractVitestIssues(output: string, _exitCode: number): VerificationIssue[] {
+  const issues: VerificationIssue[] = [];
+
+  try {
+    // Try to parse JSON output
+    const jsonMatch = output.match(/\{[\s\S]*"testResults"[\s\S]*\}/);
+    if (jsonMatch) {
+      const data = JSON.parse(jsonMatch[0]);
+      for (const testResult of data.testResults ?? []) {
+        for (const assertion of testResult.assertionResults ?? []) {
+          if (assertion.status === 'failed') {
+            issues.push({
+              type: 'incorrect',
+              description: `${assertion.fullName}: ${assertion.failureMessages?.[0]?.slice(0, 200) ?? 'Failed'}`,
+              severity: 'major',
+              file: testResult.name,
+            });
+          }
+        }
+      }
+    }
+  } catch {
+    // Fall back to pattern matching
+    const failureMatches = output.matchAll(/FAIL\s+(.+?)\n.*?(\d+)\s+failed/g);
+    for (const match of failureMatches) {
+      issues.push({
+        type: 'incorrect',
+        description: `Test suite failed: ${match[1]}`,
+        severity: 'major',
+        file: match[1],
+      });
+    }
+  }
+
+  return issues;
+}
+
+function extractJestIssues(output: string, _exitCode: number): VerificationIssue[] {
+  const issues: VerificationIssue[] = [];
+
+  try {
+    const jsonMatch = output.match(/\{[\s\S]*"testResults"[\s\S]*\}/);
+    if (jsonMatch) {
+      const data = JSON.parse(jsonMatch[0]);
+      for (const testResult of data.testResults ?? []) {
+        for (const assertion of testResult.assertionResults ?? []) {
+          if (assertion.status === 'failed') {
+            issues.push({
+              type: 'incorrect',
+              description: `${assertion.fullName}: ${assertion.failureMessages?.[0]?.slice(0, 200) ?? 'Failed'}`,
+              severity: 'major',
+              file: testResult.name,
+            });
+          }
+        }
+      }
+    }
+  } catch {
+    const failureMatches = output.matchAll(/● (.+)/g);
+    for (const match of failureMatches) {
+      issues.push({
+        type: 'incorrect',
+        description: match[1].slice(0, 200),
+        severity: 'major',
+      });
+    }
+  }
+
+  return issues;
+}
+
+function extractPytestIssues(output: string, _exitCode: number): VerificationIssue[] {
+  const issues: VerificationIssue[] = [];
+
+  // Match pytest failure lines
+  const failureMatches = output.matchAll(/FAILED\s+(.+?)(?:::|$)/g);
+  for (const match of failureMatches) {
+    issues.push({
+      type: 'incorrect',
+      description: `Test failed: ${match[1]}`,
+      severity: 'major',
+    });
+  }
+
+  // Match pytest errors
+  const errorMatches = output.matchAll(/ERROR\s+(.+?)(?:\n|$)/g);
+  for (const match of errorMatches) {
+    issues.push({
+      type: 'error',
+      description: match[1].slice(0, 200),
+      severity: 'critical',
+    });
+  }
+
+  return issues;
+}
+
+function extractTypescriptIssues(output: string, _exitCode: number): VerificationIssue[] {
+  const issues: VerificationIssue[] = [];
+
+  // Match TypeScript error format: file(line,col): error TS####: message
+  const errorMatches = output.matchAll(/(.+?)\((\d+),\d+\):\s+error\s+TS\d+:\s+(.+?)(?:\n|$)/g);
+  for (const match of errorMatches) {
+    issues.push({
+      type: 'error',
+      description: match[3].slice(0, 200),
+      severity: 'critical',
+      file: match[1],
+      line: parseInt(match[2], 10),
+    });
+  }
+
+  return issues;
+}
+
+function extractEslintIssues(output: string, _exitCode: number): VerificationIssue[] {
+  const issues: VerificationIssue[] = [];
+
+  try {
+    const data = JSON.parse(output);
+    for (const file of data) {
+      for (const message of file.messages ?? []) {
+        const severity: 'critical' | 'major' | 'minor' =
+          message.severity === 2 ? 'major' : 'minor';
+
+        issues.push({
+          type: message.fatal ? 'error' : 'incorrect',
+          description: `${message.ruleId}: ${message.message}`,
+          severity,
+          file: file.filePath,
+          line: message.line,
+        });
+      }
+    }
+  } catch {
+    // ESLint output wasn't JSON
+    const errorMatches = output.matchAll(/(\d+):\d+\s+error\s+(.+?)\s+(\S+)/g);
+    for (const match of errorMatches) {
+      issues.push({
+        type: 'error',
+        description: `${match[3]}: ${match[2]}`,
+        severity: 'major',
+        line: parseInt(match[1], 10),
+      });
+    }
+  }
+
+  return issues;
+}