outcome-cli 1.0.0

package/src/eval/code-validators.ts

@@ -0,0 +1,414 @@
+/**
+ * Code Validators - SWE-bench style validation for code generation tasks
+ *
+ * Integrates software engineering evaluation into our existing validator system.
+ * These validators enable battles between code-focused agents (like Codeon).
+ *
+ * @module eval/code-validators
+ */
+
+import type { ValidationResult } from './validators.js';
+
+/**
+ * Code artifact content for SWE-bench style evaluation.
+ * Extends base artifact pattern to support code generation tasks.
+ */
+export interface CodeArtifactContent {
+  /** The generated code/patch */
+  code: string;
+  /** Language of the code (python, typescript, etc.) */
+  language: string;
+  /** The issue/problem description that was given */
+  issueDescription: string;
+  /** Optional: File paths that were modified */
+  modifiedFiles?: string[];
+  /** Optional: Explanation of the fix */
+  explanation?: string;
+}
+
+/**
+ * Test case for code evaluation.
+ */
+export interface TestCase {
+  /** Test name/description */
+  name: string;
+  /** Input to the function/code */
+  input: unknown;
+  /** Expected output */
+  expectedOutput: unknown;
+  /** Optional: Test type (unit, integration, edge-case) */
+  type?: 'unit' | 'integration' | 'edge-case';
+}
+
+/**
+ * Validates that generated code contains required syntax elements.
+ * 
+ * @param code - The generated code to validate
+ * @param requiredElements - Array of syntax elements that must be present
+ * @returns ValidationResult indicating if all required elements are present
+ *
+ * @example
+ * validateCodeSyntax("def foo(): return 42", ["def", "return"])
+ * // { valid: true, errors: [] }
+ */
+export function validateCodeSyntax(
+  code: string,
+  requiredElements: string[]
+): ValidationResult {
+  const missingElements = requiredElements.filter(
+    (element) => !code.includes(element)
+  );
+
+  if (missingElements.length === 0) {
+    return { valid: true, errors: [] };
+  }
+
+  return {
+    valid: false,
+    errors: [
+      `Missing required syntax elements: ${missingElements.join(', ')}`,
+    ],
+  };
+}
+
+/**
+ * Validates that code follows basic structure requirements.
+ * Checks for proper function/class definitions, imports, etc.
+ *
+ * @param code - The generated code
+ * @param language - Programming language
+ * @param requirements - Structure requirements to check
+ * @returns ValidationResult
+ */
+export function validateCodeStructure(
+  code: string,
+  language: string,
+  requirements: {
+    mustHaveFunction?: boolean;
+    mustHaveClass?: boolean;
+    mustHaveImports?: boolean;
+    minLines?: number;
+    maxLines?: number;
+  }
+): ValidationResult {
+  const errors: string[] = [];
+  const lines = code.split('\n').filter((line) => line.trim().length > 0);
+
+  // Language-specific patterns
+  const patterns: Record<string, { function: RegExp; class: RegExp; import: RegExp }> = {
+    python: {
+      function: /def\s+\w+\s*\(/,
+      class: /class\s+\w+/,
+      import: /^(import|from)\s+/m,
+    },
+    typescript: {
+      function: /(function\s+\w+|const\s+\w+\s*=\s*(async\s*)?\(|=>\s*{)/,
+      class: /class\s+\w+/,
+      import: /^import\s+/m,
+    },
+    javascript: {
+      function: /(function\s+\w+|const\s+\w+\s*=\s*(async\s*)?\(|=>\s*{)/,
+      class: /class\s+\w+/,
+      import: /^(import|require)\s*/m,
+    },
+  };
+
+  const langPatterns = patterns[language] || patterns.javascript;
+
+  if (requirements.mustHaveFunction && !langPatterns.function.test(code)) {
+    errors.push(`Code must contain a function definition`);
+  }
+
+  if (requirements.mustHaveClass && !langPatterns.class.test(code)) {
+    errors.push(`Code must contain a class definition`);
+  }
+
+  if (requirements.mustHaveImports && !langPatterns.import.test(code)) {
+    errors.push(`Code must contain import statements`);
+  }
+
+  if (requirements.minLines && lines.length < requirements.minLines) {
+    errors.push(`Code too short: ${lines.length} lines, minimum ${requirements.minLines}`);
+  }
+
+  if (requirements.maxLines && lines.length > requirements.maxLines) {
+    errors.push(`Code too long: ${lines.length} lines, maximum ${requirements.maxLines}`);
+  }
+
+  return {
+    valid: errors.length === 0,
+    errors,
+  };
+}
+
+/**
+ * Validates that code produces expected outputs for given test cases.
+ * This is the core SWE-bench style evaluation.
+ *
+ * NOTE: In production, this would execute code in a sandboxed environment.
+ * For now, we do static analysis and pattern matching.
+ *
+ * @param code - The generated code
+ * @param testCases - Array of test cases to validate against
+ * @param options - Evaluation options
+ * @returns ValidationResult with test pass/fail details
+ */
+export function validateTestCases(
+  code: string,
+  testCases: TestCase[],
+  options: {
+    requireAllPass?: boolean;
+    minPassRate?: number;
+  } = {}
+): ValidationResult {
+  const { requireAllPass = true, minPassRate = 1.0 } = options;
+  const errors: string[] = [];
+  let passedCount = 0;
+
+  // Static analysis: Check if code handles the test cases conceptually
+  // In production, this would actually run the code in a sandbox (Docker like SWE-bench)
+  for (const testCase of testCases) {
+    // Basic heuristic: Check if code could plausibly handle this case
+    // Real implementation would execute in Docker container like SWE-bench
+    const hasRelevantLogic = 
+      code.includes('return') || 
+      code.includes('yield') ||
+      code.includes('console.log') ||
+      code.includes('print');
+
+    if (hasRelevantLogic) {
+      passedCount++;
+    } else {
+      errors.push(`Test "${testCase.name}" may fail - no return/output statement found`);
+    }
+  }
+
+  const passRate = testCases.length > 0 ? passedCount / testCases.length : 0;
+
+  if (requireAllPass && passedCount < testCases.length) {
+    return {
+      valid: false,
+      errors: [`Only ${passedCount}/${testCases.length} tests passed. ${errors.join('; ')}`],
+    };
+  }
+
+  if (passRate < minPassRate) {
+    return {
+      valid: false,
+      errors: [`Pass rate ${(passRate * 100).toFixed(1)}% below minimum ${(minPassRate * 100).toFixed(1)}%`],
+    };
+  }
+
+  return { valid: true, errors: [] };
+}
+
+/**
+ * Validates that a patch/diff correctly addresses the issue.
+ * SWE-bench core: Does this patch fix the GitHub issue?
+ *
+ * @param patch - The generated patch/diff
+ * @param issueKeywords - Keywords from the issue that should be addressed
+ * @param options - Validation options
+ * @returns ValidationResult
+ */
+export function validatePatchRelevance(
+  patch: string,
+  issueKeywords: string[],
+  options: {
+    minKeywordsCovered?: number;
+    mustHaveAdditions?: boolean;
+    mustHaveDeletions?: boolean;
+  } = {}
+): ValidationResult {
+  const { 
+    minKeywordsCovered = 1, 
+    mustHaveAdditions = true,
+    mustHaveDeletions = false 
+  } = options;
+  const errors: string[] = [];
+  const patchLower = patch.toLowerCase();
+
+  // Check keyword coverage
+  const coveredKeywords = issueKeywords.filter((kw) =>
+    patchLower.includes(kw.toLowerCase())
+  );
+
+  if (coveredKeywords.length < minKeywordsCovered) {
+    errors.push(
+      `Patch only addresses ${coveredKeywords.length}/${minKeywordsCovered} required keywords`
+    );
+  }
+
+  // Check for additions (lines starting with +)
+  const hasAdditions = /^\+[^+]/m.test(patch);
+  if (mustHaveAdditions && !hasAdditions) {
+    errors.push('Patch must contain code additions');
+  }
+
+  // Check for deletions (lines starting with -)
+  const hasDeletions = /^-[^-]/m.test(patch);
+  if (mustHaveDeletions && !hasDeletions) {
+    errors.push('Patch must contain code deletions');
+  }
+
+  return {
+    valid: errors.length === 0,
+    errors,
+  };
+}
+
+/**
+ * Validates code quality metrics.
+ * Checks for common issues like missing error handling, no comments, etc.
+ *
+ * @param code - The generated code
+ * @param requirements - Quality requirements
+ * @returns ValidationResult
+ */
+export function validateCodeQuality(
+  code: string,
+  requirements: {
+    mustHaveErrorHandling?: boolean;
+    mustHaveComments?: boolean;
+    maxComplexity?: number;
+    noConsoleLog?: boolean;
+  } = {}
+): ValidationResult {
+  const errors: string[] = [];
+
+  // Error handling check
+  if (requirements.mustHaveErrorHandling) {
+    const hasErrorHandling =
+      code.includes('try') ||
+      code.includes('catch') ||
+      code.includes('except') ||
+      code.includes('throw') ||
+      code.includes('raise');
+    
+    if (!hasErrorHandling) {
+      errors.push('Code should include error handling (try/catch/except)');
+    }
+  }
+
+  // Comments check
+  if (requirements.mustHaveComments) {
+    const hasComments =
+      code.includes('//') ||
+      code.includes('/*') ||
+      code.includes('#') ||
+      code.includes('"""') ||
+      code.includes("'''");
+    
+    if (!hasComments) {
+      errors.push('Code should include comments or documentation');
+    }
+  }
+
+  // Console.log check (often unwanted in production code)
+  if (requirements.noConsoleLog) {
+    if (code.includes('console.log') || code.includes('print(')) {
+      errors.push('Code should not contain debug print statements');
+    }
+  }
+
+  // Complexity check (simplified: count control flow statements)
+  if (requirements.maxComplexity) {
+    const controlFlowPatterns = /\b(if|else|for|while|switch|case|try|catch)\b/g;
+    const matches = code.match(controlFlowPatterns) || [];
+    if (matches.length > requirements.maxComplexity) {
+      errors.push(
+        `Code complexity ${matches.length} exceeds maximum ${requirements.maxComplexity}`
+      );
+    }
+  }
+
+  return {
+    valid: errors.length === 0,
+    errors,
+  };
+}
+
+/**
+ * Combined SWE-bench style evaluation.
+ * Runs multiple validators to get a comprehensive score.
+ *
+ * @param artifact - The code artifact to evaluate
+ * @param config - Evaluation configuration
+ * @returns Combined ValidationResult with score
+ */
+export function evaluateSWEBenchStyle(
+  artifact: CodeArtifactContent,
+  config: {
+    issueKeywords: string[];
+    requiredSyntax?: string[];
+    testCases?: TestCase[];
+    structureRequirements?: Parameters<typeof validateCodeStructure>[2];
+    qualityRequirements?: Parameters<typeof validateCodeQuality>[1];
+  }
+): ValidationResult & { score: number; breakdown: Record<string, boolean> } {
+  const breakdown: Record<string, boolean> = {};
+  const allErrors: string[] = [];
+
+  // 1. Syntax validation
+  if (config.requiredSyntax) {
+    const syntaxResult = validateCodeSyntax(artifact.code, config.requiredSyntax);
+    breakdown.syntax = syntaxResult.valid;
+    if (!syntaxResult.valid) allErrors.push(...syntaxResult.errors);
+  } else {
+    breakdown.syntax = true;
+  }
+
+  // 2. Structure validation
+  if (config.structureRequirements) {
+    const structureResult = validateCodeStructure(
+      artifact.code,
+      artifact.language,
+      config.structureRequirements
+    );
+    breakdown.structure = structureResult.valid;
+    if (!structureResult.valid) allErrors.push(...structureResult.errors);
+  } else {
+    breakdown.structure = true;
+  }
+
+  // 3. Test case validation
+  if (config.testCases && config.testCases.length > 0) {
+    const testResult = validateTestCases(artifact.code, config.testCases, {
+      requireAllPass: false,
+      minPassRate: 0.7,
+    });
+    breakdown.tests = testResult.valid;
+    if (!testResult.valid) allErrors.push(...testResult.errors);
+  } else {
+    breakdown.tests = true;
+  }
+
+  // 4. Relevance validation (does it address the issue?)
+  const relevanceResult = validatePatchRelevance(artifact.code, config.issueKeywords, {
+    minKeywordsCovered: Math.min(2, config.issueKeywords.length),
+  });
+  breakdown.relevance = relevanceResult.valid;
+  if (!relevanceResult.valid) allErrors.push(...relevanceResult.errors);
+
+  // 5. Quality validation
+  if (config.qualityRequirements) {
+    const qualityResult = validateCodeQuality(artifact.code, config.qualityRequirements);
+    breakdown.quality = qualityResult.valid;
+    if (!qualityResult.valid) allErrors.push(...qualityResult.errors);
+  } else {
+    breakdown.quality = true;
+  }
+
+  // Calculate score (weighted average)
+  const weights = { syntax: 0.15, structure: 0.15, tests: 0.35, relevance: 0.25, quality: 0.10 };
+  const score = Object.entries(breakdown).reduce((acc, [key, passed]) => {
+    return acc + (passed ? weights[key as keyof typeof weights] || 0 : 0);
+  }, 0);
+
+  return {
+    valid: allErrors.length === 0,
+    errors: allErrors,
+    score,
+    breakdown,
+  };
+}