@tyvm/knowhow 0.0.32 → 0.0.34

package/benchmarks/scripts/build-and-run.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Build and run Knowhow benchmarks
+# Usage: ./build-and-run.sh [command] [options...]
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+echo "🏗️  Building Knowhow benchmark container..."
+
+# Build the Docker container
+docker build -f "$PROJECT_ROOT/benchmarks/docker/Dockerfile" -t knowhow-bench "$PROJECT_ROOT"
+
+echo "✅ Container built successfully!"
+
+# Create results directory if it doesn't exist
+mkdir -p "$PROJECT_ROOT/benchmarks/results"
+
+# If no arguments provided, show usage
+if [ $# -eq 0 ]; then
+    echo ""
+    echo "Usage: $0 <command> [options...]"
+    echo ""
+    echo "Examples:"
+    echo "  $0 setup --language javascript --count 5"
+    echo "  $0 run --language javascript --count 5 --model gpt-4o-mini"
+    echo "  $0 run --language python --count 10 --provider anthropic --model claude-3-sonnet-20240229"
+    echo ""
+    exit 0
+fi
+
+echo "🚀 Running benchmarks..."
+
+# Run the container with all provided arguments
+docker run --rm \
+    -v "$PROJECT_ROOT/benchmarks/results:/app/benchmarks/results" \
+    -e OPENAI_KEY \
+    -e ANTHROPIC_API_KEY \
+    -e GEMINI_API_KEY \
+    -e XAI_API_KEY \
+    --env-file "$PROJECT_ROOT/benchmarks/.env" \
+    knowhow-bench "$@"
+
+echo "✅ Benchmarks completed!"
+echo "📊 Results available in: benchmarks/results/"

package/benchmarks/scripts/clone-exercism.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+# Clone Exercism exercises for benchmarking
+# Based on Aider's clone-exercism.sh approach
+
+set -e
+
+# Configuration
+EXERCISM_REPO="https://github.com/exercism/problem-specifications.git"
+LANGUAGE=${1:-"javascript"}  # Default to JavaScript
+MAX_EXERCISES=${2:-10}       # Default to 10 exercises
+
+# Use different paths for local vs container
+if [ -n "$CONTAINER" ]; then
+    EXERCISES_DIR="/app/exercises"
+else
+    EXERCISES_DIR="$(cd "$(dirname "$0")/.." && pwd)/exercises"
+fi
+
+echo "Cloning Exercism exercises for language: $LANGUAGE"
+echo "Maximum exercises: $MAX_EXERCISES"
+echo "Target directory: $EXERCISES_DIR"
+
+# Create exercises directory if it doesn't exist
+mkdir -p "$EXERCISES_DIR"
+
+# Clone the problem specifications repo if not already cloned
+if [ ! -d "$EXERCISES_DIR/problem-specifications" ]; then
+    echo "Cloning Exercism problem specifications..."
+    cd "$EXERCISES_DIR"
+    git clone "$EXERCISM_REPO" problem-specifications
+fi
+
+# Clone the language track
+LANGUAGE_REPO="https://github.com/exercism/${LANGUAGE}.git"
+LANGUAGE_DIR="$EXERCISES_DIR/$LANGUAGE"
+
+if [ ! -d "$LANGUAGE_DIR" ]; then
+    echo "Cloning $LANGUAGE track..."
+    cd "$EXERCISES_DIR"
+    git clone "$LANGUAGE_REPO" "$LANGUAGE"
+fi
+
+# Find exercises with both problem specification and language implementation
+echo "Finding exercises with both specification and implementation..."
+
+SPEC_DIR="$EXERCISES_DIR/problem-specifications/exercises"
+IMPL_DIR="$LANGUAGE_DIR/exercises"
+
+# Create filtered exercises directory
+FILTERED_DIR="$EXERCISES_DIR/filtered"
+if [ -d "$FILTERED_DIR" ]; then
+    echo "Removing existing filtered directory: $FILTERED_DIR"
+    rm -rf "$FILTERED_DIR"
+fi
+mkdir -p "$FILTERED_DIR"
+
+count=0
+for exercise in $(ls "$SPEC_DIR" 2>/dev/null | sort); do
+    if [ $count -ge $MAX_EXERCISES ]; then
+        break
+    fi
+
+    if [ -d "$IMPL_DIR/practice/$exercise" ] || [ -d "$IMPL_DIR/$exercise" ]; then
+        echo "Found exercise: $exercise"
+
+        # Create exercise directory
+        exercise_dir="$FILTERED_DIR/$exercise"
+        mkdir -p "$exercise_dir"
+
+        # Copy problem specification
+        if [ -f "$SPEC_DIR/$exercise/description.md" ]; then
+            cp "$SPEC_DIR/$exercise/description.md" "$exercise_dir/"
+        fi
+
+        if [ -f "$SPEC_DIR/$exercise/metadata.yml" ]; then
+            cp "$SPEC_DIR/$exercise/metadata.yml" "$exercise_dir/"
+        fi
+
+        # Copy language implementation
+        if [ -d "$IMPL_DIR/practice/$exercise" ]; then
+            cp -r "$IMPL_DIR/practice/$exercise"/* "$exercise_dir/"
+        elif [ -d "$IMPL_DIR/$exercise" ]; then
+            cp -r "$IMPL_DIR/$exercise"/* "$exercise_dir/"
+        fi
+
+        count=$((count + 1))
+    fi
+done
+
+echo "Successfully set up $count exercises in $FILTERED_DIR"
+echo "Ready for benchmarking!"

package/benchmarks/scripts/validate.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Simple validation script to test the benchmark setup
+# This runs without the full Docker setup for quick validation
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BENCHMARK_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+echo "🔍 Validating Knowhow Benchmarks setup..."
+
+# 1. Check that benchmarks can be built
+echo "1. Building benchmarks package..."
+cd "$BENCHMARK_DIR"
+npm run build > /dev/null 2>&1
+echo "   ✅ Build successful"
+
+# 2. Check that tests pass
+echo "2. Running tests..."
+npm test > /dev/null 2>&1
+echo "   ✅ Tests passed"
+
+# 3. Check that CLI can show help
+echo "3. Testing CLI..."
+node dist/cli.js --help > /dev/null 2>&1
+echo "   ✅ CLI working"
+
+# 4. Check that Docker can build (optional - requires Docker)
+if command -v docker &> /dev/null; then
+    echo "4. Testing Docker build..."
+    cd "$(dirname "$BENCHMARK_DIR")"
+    docker build -f benchmarks/docker/Dockerfile -t knowhow-bench-test . > /dev/null 2>&1
+    echo "   ✅ Docker build successful"
+    
+    # Clean up test image
+    docker rmi knowhow-bench-test > /dev/null 2>&1
+else
+    echo "4. Skipping Docker test (Docker not available)"
+fi
+
+echo ""
+echo "🎉 All validations passed!"
+echo ""
+echo "Ready to run benchmarks. Example usage:"
+echo "  ./scripts/build-and-run.sh setup --language javascript --count 5"
+echo "  ./scripts/build-and-run.sh run --language javascript --count 5 --model gpt-4o-mini"
+echo ""

package/benchmarks/src/__tests__/runner.test.ts

@@ -0,0 +1,27 @@
+import { BenchmarkRunner } from '../runner';
+import { BenchmarkConfig } from '../types';
+
+describe('BenchmarkRunner', () => {
+  const mockConfig: BenchmarkConfig = {
+    language: 'javascript',
+    maxExercises: 5,
+    model: 'gpt-4o-mini',
+    provider: 'openai',
+    limits: {
+      maxTurns: 20,
+      maxTime: 300,
+      maxCost: 1.0
+    },
+    outputFile: 'test-results.json'
+  };
+
+  it('should create a BenchmarkRunner instance', () => {
+    const runner = new BenchmarkRunner(mockConfig);
+    expect(runner).toBeInstanceOf(BenchmarkRunner);
+  });
+
+  it('should have the correct configuration', () => {
+    const runner = new BenchmarkRunner(mockConfig);
+    expect(runner['config']).toEqual(mockConfig);
+  });
+});

package/benchmarks/src/cli.ts

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+
+import { Command } from "commander";
+import { BenchmarkRunner } from "./runner";
+import { BenchmarkConfig } from "./types";
+import chalk from "chalk";
+
+const program = new Command();
+
+program
+  .name("knowhow-bench")
+  .description("Benchmark Knowhow terminal agent against coding exercises")
+  .version("0.0.1");
+
+program
+  .command("run")
+  .description("Run benchmarks against Exercism exercises")
+  .option(
+    "-l, --language <language>",
+    "Programming language to test",
+    "javascript"
+  )
+  .option("-c, --count <count>", "Maximum number of exercises to run", "10")
+  .option("-m, --model <model>", "AI model to use", "gpt-4o-mini")
+  .option("-p, --provider <provider>", "AI provider to use", "openai")
+  .option("--max-turns <turns>", "Maximum turns per exercise", "30")
+  .option("--max-time <seconds>", "Maximum time per exercise in seconds", "300")
+  .option("--max-cost <dollars>", "Maximum cost per exercise in dollars", "1.0")
+  .option("--output <file>", "Output file for results", "results.json")
+  .action(async (options) => {
+    try {
+      console.log(chalk.blue("🚀 Starting Knowhow benchmarks..."));
+
+      const config: BenchmarkConfig = {
+        language: options.language,
+        maxExercises: parseInt(options.count),
+        model: options.model,
+        provider: options.provider,
+        limits: {
+          maxTurns: parseInt(options.maxTurns),
+          maxTime: parseInt(options.maxTime),
+          maxCost: parseFloat(options.maxCost),
+        },
+        outputFile: options.output,
+      };
+
+      const runner = new BenchmarkRunner(config);
+      await runner.run();
+
+      console.log(chalk.green("✅ Benchmarks completed successfully!"));
+      process.exit(0);
+    } catch (error) {
+      console.error(chalk.red("❌ Benchmark failed:"), error);
+      process.exit(1);
+    }
+  });
+
+program
+  .command("setup")
+  .description("Set up exercises for benchmarking")
+  .option(
+    "-l, --language <language>",
+    "Programming language to setup",
+    "javascript"
+  )
+  .option("-c, --count <count>", "Maximum number of exercises to setup", "10")
+  .action(async (options) => {
+    try {
+      console.log(chalk.blue("📦 Setting up exercises..."));
+
+      const runner = new BenchmarkRunner({
+        language: options.language,
+        maxExercises: parseInt(options.count),
+        model: "gpt-4o-mini", // Dummy values for setup
+        provider: "openai",
+        limits: { maxTurns: 20, maxTime: 300, maxCost: 1.0 },
+        outputFile: "results.json",
+      });
+
+      await runner.setupExercises();
+
+      console.log(chalk.green("✅ Exercises setup completed!"));
+      process.exit(0);
+    } catch (error) {
+      console.error(chalk.red("❌ Setup failed:"), error);
+      process.exit(1);
+    }
+  });
+
+program.parse();

package/benchmarks/src/evaluators/EvaluatorRegistry.ts

@@ -0,0 +1,64 @@
+import { ExerciseEvaluator, TestResult, TestEvaluationResult } from './types';
+import { JavaScriptEvaluator } from './JavaScriptEvaluator';
+
+export class EvaluatorRegistry {
+  private evaluators: ExerciseEvaluator[] = [];
+
+  constructor() {
+    // Register default evaluators
+    this.registerEvaluator(new JavaScriptEvaluator());
+  }
+
+  registerEvaluator(evaluator: ExerciseEvaluator): void {
+    this.evaluators.push(evaluator);
+  }
+
+  evalForExercise(exercisePath: string): ExerciseEvaluator | null {
+    return this.evaluators.find(e => e.canEvaluate(exercisePath)) || null;
+  }
+
+  async evaluateExercise(exercisePath: string, exerciseName: string): Promise<TestEvaluationResult | null> {
+    // Find the first evaluator that can handle this exercise
+    const evaluator = this.evalForExercise(exercisePath);
+
+    if (!evaluator) {
+      console.warn(`No evaluator found for exercise: ${exerciseName} at ${exercisePath}`);
+      return null;
+    }
+
+    try {
+      console.log(`Evaluating ${exerciseName} using ${evaluator.language} evaluator...`);
+      const testResult = await evaluator.evaluate(exercisePath);
+
+      return {
+        exerciseName,
+        testResult,
+        evaluatedBy: evaluator.language
+      };
+    } catch (error) {
+      console.error(`Error evaluating exercise ${exerciseName}:`, error);
+
+      // Return a failed test result instead of null
+      return {
+        exerciseName,
+        testResult: {
+          passed: 0,
+          failed: 0,
+          total: 0,
+          success: false,
+          output: '',
+          errorMessage: `Evaluation failed: ${error instanceof Error ? error.message : String(error)}`
+        },
+        evaluatedBy: evaluator.language
+      };
+    }
+  }
+
+  getAvailableEvaluators(): string[] {
+    return this.evaluators.map(e => e.language);
+  }
+
+  canEvaluateExercise(exercisePath: string): boolean {
+    return this.evaluators.some(e => e.canEvaluate(exercisePath));
+  }
+}

package/benchmarks/src/evaluators/JavaScriptEvaluator.ts

@@ -0,0 +1,183 @@
+import { ExerciseEvaluator, TestResult } from './types';
+import { execSync } from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+
+export class JavaScriptEvaluator implements ExerciseEvaluator {
+  language = 'javascript';
+
+  canEvaluate(exercisePath: string): boolean {
+    // Check for package.json with test script or jest config
+    const packageJsonPath = path.join(exercisePath, 'package.json');
+    
+    if (!fs.existsSync(packageJsonPath)) {
+      return false;
+    }
+
+    try {
+      const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
+      
+      // Check if there's a test script or jest configuration
+      return !!(
+        packageJson.scripts?.test ||
+        packageJson.devDependencies?.jest ||
+        packageJson.dependencies?.jest ||
+        packageJson.jest ||
+        fs.existsSync(path.join(exercisePath, 'jest.config.js')) ||
+        fs.existsSync(path.join(exercisePath, 'jest.config.json'))
+      );
+    } catch (error) {
+      return false;
+    }
+  }
+
+  async evaluate(exercisePath: string): Promise<TestResult> {
+    try {
+      // First try to install dependencies if node_modules doesn't exist
+      const nodeModulesPath = path.join(exercisePath, 'node_modules');
+      if (!fs.existsSync(nodeModulesPath)) {
+        try {
+          execSync('npm install', { 
+            cwd: exercisePath, 
+            stdio: 'pipe',
+            timeout: 60000 // 60 second timeout
+          });
+        } catch (installError) {
+          // Continue anyway, maybe dependencies are not needed
+          console.warn(`Failed to install dependencies in ${exercisePath}:`, installError);
+        }
+      }
+
+      // Try to run tests with JSON output
+      let command = 'npm test';
+      
+      // Check if we can use Jest directly with JSON reporter
+      const packageJsonPath = path.join(exercisePath, 'package.json');
+      if (fs.existsSync(packageJsonPath)) {
+        const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
+        
+        // If jest is available, use it directly with JSON reporter
+        if (packageJson.devDependencies?.jest || packageJson.dependencies?.jest) {
+          command = 'npx jest --json --verbose';
+        } else if (packageJson.scripts?.test) {
+          // Try to modify the test script to include JSON output
+          const testScript = packageJson.scripts.test;
+          if (testScript.includes('jest')) {
+            command = `${testScript} --json --verbose`;
+          }
+        }
+      }
+
+      const output = execSync(command, {
+        cwd: exercisePath,
+        stdio: 'pipe',
+        encoding: 'utf8',
+        timeout: 120000 // 2 minute timeout for tests
+      });
+
+      return this.parseJestOutput(output);
+
+    } catch (error: any) {
+      // Jest exits with non-zero code when tests fail, so we need to parse the output
+      if (error.stdout) {
+        try {
+          return this.parseJestOutput(error.stdout);
+        } catch (parseError) {
+          // If JSON parsing fails, try to extract basic info from text output
+          return this.parseTextOutput(error.stdout || error.stderr || '');
+        }
+      }
+
+      return {
+        passed: 0,
+        failed: 0,
+        total: 0,
+        success: false,
+        output: error.message || 'Test execution failed',
+        errorMessage: error.message,
+        details: error
+      };
+    }
+  }
+
+  private parseJestOutput(output: string): TestResult {
+    try {
+      // Try to find JSON output in the string
+      const lines = output.split('\n');
+      let jsonLine = '';
+      
+      for (const line of lines) {
+        const trimmed = line.trim();
+        if (trimmed.startsWith('{') && (trimmed.includes('"success"') || trimmed.includes('"numTotalTests"'))) {
+          jsonLine = trimmed;
+          break;
+        }
+      }
+
+      if (jsonLine) {
+        const result = JSON.parse(jsonLine);
+        
+        return {
+          passed: result.numPassedTests || 0,
+          failed: result.numFailedTests || 0,
+          total: result.numTotalTests || 0,
+          skipped: result.numPendingTests || 0,
+          success: result.success || false,
+          output: output,
+          details: result
+        };
+      }
+    } catch (error) {
+      // Fall back to text parsing
+    }
+
+    return this.parseTextOutput(output);
+  }
+
+  private parseTextOutput(output: string): TestResult {
+    // Try to parse Jest text output
+    let passed = 0;
+    let failed = 0;
+    let total = 0;
+    let success = false;
+
+    // Look for Jest summary patterns
+    const passedMatch = output.match(/(\d+) passed/);
+    const failedMatch = output.match(/(\d+) failed/);
+    const totalMatch = output.match(/(\d+) total/);
+
+    if (passedMatch) passed = parseInt(passedMatch[1]);
+    if (failedMatch) failed = parseInt(failedMatch[1]);
+    if (totalMatch) total = parseInt(totalMatch[1]);
+
+    // If we couldn't find specific numbers, try other patterns
+    if (total === 0) {
+      // Look for "Tests: " summary
+      const testsMatch = output.match(/Tests:\s+(\d+)\s+failed,\s+(\d+)\s+passed,\s+(\d+)\s+total/);
+      if (testsMatch) {
+        failed = parseInt(testsMatch[1]);
+        passed = parseInt(testsMatch[2]);
+        total = parseInt(testsMatch[3]);
+      } else {
+        // Look for individual test results
+        const testResults = output.match(/✓|✗|PASS|FAIL/g);
+        if (testResults) {
+          total = testResults.length;
+          passed = testResults.filter(r => r === '✓' || r === 'PASS').length;
+          failed = total - passed;
+        }
+      }
+    }
+
+    success = failed === 0 && total > 0;
+
+    return {
+      passed,
+      failed,
+      total,
+      success,
+      output,
+      errorMessage: success ? undefined : 'Some tests failed'
+    };
+  }
+}

package/benchmarks/src/evaluators/index.ts

@@ -0,0 +1,3 @@
+export * from './types';
+export * from './JavaScriptEvaluator';
+export * from './EvaluatorRegistry';

package/benchmarks/src/evaluators/types.ts

@@ -0,0 +1,22 @@
+export interface TestResult {
+  passed: number;
+  failed: number;
+  total: number;
+  skipped?: number;
+  success: boolean;
+  output: string;
+  errorMessage?: string;
+  details?: any; // Raw test runner output
+}
+
+export interface ExerciseEvaluator {
+  language: string;
+  canEvaluate(exercisePath: string): boolean;
+  evaluate(exercisePath: string): Promise<TestResult>;
+}
+
+export interface TestEvaluationResult {
+  exerciseName: string;
+  testResult: TestResult;
+  evaluatedBy: string; // Which evaluator was used
+}

package/benchmarks/src/index.ts

@@ -0,0 +1,3 @@
+export { BenchmarkRunner } from './runner';
+export * from './types';
+import 'dotenv/config'

package/benchmarks/src/providers.ts

@@ -0,0 +1,13 @@
+import { AIClient, HttpClient } from "../../ts_build/src/clients";
+
+export async function registerProvider(
+  provider: string,
+  url: string,
+  headers: Record<string, string>,
+  clients: AIClient
+): Promise<void> {
+  const client = new HttpClient(url, headers);
+
+  clients.registerClient(provider, client);
+  await clients.loadProviderModels(provider);
+}