@tyvm/knowhow 0.0.90 → 0.0.92

package/benchmarks/scripts/build-and-run.sh

@@ -1,47 +0,0 @@
-#!/bin/bash
-
-# Build and run Knowhow benchmarks
-# Usage: ./build-and-run.sh [command] [options...]
-
-set -e
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
-
-echo "🏗️  Building Knowhow benchmark container..."
-
-# Build the Docker container
-docker build -f "$PROJECT_ROOT/benchmarks/docker/Dockerfile" -t knowhow-bench "$PROJECT_ROOT"
-
-echo "✅ Container built successfully!"
-
-# Create results directory if it doesn't exist
-mkdir -p "$PROJECT_ROOT/benchmarks/results"
-
-# If no arguments provided, show usage
-if [ $# -eq 0 ]; then
-    echo ""
-    echo "Usage: $0 <command> [options...]"
-    echo ""
-    echo "Examples:"
-    echo "  $0 setup --language javascript --count 5"
-    echo "  $0 run --language javascript --count 5 --model gpt-4o-mini"
-    echo "  $0 run --language python --count 10 --provider anthropic --model claude-3-sonnet-20240229"
-    echo ""
-    exit 0
-fi
-
-echo "🚀 Running benchmarks..."
-
-# Run the container with all provided arguments
-docker run --rm \
-    -v "$PROJECT_ROOT/benchmarks/results:/app/benchmarks/results" \
-    -e OPENAI_KEY \
-    -e ANTHROPIC_API_KEY \
-    -e GEMINI_API_KEY \
-    -e XAI_API_KEY \
-    --env-file "$PROJECT_ROOT/benchmarks/.env" \
-    knowhow-bench "$@"
-
-echo "✅ Benchmarks completed!"
-echo "📊 Results available in: benchmarks/results/"

package/benchmarks/scripts/clone-exercism.sh

@@ -1,92 +0,0 @@
-#!/bin/bash
-
-# Clone Exercism exercises for benchmarking
-# Based on Aider's clone-exercism.sh approach
-
-set -e
-
-# Configuration
-EXERCISM_REPO="https://github.com/exercism/problem-specifications.git"
-LANGUAGE=${1:-"javascript"}  # Default to JavaScript
-MAX_EXERCISES=${2:-10}       # Default to 10 exercises
-
-# Use different paths for local vs container
-if [ -n "$CONTAINER" ]; then
-    EXERCISES_DIR="/app/exercises"
-else
-    EXERCISES_DIR="$(cd "$(dirname "$0")/.." && pwd)/exercises"
-fi
-
-echo "Cloning Exercism exercises for language: $LANGUAGE"
-echo "Maximum exercises: $MAX_EXERCISES"
-echo "Target directory: $EXERCISES_DIR"
-
-# Create exercises directory if it doesn't exist
-mkdir -p "$EXERCISES_DIR"
-
-# Clone the problem specifications repo if not already cloned
-if [ ! -d "$EXERCISES_DIR/problem-specifications" ]; then
-    echo "Cloning Exercism problem specifications..."
-    cd "$EXERCISES_DIR"
-    git clone "$EXERCISM_REPO" problem-specifications
-fi
-
-# Clone the language track
-LANGUAGE_REPO="https://github.com/exercism/${LANGUAGE}.git"
-LANGUAGE_DIR="$EXERCISES_DIR/$LANGUAGE"
-
-if [ ! -d "$LANGUAGE_DIR" ]; then
-    echo "Cloning $LANGUAGE track..."
-    cd "$EXERCISES_DIR"
-    git clone "$LANGUAGE_REPO" "$LANGUAGE"
-fi
-
-# Find exercises with both problem specification and language implementation
-echo "Finding exercises with both specification and implementation..."
-
-SPEC_DIR="$EXERCISES_DIR/problem-specifications/exercises"
-IMPL_DIR="$LANGUAGE_DIR/exercises"
-
-# Create filtered exercises directory
-FILTERED_DIR="$EXERCISES_DIR/filtered"
-if [ -d "$FILTERED_DIR" ]; then
-    echo "Removing existing filtered directory: $FILTERED_DIR"
-    rm -rf "$FILTERED_DIR"
-fi
-mkdir -p "$FILTERED_DIR"
-
-count=0
-for exercise in $(ls "$SPEC_DIR" 2>/dev/null | sort); do
-    if [ $count -ge $MAX_EXERCISES ]; then
-        break
-    fi
-
-    if [ -d "$IMPL_DIR/practice/$exercise" ] || [ -d "$IMPL_DIR/$exercise" ]; then
-        echo "Found exercise: $exercise"
-
-        # Create exercise directory
-        exercise_dir="$FILTERED_DIR/$exercise"
-        mkdir -p "$exercise_dir"
-
-        # Copy problem specification
-        if [ -f "$SPEC_DIR/$exercise/description.md" ]; then
-            cp "$SPEC_DIR/$exercise/description.md" "$exercise_dir/"
-        fi
-
-        if [ -f "$SPEC_DIR/$exercise/metadata.yml" ]; then
-            cp "$SPEC_DIR/$exercise/metadata.yml" "$exercise_dir/"
-        fi
-
-        # Copy language implementation
-        if [ -d "$IMPL_DIR/practice/$exercise" ]; then
-            cp -r "$IMPL_DIR/practice/$exercise"/* "$exercise_dir/"
-        elif [ -d "$IMPL_DIR/$exercise" ]; then
-            cp -r "$IMPL_DIR/$exercise"/* "$exercise_dir/"
-        fi
-
-        count=$((count + 1))
-    fi
-done
-
-echo "Successfully set up $count exercises in $FILTERED_DIR"
-echo "Ready for benchmarking!"

package/benchmarks/scripts/validate.sh

@@ -1,48 +0,0 @@
-#!/bin/bash
-
-# Simple validation script to test the benchmark setup
-# This runs without the full Docker setup for quick validation
-
-set -e
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-BENCHMARK_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
-
-echo "🔍 Validating Knowhow Benchmarks setup..."
-
-# 1. Check that benchmarks can be built
-echo "1. Building benchmarks package..."
-cd "$BENCHMARK_DIR"
-npm run build > /dev/null 2>&1
-echo "   ✅ Build successful"
-
-# 2. Check that tests pass
-echo "2. Running tests..."
-npm test > /dev/null 2>&1
-echo "   ✅ Tests passed"
-
-# 3. Check that CLI can show help
-echo "3. Testing CLI..."
-node dist/cli.js --help > /dev/null 2>&1
-echo "   ✅ CLI working"
-
-# 4. Check that Docker can build (optional - requires Docker)
-if command -v docker &> /dev/null; then
-    echo "4. Testing Docker build..."
-    cd "$(dirname "$BENCHMARK_DIR")"
-    docker build -f benchmarks/docker/Dockerfile -t knowhow-bench-test . > /dev/null 2>&1
-    echo "   ✅ Docker build successful"
-    
-    # Clean up test image
-    docker rmi knowhow-bench-test > /dev/null 2>&1
-else
-    echo "4. Skipping Docker test (Docker not available)"
-fi
-
-echo ""
-echo "🎉 All validations passed!"
-echo ""
-echo "Ready to run benchmarks. Example usage:"
-echo "  ./scripts/build-and-run.sh setup --language javascript --count 5"
-echo "  ./scripts/build-and-run.sh run --language javascript --count 5 --model gpt-4o-mini"
-echo ""

package/benchmarks/src/__tests__/runner.test.ts

@@ -1,27 +0,0 @@
-import { BenchmarkRunner } from '../runner';
-import { BenchmarkConfig } from '../types';
-
-describe('BenchmarkRunner', () => {
-  const mockConfig: BenchmarkConfig = {
-    language: 'javascript',
-    maxExercises: 5,
-    model: 'gpt-4o-mini',
-    provider: 'openai',
-    limits: {
-      maxTurns: 20,
-      maxTime: 300,
-      maxCost: 1.0
-    },
-    outputFile: 'test-results.json'
-  };
-
-  it('should create a BenchmarkRunner instance', () => {
-    const runner = new BenchmarkRunner(mockConfig);
-    expect(runner).toBeInstanceOf(BenchmarkRunner);
-  });
-
-  it('should have the correct configuration', () => {
-    const runner = new BenchmarkRunner(mockConfig);
-    expect(runner['config']).toEqual(mockConfig);
-  });
-});

package/benchmarks/src/cli.ts

@@ -1,90 +0,0 @@
-#!/usr/bin/env node
-
-import { Command } from "commander";
-import { BenchmarkRunner } from "./runner";
-import { BenchmarkConfig } from "./types";
-import chalk from "chalk";
-
-const program = new Command();
-
-program
-  .name("knowhow-bench")
-  .description("Benchmark Knowhow terminal agent against coding exercises")
-  .version("0.0.1");
-
-program
-  .command("run")
-  .description("Run benchmarks against Exercism exercises")
-  .option(
-    "-l, --language <language>",
-    "Programming language to test",
-    "javascript"
-  )
-  .option("-c, --count <count>", "Maximum number of exercises to run", "10")
-  .option("-m, --model <model>", "AI model to use", "gpt-4o-mini")
-  .option("-p, --provider <provider>", "AI provider to use", "openai")
-  .option("--max-turns <turns>", "Maximum turns per exercise", "30")
-  .option("--max-time <seconds>", "Maximum time per exercise in seconds", "300")
-  .option("--max-cost <dollars>", "Maximum cost per exercise in dollars", "1.0")
-  .option("--output <file>", "Output file for results", "results.json")
-  .action(async (options) => {
-    try {
-      console.log(chalk.blue("🚀 Starting Knowhow benchmarks..."));
-
-      const config: BenchmarkConfig = {
-        language: options.language,
-        maxExercises: parseInt(options.count),
-        model: options.model,
-        provider: options.provider,
-        limits: {
-          maxTurns: parseInt(options.maxTurns),
-          maxTime: parseInt(options.maxTime),
-          maxCost: parseFloat(options.maxCost),
-        },
-        outputFile: options.output,
-      };
-
-      const runner = new BenchmarkRunner(config);
-      await runner.run();
-
-      console.log(chalk.green("✅ Benchmarks completed successfully!"));
-      process.exit(0);
-    } catch (error) {
-      console.error(chalk.red("❌ Benchmark failed:"), error);
-      process.exit(1);
-    }
-  });
-
-program
-  .command("setup")
-  .description("Set up exercises for benchmarking")
-  .option(
-    "-l, --language <language>",
-    "Programming language to setup",
-    "javascript"
-  )
-  .option("-c, --count <count>", "Maximum number of exercises to setup", "10")
-  .action(async (options) => {
-    try {
-      console.log(chalk.blue("📦 Setting up exercises..."));
-
-      const runner = new BenchmarkRunner({
-        language: options.language,
-        maxExercises: parseInt(options.count),
-        model: "gpt-4o-mini", // Dummy values for setup
-        provider: "openai",
-        limits: { maxTurns: 20, maxTime: 300, maxCost: 1.0 },
-        outputFile: "results.json",
-      });
-
-      await runner.setupExercises();
-
-      console.log(chalk.green("✅ Exercises setup completed!"));
-      process.exit(0);
-    } catch (error) {
-      console.error(chalk.red("❌ Setup failed:"), error);
-      process.exit(1);
-    }
-  });
-
-program.parse();

package/benchmarks/src/evaluators/EvaluatorRegistry.ts

@@ -1,64 +0,0 @@
-import { ExerciseEvaluator, TestResult, TestEvaluationResult } from './types';
-import { JavaScriptEvaluator } from './JavaScriptEvaluator';
-
-export class EvaluatorRegistry {
-  private evaluators: ExerciseEvaluator[] = [];
-
-  constructor() {
-    // Register default evaluators
-    this.registerEvaluator(new JavaScriptEvaluator());
-  }
-
-  registerEvaluator(evaluator: ExerciseEvaluator): void {
-    this.evaluators.push(evaluator);
-  }
-
-  evalForExercise(exercisePath: string): ExerciseEvaluator | null {
-    return this.evaluators.find(e => e.canEvaluate(exercisePath)) || null;
-  }
-
-  async evaluateExercise(exercisePath: string, exerciseName: string): Promise<TestEvaluationResult | null> {
-    // Find the first evaluator that can handle this exercise
-    const evaluator = this.evalForExercise(exercisePath);
-
-    if (!evaluator) {
-      console.warn(`No evaluator found for exercise: ${exerciseName} at ${exercisePath}`);
-      return null;
-    }
-
-    try {
-      console.log(`Evaluating ${exerciseName} using ${evaluator.language} evaluator...`);
-      const testResult = await evaluator.evaluate(exercisePath);
-
-      return {
-        exerciseName,
-        testResult,
-        evaluatedBy: evaluator.language
-      };
-    } catch (error) {
-      console.error(`Error evaluating exercise ${exerciseName}:`, error);
-
-      // Return a failed test result instead of null
-      return {
-        exerciseName,
-        testResult: {
-          passed: 0,
-          failed: 0,
-          total: 0,
-          success: false,
-          output: '',
-          errorMessage: `Evaluation failed: ${error instanceof Error ? error.message : String(error)}`
-        },
-        evaluatedBy: evaluator.language
-      };
-    }
-  }
-
-  getAvailableEvaluators(): string[] {
-    return this.evaluators.map(e => e.language);
-  }
-
-  canEvaluateExercise(exercisePath: string): boolean {
-    return this.evaluators.some(e => e.canEvaluate(exercisePath));
-  }
-}

package/benchmarks/src/evaluators/JavaScriptEvaluator.ts

@@ -1,183 +0,0 @@
-import { ExerciseEvaluator, TestResult } from './types';
-import { execSync } from 'child_process';
-import * as fs from 'fs';
-import * as path from 'path';
-
-export class JavaScriptEvaluator implements ExerciseEvaluator {
-  language = 'javascript';
-
-  canEvaluate(exercisePath: string): boolean {
-    // Check for package.json with test script or jest config
-    const packageJsonPath = path.join(exercisePath, 'package.json');
-    
-    if (!fs.existsSync(packageJsonPath)) {
-      return false;
-    }
-
-    try {
-      const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
-      
-      // Check if there's a test script or jest configuration
-      return !!(
-        packageJson.scripts?.test ||
-        packageJson.devDependencies?.jest ||
-        packageJson.dependencies?.jest ||
-        packageJson.jest ||
-        fs.existsSync(path.join(exercisePath, 'jest.config.js')) ||
-        fs.existsSync(path.join(exercisePath, 'jest.config.json'))
-      );
-    } catch (error) {
-      return false;
-    }
-  }
-
-  async evaluate(exercisePath: string): Promise<TestResult> {
-    try {
-      // First try to install dependencies if node_modules doesn't exist
-      const nodeModulesPath = path.join(exercisePath, 'node_modules');
-      if (!fs.existsSync(nodeModulesPath)) {
-        try {
-          execSync('npm install', { 
-            cwd: exercisePath, 
-            stdio: 'pipe',
-            timeout: 60000 // 60 second timeout
-          });
-        } catch (installError) {
-          // Continue anyway, maybe dependencies are not needed
-          console.warn(`Failed to install dependencies in ${exercisePath}:`, installError);
-        }
-      }
-
-      // Try to run tests with JSON output
-      let command = 'npm test';
-      
-      // Check if we can use Jest directly with JSON reporter
-      const packageJsonPath = path.join(exercisePath, 'package.json');
-      if (fs.existsSync(packageJsonPath)) {
-        const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
-        
-        // If jest is available, use it directly with JSON reporter
-        if (packageJson.devDependencies?.jest || packageJson.dependencies?.jest) {
-          command = 'npx jest --json --verbose';
-        } else if (packageJson.scripts?.test) {
-          // Try to modify the test script to include JSON output
-          const testScript = packageJson.scripts.test;
-          if (testScript.includes('jest')) {
-            command = `${testScript} --json --verbose`;
-          }
-        }
-      }
-
-      const output = execSync(command, {
-        cwd: exercisePath,
-        stdio: 'pipe',
-        encoding: 'utf8',
-        timeout: 120000 // 2 minute timeout for tests
-      });
-
-      return this.parseJestOutput(output);
-
-    } catch (error: any) {
-      // Jest exits with non-zero code when tests fail, so we need to parse the output
-      if (error.stdout) {
-        try {
-          return this.parseJestOutput(error.stdout);
-        } catch (parseError) {
-          // If JSON parsing fails, try to extract basic info from text output
-          return this.parseTextOutput(error.stdout || error.stderr || '');
-        }
-      }
-
-      return {
-        passed: 0,
-        failed: 0,
-        total: 0,
-        success: false,
-        output: error.message || 'Test execution failed',
-        errorMessage: error.message,
-        details: error
-      };
-    }
-  }
-
-  private parseJestOutput(output: string): TestResult {
-    try {
-      // Try to find JSON output in the string
-      const lines = output.split('\n');
-      let jsonLine = '';
-      
-      for (const line of lines) {
-        const trimmed = line.trim();
-        if (trimmed.startsWith('{') && (trimmed.includes('"success"') || trimmed.includes('"numTotalTests"'))) {
-          jsonLine = trimmed;
-          break;
-        }
-      }
-
-      if (jsonLine) {
-        const result = JSON.parse(jsonLine);
-        
-        return {
-          passed: result.numPassedTests || 0,
-          failed: result.numFailedTests || 0,
-          total: result.numTotalTests || 0,
-          skipped: result.numPendingTests || 0,
-          success: result.success || false,
-          output: output,
-          details: result
-        };
-      }
-    } catch (error) {
-      // Fall back to text parsing
-    }
-
-    return this.parseTextOutput(output);
-  }
-
-  private parseTextOutput(output: string): TestResult {
-    // Try to parse Jest text output
-    let passed = 0;
-    let failed = 0;
-    let total = 0;
-    let success = false;
-
-    // Look for Jest summary patterns
-    const passedMatch = output.match(/(\d+) passed/);
-    const failedMatch = output.match(/(\d+) failed/);
-    const totalMatch = output.match(/(\d+) total/);
-
-    if (passedMatch) passed = parseInt(passedMatch[1]);
-    if (failedMatch) failed = parseInt(failedMatch[1]);
-    if (totalMatch) total = parseInt(totalMatch[1]);
-
-    // If we couldn't find specific numbers, try other patterns
-    if (total === 0) {
-      // Look for "Tests: " summary
-      const testsMatch = output.match(/Tests:\s+(\d+)\s+failed,\s+(\d+)\s+passed,\s+(\d+)\s+total/);
-      if (testsMatch) {
-        failed = parseInt(testsMatch[1]);
-        passed = parseInt(testsMatch[2]);
-        total = parseInt(testsMatch[3]);
-      } else {
-        // Look for individual test results
-        const testResults = output.match(/✓|✗|PASS|FAIL/g);
-        if (testResults) {
-          total = testResults.length;
-          passed = testResults.filter(r => r === '✓' || r === 'PASS').length;
-          failed = total - passed;
-        }
-      }
-    }
-
-    success = failed === 0 && total > 0;
-
-    return {
-      passed,
-      failed,
-      total,
-      success,
-      output,
-      errorMessage: success ? undefined : 'Some tests failed'
-    };
-  }
-}

package/benchmarks/src/evaluators/index.ts

@@ -1,3 +0,0 @@
-export * from './types';
-export * from './JavaScriptEvaluator';
-export * from './EvaluatorRegistry';

package/benchmarks/src/evaluators/types.ts

@@ -1,22 +0,0 @@
-export interface TestResult {
-  passed: number;
-  failed: number;
-  total: number;
-  skipped?: number;
-  success: boolean;
-  output: string;
-  errorMessage?: string;
-  details?: any; // Raw test runner output
-}
-
-export interface ExerciseEvaluator {
-  language: string;
-  canEvaluate(exercisePath: string): boolean;
-  evaluate(exercisePath: string): Promise<TestResult>;
-}
-
-export interface TestEvaluationResult {
-  exerciseName: string;
-  testResult: TestResult;
-  evaluatedBy: string; // Which evaluator was used
-}

package/benchmarks/src/index.ts

@@ -1,3 +0,0 @@
-export { BenchmarkRunner } from './runner';
-export * from './types';
-import 'dotenv/config'

package/benchmarks/src/providers.ts

@@ -1,13 +0,0 @@
-import { AIClient, HttpClient } from "../../ts_build/src/clients";
-
-export async function registerProvider(
-  provider: string,
-  url: string,
-  headers: Record<string, string>,
-  clients: AIClient
-): Promise<void> {
-  const client = new HttpClient(url, headers);
-
-  clients.registerClient(provider, client);
-  await clients.loadProviderModels(provider);
-}