@tyvm/knowhow 0.0.33 → 0.0.35

package/benchmarks/src/types.ts

@@ -0,0 +1,63 @@
+import { TestResult } from './evaluators/types';
+
+export interface BenchmarkConfig {
+  language: string;
+  maxExercises: number;
+  model: string;
+  provider: string;
+  agent?: string; // Agent type to use (default: 'Patcher')
+  limits: BenchmarkLimits;
+  outputFile: string;
+}
+
+export interface BenchmarkLimits {
+  maxTurns: number;
+  maxTime: number; // in seconds
+  maxCost: number; // in dollars
+}
+
+export interface ExerciseResult {
+  exerciseName: string;
+  status: 'success' | 'failure' | 'timeout' | 'cost_limit' | 'turn_limit';
+  testResult?: TestResult; // Actual test execution results
+  turns: number;
+  timeElapsed: number; // in seconds
+  cost: number; // in dollars
+  startTime: Date;
+  endTime: Date;
+  errorMessage?: string;
+  finalOutput?: string;
+}
+
+export interface BenchmarkResults {
+  config: BenchmarkConfig;
+  exercises: ExerciseResult[];
+  summary: {
+    totalExercises: number;
+    testableExercises: number; // Exercises that had evaluatable tests
+    testsPassedCount: number; // Exercises where all tests passed
+    testsFailedCount: number; // Exercises where some tests failed
+    testPassRate: number; // Percentage of testable exercises where tests passed
+    agentSuccessRate: number; // Original success rate (agent thinks it succeeded)
+    successCount: number;
+    failureCount: number;
+    timeoutCount: number;
+    costLimitCount: number;
+    turnLimitCount: number;
+    totalTime: number;
+    totalCost: number;
+    averageTurns: number;
+    averageTime: number;
+    successRate: number;
+  };
+  startTime: Date;
+  endTime: Date;
+}
+
+export interface Exercise {
+  name: string;
+  path: string;
+  description?: string;
+  hasTests: boolean;
+  files: string[];
+}

package/benchmarks/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "commonjs",
+    "lib": ["ES2020"],
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "**/*.test.ts"]
+}

package/jest.config.js

@@ -14,5 +14,6 @@ module.exports = {
   testEnvironment: 'node',
   testRegex: '/tests/.*\.(test|spec)?\.(ts|tsx|js)$',
   moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
-  modulePathIgnorePatterns: ["ts_build"]
+  modulePathIgnorePatterns: ["ts_build", "benchmarks"],
+  testPathIgnorePatterns: ["<rootDir>/benchmarks/"]
 };

package/leaderboard/README.md

@@ -0,0 +1,148 @@
+# Benchmark Results Leaderboard
+
+A Next.js application to display and analyze benchmark results from coding exercise evaluations.
+
+## Features
+
+- **Interactive Leaderboard**: Sortable table showing model performance metrics
+- **Data Visualization**: Charts comparing success rates and cost vs performance
+- **Model Comparison**: Detailed statistics for each model/provider/language combination
+- **Responsive Design**: Works on desktop and mobile devices
+- **Real-time Data**: Automatically loads latest benchmark results
+
+## Getting Started
+
+### Prerequisites
+
+- Node.js 18+ 
+- npm or yarn
+
+### Installation
+
+1. Install dependencies:
+```bash
+npm install
+```
+
+2. Run the development server:
+```bash
+npm run dev
+```
+
+3. Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+### Data Source
+
+The application reads benchmark results from `../benchmarks/results/results.json`. Make sure to run benchmarks first to generate data.
+
+Expected file structure:
+```
+benchmarks/
+  results/
+    results.json    # Main results file
+    # Additional result files can be added here
+```
+
+## Project Structure
+
+```
+leaderboard/
+├── app/
+│   ├── globals.css       # Global styles
+│   ├── layout.tsx        # Root layout
+│   └── page.tsx          # Main page
+├── components/
+│   ├── LeaderboardTable.tsx    # Sortable results table
+│   └── PerformanceChart.tsx     # Data visualization
+├── types/
+│   └── benchmark.ts      # TypeScript interfaces
+├── utils/
+│   └── dataProcessor.ts  # Data loading and aggregation
+└── package.json
+```
+
+## Available Scripts
+
+- `npm run dev` - Start development server
+- `npm run build` - Build for production
+- `npm run start` - Start production server
+- `npm run lint` - Run ESLint
+
+## Metrics Displayed
+
+- **Success Rate**: Percentage of exercises completed successfully
+- **Total Exercises**: Number of exercises attempted
+- **Average Cost**: Mean cost per exercise in USD
+- **Average Time**: Mean time per exercise in seconds
+- **Average Turns**: Mean number of agent turns per exercise
+- **Total Runs**: Number of benchmark runs for this model/language
+
+## Charts
+
+1. **Success Rate Comparison**: Bar chart showing top 10 models by success rate
+2. **Cost vs Performance**: Scatter plot comparing cost efficiency vs success rate
+
+## Customization
+
+### Adding New Data Sources
+
+Modify `utils/dataProcessor.ts` to load additional result files or change the aggregation logic.
+
+### Styling
+
+The application uses Tailwind CSS. Modify component styles directly in the JSX files or update `globals.css` for global changes.
+
+### Adding New Charts
+
+Use the Recharts library to create additional visualizations in `components/PerformanceChart.tsx`.
+
+## Deployment
+
+### Build for Production
+
+```bash
+npm run build
+npm run start
+```
+
+### Deploy to Vercel
+
+The easiest way to deploy is using the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme).
+
+## Data Format
+
+The application expects benchmark results in this format:
+
+```json
+{
+  "config": {
+    "language": "javascript",
+    "model": "claude-sonnet-4",
+    "provider": "openai",
+    "maxExercises": 1,
+    "limits": {
+      "maxTurns": 20,
+      "maxTime": 300,
+      "maxCost": 1
+    }
+  },
+  "exercises": [
+    {
+      "exerciseName": "accumulate",
+      "status": "success",
+      "turns": 1,
+      "timeElapsed": 46.668,
+      "cost": 0.090424,
+      "startTime": "2025-08-02T07:26:04.029Z",
+      "endTime": "2025-08-02T07:26:50.697Z"
+    }
+  ],
+  "summary": {
+    "totalExercises": 1,
+    "successCount": 1,
+    "totalTime": 46.668,
+    "totalCost": 0.090424,
+    "successRate": 1
+  }
+}
+```

package/leaderboard/app/api/benchmark-data/route.ts

@@ -0,0 +1,131 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { BenchmarkResults, LeaderboardEntry } from '@/types/benchmark';
+import fs from 'fs';
+import path from 'path';
+
+export async function GET(request: NextRequest) {
+  try {
+    const results = await loadAllBenchmarkResults();
+
+    const leaderboardData = aggregateResults(results);
+    return NextResponse.json(leaderboardData);
+  } catch (error) {
+    console.error('Error loading benchmark results:', error);
+    
+    // Return mock data for development
+    const mockData: LeaderboardEntry[] = [
+      {
+        model: 'sample-model',
+        provider: 'sample-provider',
+        language: 'javascript',
+        successRate: 85.5,
+        totalExercises: 6,
+        averageCost: 0.05,
+        averageTime: 145.2,
+        averageTurns: 12.4,
+        totalRuns: 1,
+        lastRun: new Date().toISOString()
+      }
+    ];
+    
+    return NextResponse.json(mockData);
+  }
+}
+
+// Recursive function to find JSON files in nested directories
+function findBenchmarkFiles(dir: string): string[] {
+  const files: string[] = [];
+  
+  try {
+    const items = fs.readdirSync(dir, { withFileTypes: true });
+    
+    for (const item of items) {
+      const fullPath = path.join(dir, item.name);
+      
+      if (item.isDirectory()) {
+        // Recursively search subdirectories
+        files.push(...findBenchmarkFiles(fullPath));
+      } else if (item.isFile() && item.name.endsWith('.json')) {
+        // Add JSON files to our list
+        files.push(fullPath);
+      }
+    }
+  } catch (error) {
+    // Ignore directories we can't read
+  }
+  
+  return files;
+}
+
+async function loadAllBenchmarkResults(): Promise<BenchmarkResults[]> {
+  const resultsPath = path.join(process.cwd(), '..', 'benchmarks', 'results');
+  const results: BenchmarkResults[] = [];
+
+  if (!fs.existsSync(resultsPath)) {
+    console.warn('Benchmark results directory not found:', resultsPath);
+    return results;
+  }
+
+  // Find all JSON files recursively - handles both old and new file structures
+  const allFiles = findBenchmarkFiles(resultsPath);
+  
+  for (const filePath of allFiles) {
+    try {
+      const data = fs.readFileSync(filePath, 'utf8');
+      const parsed = JSON.parse(data);
+      
+      // Validate that this is a valid benchmark result
+      if (parsed.config && parsed.summary && parsed.exercises) {
+        results.push(parsed);
+      }
+    } catch (error) {
+      console.error(`Error loading result file ${filePath}:`, error);
+    }
+  }
+
+  return results;
+}
+
+function aggregateResults(results: BenchmarkResults[]): LeaderboardEntry[] {
+  const entriesMap = new Map<string, LeaderboardEntry>();
+  
+  for (const result of results) {
+    const key = `${result.config.model}-${result.config.provider}-${result.config.language}`;
+    
+    if (entriesMap.has(key)) {
+      // Keep track of total runs, but only show most recent performance
+      const existing = entriesMap.get(key)!;
+      
+      // Increment total runs count
+      existing.totalRuns = existing.totalRuns + 1;
+      
+      // If this result is more recent, replace the performance data
+      if (result.endTime > existing.lastRun) {
+        existing.successRate = result.summary.successRate * 100; // Convert from decimal to percentage
+        existing.totalExercises = result.summary.totalExercises;
+        existing.averageCost = result.summary.totalCost / result.summary.totalExercises;
+        existing.averageTime = result.summary.averageTime;
+        existing.averageTurns = result.summary.averageTurns;
+        existing.lastRun = result.endTime;
+      }
+    } else {
+      // Create new entry
+      const entry: LeaderboardEntry = {
+        model: result.config.model,
+        provider: result.config.provider,
+        language: result.config.language,
+        successRate: result.summary.successRate * 100, // Convert from decimal to percentage
+        totalExercises: result.summary.totalExercises,
+        averageCost: result.summary.totalCost / result.summary.totalExercises,
+        averageTime: result.summary.averageTime,
+        averageTurns: result.summary.averageTurns,
+        totalRuns: 1,
+        lastRun: result.endTime
+      };
+      
+      entriesMap.set(key, entry);
+    }
+  }
+  
+  return Array.from(entriesMap.values());
+}

package/leaderboard/app/api/benchmark-detail/route.ts

@@ -0,0 +1,172 @@
+import { NextRequest, NextResponse } from 'next/server';
+import fs from 'fs';
+import path from 'path';
+import { BenchmarkResults } from '@/types/benchmark';
+
+// Recursive function to find JSON files in nested directories
+function findBenchmarkFiles(dir: string): string[] {
+  const files: string[] = [];
+
+  try {
+    const items = fs.readdirSync(dir, { withFileTypes: true });
+
+    for (const item of items) {
+      const fullPath = path.join(dir, item.name);
+
+      if (item.isDirectory()) {
+        // Recursively search subdirectories
+        files.push(...findBenchmarkFiles(fullPath));
+      } else if (item.isFile() && item.name.endsWith('.json')) {
+        // Add JSON files to our list
+        files.push(fullPath);
+      }
+    }
+  } catch (error) {
+    // Ignore directories we can't read
+  }
+
+  return files;
+}
+
+export async function GET(request: NextRequest) {
+  const { searchParams } = new URL(request.url);
+  const model = searchParams.get('model');
+  const provider = searchParams.get('provider');
+  const language = searchParams.get('language');
+  const timestamp = searchParams.get('timestamp'); // Optional parameter to get specific run
+
+  if (!model || !provider || !language) {
+    return NextResponse.json(
+      { error: 'Missing required parameters: model, provider, language' },
+      { status: 400 }
+    );
+  }
+
+  try {
+    // Look for benchmark result files in the results directory
+    const resultsDir = path.join(process.cwd(), '..', 'benchmarks', 'results');
+
+    if (!fs.existsSync(resultsDir)) {
+      return NextResponse.json(
+        { error: 'Results directory not found' },
+        { status: 404 }
+      );
+    }
+
+    // Find all JSON files recursively in the results directory
+    const allFiles = findBenchmarkFiles(resultsDir);
+
+    // Filter files that match our model/provider/language criteria
+    const matchingFiles = allFiles.filter(filePath => {
+      try {
+        // Read and parse the JSON file to check its config
+        const fileContent = fs.readFileSync(filePath, 'utf-8');
+        const data = JSON.parse(fileContent);
+
+        if (!data.config) {
+          return false;
+        }
+
+        const configModel = data.config.model;
+        const configProvider = data.config.provider;
+        const configLanguage = data.config.language;
+
+        // Exact match on all three parameters
+        return configModel === model &&
+               configProvider === provider &&
+               configLanguage === language;
+      } catch (error) {
+        return false;
+      }
+    });
+
+    if (matchingFiles.length === 0) {
+      return NextResponse.json(
+        { error: 'No benchmark results found for the specified model, provider, and language' },
+        { status: 404 }
+      );
+    }
+
+    // Load all matching benchmark results
+    const allResults: BenchmarkResults[] = [];
+    const filePathMap = new Map<BenchmarkResults, string>(); // Track file paths for commit extraction
+
+    for (const filePath of matchingFiles) {
+      try {
+        const fileContent = fs.readFileSync(filePath, 'utf-8');
+        const benchmarkData = JSON.parse(fileContent);
+
+        // Validate that we have the expected structure
+        if (benchmarkData.exercises && benchmarkData.summary && benchmarkData.config) {
+          allResults.push(benchmarkData);
+          filePathMap.set(benchmarkData, filePath);
+        }
+      } catch (parseError) {
+        console.error(`Error parsing file ${filePath}:`, parseError);
+        // Continue with other files
+      }
+    }
+
+    if (allResults.length === 0) {
+      return NextResponse.json(
+        { error: 'No valid benchmark results found' },
+        { status: 404 }
+      );
+    }
+
+    // Sort results by endTime (most recent first)
+    allResults.sort((a, b) => new Date(b.endTime).getTime() - new Date(a.endTime).getTime());
+
+    // If timestamp is provided, return that specific run
+    if (timestamp) {
+      const targetTime = timestamp;
+      const specificRun = allResults.find(result => result.endTime === targetTime);
+
+      if (specificRun) {
+        return NextResponse.json({
+          latest: specificRun,
+          history: [], // Don't need history for specific run view
+          totalRuns: allResults.length
+        });
+      }
+    }
+
+    // Get the most recent result as the main data
+    const latestResult = allResults[0];
+
+    // Create historical summary for previous runs (excluding the latest)
+    const previousRuns = allResults; // Skip the first (latest) result
+    const historicalRuns = previousRuns.map(result => ({
+      endTime: result.endTime,
+      successRate: result.summary.successRate * 100, // Convert to percentage
+      totalExercises: result.summary.totalExercises,
+      totalCost: result.summary.totalCost,
+      averageTime: result.summary.averageTime,
+      averageTurns: result.summary.averageTurns,
+      // Include commit info if available
+      commitHash: result.commitHash || 'unknown',
+      // Calculate average cost per exercise
+      averageCost: result.summary.totalCost / result.summary.totalExercises
+    }));
+
+    // Return both the latest detailed result and historical summary
+    const response = {
+      // Latest detailed benchmark data
+      latest: latestResult,
+      // Historical performance summary
+      history: historicalRuns,
+      // Total number of runs
+      totalRuns: allResults.length
+    };
+
+    return NextResponse.json(response);
+
+
+  } catch (error) {
+    console.error('Error reading benchmark detail:', error);
+    return NextResponse.json(
+      { error: 'Internal server error' },
+      { status: 500 }
+    );
+  }
+}