@pennyfarthing/benchmark 10.2.0

package/scripts/aggregate-benchmark-stats.js

@@ -0,0 +1,315 @@
+#!/usr/bin/env node
+/**
+ * aggregate-benchmark-stats.js - Aggregate job-fair results into benchmark statistics
+ *
+ * Scans consolidated job-fair results and generates unified statistics:
+ * - Per-role aggregate statistics (mean, std_dev, n, min, max)
+ * - Theme rankings per role
+ * - Control baseline comparisons
+ *
+ * Usage:
+ *   aggregate-benchmark-stats.js [--dry-run] [--theme THEME]
+ */
+
+import { readdirSync, writeFileSync, existsSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { execSync } from 'child_process';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const PROJECT_DIR = join(dirname(__dirname), '..', '..');
+const CONSOLIDATED_DIR = join(PROJECT_DIR, 'internal', 'results', 'job-fair', 'consolidated');
+const OUTPUT_FILE = join(PROJECT_DIR, 'internal', 'results', 'aggregate-stats.yaml');
+
+// Standard roles from job-fair
+const STANDARD_ROLES = ['dev-codegen', 'dev-debug', 'reviewer', 'tea', 'sm', 'architect'];
+
+// Baselines from control theme (hardcoded from job-fair-runner.sh)
+const BASELINES = {
+  'dev-codegen': { mean: 85.8, std: 7.30, n: 10 },
+  'dev-debug': { mean: 77.5, std: 8.54, n: 10 },
+  'reviewer': { mean: 78.5, std: 1.8, n: 10 },
+  'tea': { mean: 72.1, std: 2.3, n: 10 },
+  'sm': { mean: 80.3, std: 1.9, n: 10 },
+  'architect': { mean: 87.2, std: 3.25, n: 10 },
+};
+
+// Parse command line arguments
+function parseArgs(argv) {
+  const args = {
+    dryRun: false,
+    theme: null,
+  };
+
+  let i = 2;
+  while (i < argv.length) {
+    const arg = argv[i];
+    switch (arg) {
+      case '--dry-run':
+        args.dryRun = true;
+        break;
+      case '--theme':
+        args.theme = argv[++i];
+        break;
+      case '--help':
+      case '-h':
+        showUsage();
+        process.exit(0);
+        break;
+    }
+    i++;
+  }
+  return args;
+}
+
+function showUsage() {
+  console.log(`Usage: aggregate-benchmark-stats.js [OPTIONS]
+
+Options:
+  --dry-run       Output to stdout instead of writing file
+  --theme THEME   Only process specific theme
+  --help, -h      Show this help message
+
+Output: internal/results/aggregate-stats.yaml`);
+}
+
+// Extract YAML field using yq
+function yqGet(filePath, field) {
+  try {
+    const result = execSync(`yq -r '.${field}' "${filePath}"`, {
+      encoding: 'utf-8',
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+    const trimmed = result.trim();
+    return trimmed === 'null' ? null : trimmed;
+  } catch {
+    return null;
+  }
+}
+
+// Get all keys from a YAML object
+function yqKeys(filePath, field) {
+  try {
+    const result = execSync(`yq -r '.${field} | keys | .[]' "${filePath}"`, {
+      encoding: 'utf-8',
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+    return result.trim().split('\n').filter(Boolean);
+  } catch {
+    return [];
+  }
+}
+
+// Get numeric value from YAML
+function yqNumber(filePath, field) {
+  const val = yqGet(filePath, field);
+  return val !== null ? parseFloat(val) : null;
+}
+
+// Calculate statistics from array of numbers
+function calculateStats(values) {
+  if (!values || values.length === 0) {
+    return { mean: 0, std_dev: 0, n: 0, min: 0, max: 0 };
+  }
+
+  const n = values.length;
+  const mean = values.reduce((a, b) => a + b, 0) / n;
+  const variance = values.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / n;
+  const std_dev = Math.sqrt(variance);
+  const min = Math.min(...values);
+  const max = Math.max(...values);
+
+  return {
+    mean: Math.round(mean * 100) / 100,
+    std_dev: Math.round(std_dev * 100) / 100,
+    n,
+    min: Math.round(min * 100) / 100,
+    max: Math.round(max * 100) / 100,
+  };
+}
+
+// Load theme data from consolidated summary
+function loadThemeData(themePath) {
+  const summaryPath = join(themePath, 'summary.yaml');
+  if (!existsSync(summaryPath)) {
+    return null;
+  }
+
+  const theme = yqGet(summaryPath, 'theme');
+  if (!theme) return null;
+
+  const characters = yqKeys(summaryPath, 'matrix');
+  const roleScores = {};
+
+  // Initialize role arrays
+  for (const role of STANDARD_ROLES) {
+    roleScores[role] = [];
+  }
+
+  // Collect scores from each character
+  for (const char of characters) {
+    for (const role of STANDARD_ROLES) {
+      const mean = yqNumber(summaryPath, `matrix.${char}.${role}.mean`);
+      if (mean !== null && mean > 0) {
+        roleScores[role].push(mean);
+      }
+    }
+  }
+
+  return { theme, roleScores };
+}
+
+// Main aggregation logic
+function aggregate(args) {
+  // Get list of consolidated themes
+  let themes;
+  if (args.theme) {
+    themes = [args.theme];
+  } else {
+    themes = readdirSync(CONSOLIDATED_DIR, { withFileTypes: true })
+      .filter(d => d.isDirectory())
+      .map(d => d.name);
+  }
+
+  // Collect all scores per role across themes
+  const allRoleScores = {};
+  const themeRoleAverages = {}; // theme -> role -> mean
+
+  for (const role of STANDARD_ROLES) {
+    allRoleScores[role] = [];
+  }
+
+  let themesProcessed = 0;
+
+  for (const themeName of themes) {
+    const themePath = join(CONSOLIDATED_DIR, themeName);
+    const data = loadThemeData(themePath);
+    if (!data) continue;
+
+    themesProcessed++;
+    themeRoleAverages[data.theme] = {};
+
+    for (const role of STANDARD_ROLES) {
+      // Add all character scores for this theme/role to the global pool
+      allRoleScores[role].push(...data.roleScores[role]);
+
+      // Calculate theme average for rankings
+      if (data.roleScores[role].length > 0) {
+        const avg = data.roleScores[role].reduce((a, b) => a + b, 0) / data.roleScores[role].length;
+        themeRoleAverages[data.theme][role] = Math.round(avg * 100) / 100;
+      }
+    }
+  }
+
+  // Calculate aggregate statistics per role
+  const roles = {};
+  for (const role of STANDARD_ROLES) {
+    roles[role] = calculateStats(allRoleScores[role]);
+  }
+
+  // Generate rankings per role (sorted by mean descending)
+  const rankings = {};
+  for (const role of STANDARD_ROLES) {
+    const themeList = [];
+    for (const [theme, roleAvgs] of Object.entries(themeRoleAverages)) {
+      if (roleAvgs[role] !== undefined) {
+        themeList.push({ theme, mean: roleAvgs[role] });
+      }
+    }
+    // Sort descending by mean
+    themeList.sort((a, b) => b.mean - a.mean);
+    rankings[role] = themeList;
+  }
+
+  // Control baseline comparison
+  const control = {
+    baseline: 'from job-fair-runner.sh',
+  };
+
+  // Get control theme data if available
+  const controlPath = join(CONSOLIDATED_DIR, 'control');
+  const controlData = loadThemeData(controlPath);
+
+  for (const role of STANDARD_ROLES) {
+    const baseline = BASELINES[role];
+    let controlMean = null;
+
+    if (controlData && controlData.roleScores[role].length > 0) {
+      const scores = controlData.roleScores[role];
+      controlMean = scores.reduce((a, b) => a + b, 0) / scores.length;
+      controlMean = Math.round(controlMean * 100) / 100;
+    }
+
+    control[role] = {
+      mean: controlMean !== null ? controlMean : baseline.mean,
+      baseline_mean: baseline.mean,
+      vs_baseline: controlMean !== null
+        ? Math.round((controlMean - baseline.mean) * 100) / 100
+        : 0,
+    };
+  }
+
+  // Build output YAML
+  const timestamp = new Date().toISOString();
+
+  const output = {
+    metadata: {
+      generated_at: timestamp,
+      themes_processed: themesProcessed,
+      source: 'internal/results/job-fair/consolidated/',
+    },
+    roles,
+    rankings,
+    control,
+  };
+
+  return output;
+}
+
+// Format output as YAML
+function toYaml(obj, indent = 0) {
+  const spaces = '  '.repeat(indent);
+  let yaml = '';
+
+  for (const [key, value] of Object.entries(obj)) {
+    if (value === null || value === undefined) {
+      yaml += `${spaces}${key}: null\n`;
+    } else if (Array.isArray(value)) {
+      yaml += `${spaces}${key}:\n`;
+      for (const item of value) {
+        if (typeof item === 'object') {
+          // Inline object format for array items
+          const parts = Object.entries(item).map(([k, v]) => `${k}: ${v}`);
+          yaml += `${spaces}  - {${parts.join(', ')}}\n`;
+        } else {
+          yaml += `${spaces}  - ${item}\n`;
+        }
+      }
+    } else if (typeof value === 'object') {
+      yaml += `${spaces}${key}:\n`;
+      yaml += toYaml(value, indent + 1);
+    } else {
+      yaml += `${spaces}${key}: ${value}\n`;
+    }
+  }
+
+  return yaml;
+}
+
+// Main entry point
+function main() {
+  const args = parseArgs(process.argv);
+  const result = aggregate(args);
+  const yamlOutput = '# Aggregate Benchmark Statistics\n# Generated from job-fair consolidated results\n\n' + toYaml(result);
+
+  if (args.dryRun) {
+    console.log(yamlOutput);
+  } else {
+    writeFileSync(OUTPUT_FILE, yamlOutput);
+    console.log(`Wrote aggregate stats to ${OUTPUT_FILE}`);
+    console.log(`Themes processed: ${result.metadata.themes_processed}`);
+  }
+}
+
+main();

package/scripts/aggregate-benchmark-stats.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# aggregate-benchmark-stats.sh - Shell wrapper for aggregate-benchmark-stats.js
+# Aggregates job-fair results into unified benchmark statistics
+#
+# All logic is implemented in aggregate-benchmark-stats.js (Node.js)
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+exec node "$SCRIPT_DIR/aggregate-benchmark-stats.js" "$@"

package/scripts/benchmark-runner.js

@@ -0,0 +1,392 @@
+#!/usr/bin/env node
+/**
+ * benchmark-runner.js - Unified entry point for benchmark execution
+ * Dispatches to solo-runner.sh and job-fair-runner.sh as appropriate
+ *
+ * Usage:
+ *   benchmark-runner.js --mode catalog [--category CAT] [--format json|text]
+ *   benchmark-runner.js --mode info --case CASE_ID [--format json|text]
+ *   benchmark-runner.js --mode solo --case CASE_ID --agent AGENT_SPEC [--dry-run] [OUTPUT_DIR]
+ *   benchmark-runner.js --mode suite --category CAT --agent AGENT_SPEC [--dry-run] [OUTPUT_DIR]
+ */
+
+import { readFileSync, readdirSync, existsSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { spawn, execSync } from 'child_process';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const PROJECT_DIR = join(dirname(__dirname), '..', '..');
+const TEST_CASES_DIR = join(PROJECT_DIR, 'benchmarks', 'test-cases');
+const SCRIPTS_DIR = __dirname;
+
+// Simple YAML field extractor using yq (already installed)
+function parseYamlField(filePath, field) {
+  try {
+    const result = execSync(`yq -r '.${field}' "${filePath}"`, {
+      encoding: 'utf-8',
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+    const trimmed = result.trim();
+    return trimmed === 'null' ? '' : trimmed;
+  } catch {
+    return '';
+  }
+}
+
+// Load test case metadata from YAML file
+function loadTestCase(filePath) {
+  try {
+    const id = parseYamlField(filePath, 'id');
+    if (!id) return null;
+
+    return {
+      id,
+      name: parseYamlField(filePath, 'name'),
+      category: parseYamlField(filePath, 'category'),
+      difficulty: parseYamlField(filePath, 'difficulty'),
+      agent: parseYamlField(filePath, 'agent'),
+      version: parseYamlField(filePath, 'version'),
+      description: parseYamlField(filePath, 'description'),
+      _filePath: filePath,
+    };
+  } catch {
+    return null;
+  }
+}
+
+// Parse command line arguments
+function parseArgs(argv) {
+  const args = {
+    mode: null,
+    category: null,
+    caseId: null,
+    agent: null,
+    format: 'text',
+    dryRun: false,
+    outputDir: null,
+  };
+
+  let i = 2;
+  while (i < argv.length) {
+    const arg = argv[i];
+    switch (arg) {
+      case '--mode':
+        args.mode = argv[++i];
+        break;
+      case '--category':
+        args.category = argv[++i];
+        break;
+      case '--case':
+        args.caseId = argv[++i];
+        break;
+      case '--agent':
+        args.agent = argv[++i];
+        break;
+      case '--format':
+        args.format = argv[++i];
+        break;
+      case '--dry-run':
+        args.dryRun = true;
+        break;
+      case '--help':
+      case '-h':
+        showUsage();
+        process.exit(0);
+        break;
+      default:
+        if (!arg.startsWith('-') && !args.outputDir) {
+          args.outputDir = arg;
+        }
+        break;
+    }
+    i++;
+  }
+
+  return args;
+}
+
+function showUsage() {
+  console.log(`Usage: benchmark-runner.js [OPTIONS]
+
+Modes:
+  --mode catalog    List available test cases
+  --mode info       Show details of a specific test case
+  --mode solo       Run single agent on single test case
+  --mode suite      Run agent on all test cases in a category
+
+Options:
+  --category CAT    Filter by category (dev, architecture, code-review, etc.)
+  --case CASE_ID    Test case ID (e.g., dev-001)
+  --agent SPEC      Agent specification (theme:role, e.g., rome:dev)
+  --format FORMAT   Output format: text (default) or json
+  --dry-run         Show what would be run without executing
+
+Examples:
+  benchmark-runner.js --mode catalog
+  benchmark-runner.js --mode catalog --category dev --format json
+  benchmark-runner.js --mode info --case dev-001
+  benchmark-runner.js --mode solo --case dev-001 --agent rome:dev --dry-run
+  benchmark-runner.js --mode suite --category dev --agent rome:dev --dry-run`);
+}
+
+function error(message) {
+  console.error(`Error: ${message}`);
+  process.exit(1);
+}
+
+// Recursively find all YAML files
+function findYamlFiles(dir) {
+  const files = [];
+  if (!existsSync(dir)) return files;
+
+  const entries = readdirSync(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...findYamlFiles(fullPath));
+    } else if (entry.isFile() && entry.name.endsWith('.yaml')) {
+      files.push(fullPath);
+    }
+  }
+  return files.sort();
+}
+
+// Get all test cases
+function getAllTestCases(categoryFilter = null) {
+  const files = findYamlFiles(TEST_CASES_DIR);
+  const testCases = [];
+
+  for (const file of files) {
+    const tc = loadTestCase(file);
+    if (tc && tc.id) {
+      if (!categoryFilter || tc.category === categoryFilter) {
+        testCases.push(tc);
+      }
+    }
+  }
+
+  return testCases;
+}
+
+// Find specific test case by ID
+function findTestCase(caseId) {
+  const files = findYamlFiles(TEST_CASES_DIR);
+  for (const file of files) {
+    const tc = loadTestCase(file);
+    if (tc && tc.id === caseId) {
+      return tc;
+    }
+  }
+  return null;
+}
+
+// Get categories
+function getCategories() {
+  if (!existsSync(TEST_CASES_DIR)) return [];
+  return readdirSync(TEST_CASES_DIR, { withFileTypes: true })
+    .filter(entry => entry.isDirectory())
+    .map(entry => entry.name)
+    .sort();
+}
+
+// Catalog mode
+function doCatalog(args) {
+  const testCases = getAllTestCases(args.category);
+
+  if (args.format === 'json') {
+    const output = testCases.map(tc => ({
+      id: tc.id,
+      name: tc.name,
+      category: tc.category,
+      difficulty: tc.difficulty,
+    }));
+    console.log(JSON.stringify(output));
+  } else {
+    console.log('Available Test Cases:');
+    console.log('');
+
+    const categories = getCategories();
+    for (const cat of categories) {
+      if (args.category && cat !== args.category) continue;
+
+      const casesInCat = testCases.filter(tc => tc.category === cat);
+      if (casesInCat.length === 0 && !args.category) continue;
+
+      console.log(`[${cat}]`);
+      for (const tc of casesInCat) {
+        const id = (tc.id || '').padEnd(12);
+        const name = (tc.name || '').substring(0, 40).padEnd(40);
+        const diff = tc.difficulty || 'unknown';
+        console.log(`  ${id} ${name} (${diff})`);
+      }
+      console.log('');
+    }
+  }
+}
+
+// Info mode
+function doInfo(args) {
+  if (!args.caseId) {
+    error('Info mode requires --case CASE_ID');
+  }
+
+  const tc = findTestCase(args.caseId);
+  if (!tc) {
+    error(`Test case not found: ${args.caseId}`);
+  }
+
+  if (args.format === 'json') {
+    const output = {
+      id: tc.id,
+      name: tc.name,
+      category: tc.category,
+      difficulty: tc.difficulty,
+      agent: tc.agent,
+      version: tc.version,
+      description: tc.description,
+    };
+    console.log(JSON.stringify(output, null, 2));
+  } else {
+    console.log(`Test Case: ${tc.id}`);
+    console.log('');
+    console.log(`Name:       ${tc.name}`);
+    console.log(`Category:   ${tc.category}`);
+    console.log(`Difficulty: ${tc.difficulty}`);
+    console.log(`Agent:      ${tc.agent}`);
+    console.log(`Version:    ${tc.version}`);
+    console.log('');
+    console.log('Description:');
+    const desc = tc.description || '';
+    desc.split('\n').forEach(line => console.log(`  ${line}`));
+  }
+}
+
+// Solo mode
+function doSolo(args) {
+  if (!args.caseId) {
+    error('Solo mode requires --case CASE_ID');
+  }
+  if (!args.agent) {
+    error('Solo mode requires --agent AGENT_SPEC');
+  }
+
+  const tc = findTestCase(args.caseId);
+  if (!tc) {
+    error(`Test case not found: ${args.caseId}`);
+  }
+
+  const outputDir = args.outputDir || '/tmp/benchmark-results';
+
+  if (args.dryRun) {
+    const output = {
+      mode: 'solo',
+      test_case: args.caseId,
+      agent: args.agent,
+      dry_run: true,
+      would_execute: `solo-runner.sh ${args.agent} ${args.caseId} ${outputDir}`,
+    };
+    console.log(JSON.stringify(output, null, 2));
+  } else {
+    const soloRunner = join(SCRIPTS_DIR, 'solo-runner.sh');
+    const child = spawn(soloRunner, [args.agent, args.caseId, outputDir], {
+      stdio: 'inherit',
+    });
+    child.on('close', code => process.exit(code || 0));
+  }
+}
+
+// Suite mode
+function doSuite(args) {
+  if (!args.category) {
+    error('Suite mode requires --category CAT');
+  }
+  if (!args.agent) {
+    error('Suite mode requires --agent AGENT_SPEC');
+  }
+
+  const testCases = getAllTestCases(args.category);
+  const caseIds = testCases.map(tc => tc.id);
+  const total = caseIds.length;
+
+  if (args.dryRun) {
+    const output = {
+      mode: 'suite',
+      category: args.category,
+      agent: args.agent,
+      dry_run: true,
+      total_cases: total,
+      cases: caseIds,
+      summary: `Would run ${total} test cases in category '${args.category}'`,
+    };
+    console.log(JSON.stringify(output, null, 2));
+  } else {
+    const outputDir = args.outputDir || '/tmp/benchmark-results';
+    console.log(`Running suite: ${args.category} (${total} cases)`);
+    console.log('');
+
+    let passed = 0;
+    let failed = 0;
+
+    const runNext = (index) => {
+      if (index >= caseIds.length) {
+        console.log('');
+        console.log(`Suite complete: ${passed} passed, ${failed} failed (total: ${total})`);
+        process.exit(failed > 0 ? 1 : 0);
+        return;
+      }
+
+      const caseId = caseIds[index];
+      console.log(`[${caseId}] Running...`);
+
+      const soloRunner = join(SCRIPTS_DIR, 'solo-runner.sh');
+      const child = spawn(soloRunner, [args.agent, caseId, outputDir], {
+        stdio: 'inherit',
+      });
+
+      child.on('close', code => {
+        if (code === 0) {
+          passed++;
+        } else {
+          failed++;
+        }
+        runNext(index + 1);
+      });
+    };
+
+    runNext(0);
+  }
+}
+
+// Main
+function main() {
+  const args = parseArgs(process.argv);
+
+  if (!args.mode) {
+    showUsage();
+    process.exit(0);
+  }
+
+  const validModes = ['catalog', 'info', 'solo', 'suite'];
+  if (!validModes.includes(args.mode)) {
+    error(`Unknown mode: ${args.mode}. Valid modes: ${validModes.join(', ')}`);
+  }
+
+  switch (args.mode) {
+    case 'catalog':
+      doCatalog(args);
+      break;
+    case 'info':
+      doInfo(args);
+      break;
+    case 'solo':
+      doSolo(args);
+      break;
+    case 'suite':
+      doSuite(args);
+      break;
+  }
+}
+
+main();