@pennyfarthing/benchmark 10.2.0

package/scripts/benchmark-runner.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# benchmark-runner.sh - Shell wrapper for benchmark-runner.js
+# This wrapper ensures the tests can call benchmark-runner.sh as expected
+#
+# All logic is implemented in benchmark-runner.js (Node.js)
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+exec node "$SCRIPT_DIR/benchmark-runner.js" "$@"

package/scripts/consolidate-job-fair.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# Consolidate Job Fair Results
+# Merges multiple role-specific runs for the same theme into one summary
+# Usage: ./scripts/consolidate-job-fair.sh [theme1 theme2 ...]
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)"
+RESULTS_DIR="$PROJECT_DIR/internal/results/job-fair"
+CONSOLIDATED_DIR="$RESULTS_DIR/consolidated"
+
+mkdir -p "$CONSOLIDATED_DIR"
+
+# If themes specified, use those; otherwise find all themes
+if [[ $# -gt 0 ]]; then
+    THEMES="$@"
+else
+    # Extract unique theme names from directory names
+    THEMES=$(ls -d "$RESULTS_DIR"/*-20* 2>/dev/null | xargs -n1 basename | sed 's/-20[0-9]*T[0-9]*Z$//' | sed 's/-20[0-9]*-[0-9]*$//' | sort -u)
+fi
+
+echo "=== Consolidating Job Fair Results ==="
+echo ""
+
+for THEME in $THEMES; do
+    echo "### Processing: $THEME"
+
+    # Find all directories for this theme
+    THEME_DIRS=$(ls -d "$RESULTS_DIR/${THEME}"-20* 2>/dev/null | sort)
+
+    if [[ -z "$THEME_DIRS" ]]; then
+        echo "  [SKIP] No results found for $THEME"
+        continue
+    fi
+
+    # Create consolidated directory for this theme
+    THEME_OUT="$CONSOLIDATED_DIR/$THEME"
+    mkdir -p "$THEME_OUT"
+
+    # Merge all raw_results.txt files
+    > "$THEME_OUT/raw_results.txt"
+
+    for DIR in $THEME_DIRS; do
+        if [[ -f "$DIR/raw_results.txt" ]]; then
+            cat "$DIR/raw_results.txt" >> "$THEME_OUT/raw_results.txt"
+        fi
+    done
+
+    # Count unique roles collected
+    ROLES=$(cut -d: -f3 "$THEME_OUT/raw_results.txt" 2>/dev/null | sort -u | tr '\n' ' ')
+    ROLE_COUNT=$(echo "$ROLES" | wc -w | tr -d ' ')
+
+    echo "  Merged $ROLE_COUNT roles: $ROLES"
+
+    # Generate consolidated summary
+    TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+
+    cat > "$THEME_OUT/summary.yaml" << EOF
+# Consolidated Job Fair Results
+theme: $THEME
+consolidated_at: $TIMESTAMP
+source_runs: $(echo "$THEME_DIRS" | wc -l | tr -d ' ')
+
+scenarios:
+  dev-codegen: tdd-shopping-cart
+  dev-debug: astropy-12907
+  reviewer: order-service
+  tea: payment-processor-tests
+  sm: sprint-planning-conflict
+  architect: legacy-modernization
+
+baselines:
+  dev-codegen: {mean: 85.8, std: 7.30, n: 10}
+  dev-debug: {mean: 77.5, std: 8.54, n: 10}
+  reviewer: {mean: 78.5, std: 1.8, n: 10}
+  tea: {mean: 72.1, std: 2.3, n: 10}
+  sm: {mean: 80.3, std: 1.9, n: 10}
+  architect: {mean: 87.2, std: 3.25, n: 10}
+
+EOF
+
+    # Parse raw results and build matrix
+    if [[ -s "$THEME_OUT/raw_results.txt" ]]; then
+        echo "" >> "$THEME_OUT/summary.yaml"
+        echo "matrix:" >> "$THEME_OUT/summary.yaml"
+
+        # Get unique characters (handle names with spaces)
+        cut -d: -f2 "$THEME_OUT/raw_results.txt" | sort -u | while IFS= read -r CHAR; do
+            [[ -z "$CHAR" ]] && continue
+            CHAR_KEY=$(echo "$CHAR" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/_/g' | sed 's/__*/_/g' | sed 's/^_//;s/_$//')
+            echo "  $CHAR_KEY:" >> "$THEME_OUT/summary.yaml"
+
+            # Get all roles for this character (escape special chars in grep)
+            CHAR_ESCAPED=$(printf '%s\n' "$CHAR" | sed 's/[[\.*^$()+?{|]/\\&/g')
+            grep ":${CHAR}:" "$THEME_OUT/raw_results.txt" | while IFS=: read -r src_role char role mean n; do
+                echo "    $role: {mean: $mean, n: $n}" >> "$THEME_OUT/summary.yaml"
+            done
+        done
+    fi
+
+    echo "  -> $THEME_OUT/summary.yaml"
+done
+
+echo ""
+echo "=== Consolidation Complete ==="
+echo "Results in: $CONSOLIDATED_DIR/"

package/scripts/convert-jobfair-to-benchmarks.sh

@@ -0,0 +1,230 @@
+#!/bin/bash
+# Convert job-fair results to benchmark format for showcase
+#
+# Reads: internal/results/job-fair/{theme}-*/runs/{role}/{character}/run_*.json
+# Writes: internal/results/benchmarks/{scenario}/{theme}-{role}/summary.yaml
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)"
+
+JOBFAIR_DIR="$PROJECT_DIR/internal/results/job-fair"
+BENCHMARKS_DIR="$PROJECT_DIR/internal/results/benchmarks"
+BASELINES_DIR="$PROJECT_DIR/internal/results/baselines"
+
+# Scenario mappings (role -> scenario)
+declare -A ROLE_SCENARIO
+ROLE_SCENARIO[dev-codegen]="tdd-shopping-cart"
+ROLE_SCENARIO[dev-debug]="astropy-12907"
+ROLE_SCENARIO[reviewer]="order-service"
+ROLE_SCENARIO[tea]="payment-processor-tests"
+ROLE_SCENARIO[sm]="sprint-planning-conflict"
+ROLE_SCENARIO[architect]="legacy-modernization"
+
+# Scenario metadata
+declare -A SCENARIO_TITLE
+SCENARIO_TITLE[tdd-shopping-cart]="TDD Shopping Cart Implementation"
+SCENARIO_TITLE[astropy-12907]="Astropy Issue #12907 Debug"
+SCENARIO_TITLE[order-service]="Order Service Code Review"
+SCENARIO_TITLE[payment-processor-tests]="Payment Processor Test Suite"
+SCENARIO_TITLE[sprint-planning-conflict]="Sprint Planning Conflict Resolution"
+SCENARIO_TITLE[legacy-modernization]="Legacy System Modernization"
+
+declare -A SCENARIO_CATEGORY
+SCENARIO_CATEGORY[tdd-shopping-cart]="dev"
+SCENARIO_CATEGORY[astropy-12907]="dev"
+SCENARIO_CATEGORY[order-service]="reviewer"
+SCENARIO_CATEGORY[payment-processor-tests]="tea"
+SCENARIO_CATEGORY[sprint-planning-conflict]="sm"
+SCENARIO_CATEGORY[legacy-modernization]="architect"
+
+# Get baseline stats for a scenario/role
+get_baseline() {
+    local scenario=$1
+    local role=$2
+    local baseline_file="$BASELINES_DIR/$scenario/$role/summary.yaml"
+
+    if [[ -f "$baseline_file" ]]; then
+        local mean=$(grep "mean:" "$baseline_file" | head -1 | awk '{print $2}')
+        local std=$(grep "std_dev:" "$baseline_file" | head -1 | awk '{print $2}')
+        echo "$mean:$std"
+    else
+        echo ""
+    fi
+}
+
+# Process a single theme
+process_theme() {
+    local theme=$1
+    echo "Processing theme: $theme"
+
+    # Find all job-fair run directories for this theme
+    local run_dirs=$(find "$JOBFAIR_DIR" -maxdepth 1 -type d -name "${theme}-*" 2>/dev/null)
+
+    if [[ -z "$run_dirs" ]]; then
+        echo "  No run directories found for $theme"
+        return
+    fi
+
+    # For each role we track
+    for role in dev-codegen dev-debug reviewer tea sm architect; do
+        local scenario="${ROLE_SCENARIO[$role]}"
+        local map_role="$role"
+
+        # Map dev-codegen and dev-debug to dev for directory lookup
+        local dir_role="$role"
+
+        # Collect all scores for this theme/role
+        local scores=()
+        local characters=()
+        local total_input_tokens=0
+        local total_output_tokens=0
+        local run_count=0
+
+        # Find the native character for this role from the theme
+        local theme_file="$PROJECT_DIR/pennyfarthing-dist/personas/themes/${theme}.yaml"
+        local lookup_role="$role"
+        [[ "$role" == "dev-codegen" || "$role" == "dev-debug" ]] && lookup_role="dev"
+
+        local native_char=""
+        if [[ -f "$theme_file" ]]; then
+            native_char=$(yq ".agents.${lookup_role}.character // \"\"" "$theme_file" 2>/dev/null | tr -d '"')
+        fi
+
+        # Scan all run directories
+        for run_dir in $run_dirs; do
+            local role_dir="$run_dir/runs/$dir_role"
+
+            if [[ ! -d "$role_dir" ]]; then
+                continue
+            fi
+
+            # For each character directory
+            for char_dir in "$role_dir"/*/; do
+                [[ ! -d "$char_dir" ]] && continue
+
+                # Read run files
+                for run_file in "$char_dir"/run_*.json; do
+                    [[ ! -f "$run_file" ]] && continue
+
+                    local score=$(python3 -c "import json; d=json.load(open('$run_file')); print(d.get('score', 0))" 2>/dev/null)
+                    local char=$(python3 -c "import json; d=json.load(open('$run_file')); print(d.get('character', ''))" 2>/dev/null)
+                    local input_tok=$(python3 -c "import json; d=json.load(open('$run_file')); print(d.get('agent_tokens', 0))" 2>/dev/null)
+
+                    if [[ -n "$score" && "$score" != "0" ]]; then
+                        scores+=("$score")
+                        [[ -z "${characters[0]}" ]] && characters+=("$char")
+                        total_input_tokens=$((total_input_tokens + input_tok))
+                        run_count=$((run_count + 1))
+                    fi
+                done
+            done
+        done
+
+        # Skip if no scores found
+        if [[ ${#scores[@]} -eq 0 ]]; then
+            continue
+        fi
+
+        # Calculate statistics
+        local n=${#scores[@]}
+        local sum=0
+        local min=100
+        local max=0
+
+        for s in "${scores[@]}"; do
+            sum=$(echo "$sum + $s" | bc)
+            [[ $(echo "$s < $min" | bc) -eq 1 ]] && min=$s
+            [[ $(echo "$s > $max" | bc) -eq 1 ]] && max=$s
+        done
+
+        local mean=$(echo "scale=2; $sum / $n" | bc)
+
+        # Calculate std dev
+        local sq_sum=0
+        for s in "${scores[@]}"; do
+            local diff=$(echo "$s - $mean" | bc)
+            sq_sum=$(echo "$sq_sum + ($diff * $diff)" | bc)
+        done
+        local variance=$(echo "scale=4; $sq_sum / $n" | bc)
+        local std_dev=$(echo "scale=2; sqrt($variance)" | bc)
+
+        # Get baseline
+        local baseline_role="$lookup_role"
+        local baseline=$(get_baseline "$scenario" "$baseline_role")
+        local baseline_mean=""
+        local baseline_std=""
+        local delta=""
+
+        if [[ -n "$baseline" ]]; then
+            baseline_mean=$(echo "$baseline" | cut -d: -f1)
+            baseline_std=$(echo "$baseline" | cut -d: -f2)
+            delta=$(echo "scale=2; $mean - $baseline_mean" | bc)
+        fi
+
+        # Create output directory
+        local output_dir="$BENCHMARKS_DIR/$scenario/${theme}-${lookup_role}"
+        mkdir -p "$output_dir"
+
+        # Format scores array
+        local scores_str=$(printf ", %.0f" "${scores[@]}")
+        scores_str="[${scores_str:2}]"
+
+        # Determine character (use native if available)
+        local char_name="${native_char:-${characters[0]:-Unknown}}"
+
+        # Write summary.yaml
+        cat > "$output_dir/summary.yaml" << EOF
+# ${theme}:${lookup_role} on ${scenario}
+# Generated from job-fair data: $(date -u +%Y-%m-%dT%H:%M:%SZ)
+
+agent:
+  theme: ${theme}
+  role: ${lookup_role}
+  spec: ${theme}:${lookup_role}
+  character: ${char_name}
+
+scenario:
+  name: ${scenario}
+  title: ${SCENARIO_TITLE[$scenario]}
+  category: ${SCENARIO_CATEGORY[$scenario]}
+  difficulty: medium
+
+statistics:
+  n: ${n}
+  mean: ${mean}
+  std_dev: ${std_dev}
+  min: ${min}
+  max: ${max}
+  scores: ${scores_str}
+EOF
+
+        # Add baseline comparison if available
+        if [[ -n "$baseline_mean" ]]; then
+            cat >> "$output_dir/summary.yaml" << EOF
+
+baseline_comparison:
+  control_mean: ${baseline_mean}
+  control_stddev: ${baseline_std}
+  delta: ${delta}
+EOF
+        fi
+
+        echo "  Created: $output_dir/summary.yaml (n=$n, mean=$mean)"
+    done
+}
+
+# Main
+echo "Converting job-fair results to benchmark format..."
+echo ""
+
+# Get list of themes from consolidated
+themes=$(ls -d "$JOBFAIR_DIR/consolidated"/*/ 2>/dev/null | xargs -n1 basename)
+
+for theme in $themes; do
+    process_theme "$theme"
+    echo ""
+done
+
+echo "Done!"

package/scripts/job-fair-batch.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+# Job Fair Batch Runner
+# Runs all characters in a theme against all role scenarios
+# Usage: ./scripts/job-fair-batch.sh <theme> [runs_per_combo]
+
+set -e
+
+THEME="$1"
+RUNS="${2:-4}"
+
+if [[ -z "$THEME" ]]; then
+    echo "Usage: $0 <theme> [runs_per_combo]" >&2
+    exit 1
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)"
+PERSONA_FILE="$PROJECT_DIR/pennyfarthing-dist/personas/themes/${THEME}.yaml"
+
+if [[ ! -f "$PERSONA_FILE" ]]; then
+    echo "Error: Theme not found: $THEME" >&2
+    exit 1
+fi
+
+# Output directory
+TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+OUTPUT_DIR="$PROJECT_DIR/internal/results/job-fair/${THEME}-${TIMESTAMP}"
+mkdir -p "$OUTPUT_DIR"
+
+echo "=== Job Fair: $THEME ===" | tee "$OUTPUT_DIR/log.txt"
+echo "Runs per combo: $RUNS" | tee -a "$OUTPUT_DIR/log.txt"
+echo "Output: $OUTPUT_DIR" | tee -a "$OUTPUT_DIR/log.txt"
+
+# Role -> Scenario mapping function (macOS bash 3.x compatible)
+get_scenario_for_role() {
+    case "$1" in
+        dev) echo "astropy-12907" ;;
+        reviewer) echo "astropy-12907" ;;
+        tea) echo "checkout-component-tests" ;;
+        sm) echo "dependency-deadlock" ;;
+        architect) echo "database-selection" ;;
+        *) echo "" ;;
+    esac
+}
+
+# Get all characters from theme
+CHARACTERS=$(yq -r '.agents | to_entries[] | "\(.key):\(.value.character)"' "$PERSONA_FILE")
+
+# Initialize results file
+echo "theme: $THEME" > "$OUTPUT_DIR/summary.yaml"
+echo "timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$OUTPUT_DIR/summary.yaml"
+echo "runs_per_combo: $RUNS" >> "$OUTPUT_DIR/summary.yaml"
+echo "results:" >> "$OUTPUT_DIR/summary.yaml"
+
+TOTAL_RUNS=0
+TOTAL_COST=0
+
+# For each target role
+for TARGET_ROLE in dev reviewer tea sm architect; do
+    SCENARIO=$(get_scenario_for_role "$TARGET_ROLE")
+    echo "" | tee -a "$OUTPUT_DIR/log.txt"
+    echo "--- Testing as $TARGET_ROLE (scenario: $SCENARIO) ---" | tee -a "$OUTPUT_DIR/log.txt"
+
+    # For each character
+    while IFS=: read -r SOURCE_ROLE CHARACTER; do
+        echo "  $CHARACTER ($SOURCE_ROLE -> $TARGET_ROLE)" | tee -a "$OUTPUT_DIR/log.txt"
+
+        CHAR_SLUG=$(echo "$CHARACTER" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g' | sed 's/--*/-/g')
+        COMBO_DIR="$OUTPUT_DIR/${TARGET_ROLE}/${CHAR_SLUG}"
+        mkdir -p "$COMBO_DIR"
+
+        SCORES=""
+        for RUN in $(seq 1 $RUNS); do
+            echo -n "    Run $RUN/$RUNS... " | tee -a "$OUTPUT_DIR/log.txt"
+
+            # Run solo benchmark with cross-role
+            RESULT=$("$SCRIPT_DIR/solo-runner.sh" "${THEME}:${CHARACTER}" "$SCENARIO" "$COMBO_DIR" --as "$TARGET_ROLE" 2>&1) || {
+                echo "FAILED" | tee -a "$OUTPUT_DIR/log.txt"
+                echo "$RESULT" >> "$OUTPUT_DIR/log.txt"
+                continue
+            }
+
+            SCORE=$(echo "$RESULT" | jq -r '.score // 0')
+            COST=$(echo "$RESULT" | jq -r '.total_cost_usd // 0')
+
+            echo "score=$SCORE cost=\$$COST" | tee -a "$OUTPUT_DIR/log.txt"
+
+            SCORES="$SCORES $SCORE"
+            TOTAL_COST=$(echo "$TOTAL_COST + $COST" | bc)
+            ((TOTAL_RUNS++))
+        done
+
+        # Calculate mean for this combo
+        if [[ -n "$SCORES" ]]; then
+            MEAN=$(echo "$SCORES" | tr ' ' '\n' | grep -v '^$' | awk '{sum+=$1; n++} END {if(n>0) printf "%.2f", sum/n; else print 0}')
+            echo "  - character: \"$CHARACTER\"" >> "$OUTPUT_DIR/summary.yaml"
+            echo "    source_role: $SOURCE_ROLE" >> "$OUTPUT_DIR/summary.yaml"
+            echo "    target_role: $TARGET_ROLE" >> "$OUTPUT_DIR/summary.yaml"
+            echo "    mean: $MEAN" >> "$OUTPUT_DIR/summary.yaml"
+            echo "    scores: [$SCORES ]" >> "$OUTPUT_DIR/summary.yaml"
+        fi
+
+    done <<< "$CHARACTERS"
+done
+
+echo "" | tee -a "$OUTPUT_DIR/log.txt"
+echo "=== Complete ===" | tee -a "$OUTPUT_DIR/log.txt"
+echo "Total runs: $TOTAL_RUNS" | tee -a "$OUTPUT_DIR/log.txt"
+echo "Total cost: \$$TOTAL_COST" | tee -a "$OUTPUT_DIR/log.txt"
+echo "Results: $OUTPUT_DIR/summary.yaml" | tee -a "$OUTPUT_DIR/log.txt"
+
+# Add totals to summary
+echo "total_runs: $TOTAL_RUNS" >> "$OUTPUT_DIR/summary.yaml"
+echo "total_cost_usd: $TOTAL_COST" >> "$OUTPUT_DIR/summary.yaml"
+
+echo "$OUTPUT_DIR"

package/scripts/job-fair-progress.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Check job fair progress across all running themes
+
+echo "=== Job Fair Progress ==="
+echo ""
+
+for log in /tmp/job-fair-*.log; do
+    if [[ -f "$log" ]]; then
+        theme=$(basename "$log" | sed 's/job-fair-//' | sed 's/.log//')
+
+        # Count completed runs
+        completed=$(grep -c "score=" "$log" 2>/dev/null || echo "0")
+        total=200
+
+        # Get last few lines
+        last_char=$(grep -E "^  [A-Z]" "$log" 2>/dev/null | tail -1 | sed 's/^ *//')
+        last_score=$(grep "score=" "$log" 2>/dev/null | tail -1 | grep -oE "score=[0-9.]+" | cut -d= -f2)
+
+        # Check if still running
+        if pgrep -f "job-fair-batch.sh $theme" > /dev/null 2>&1; then
+            status="RUNNING"
+        else
+            status="DONE"
+        fi
+
+        pct=$((completed * 100 / total))
+        echo "$theme: $completed/$total ($pct%) [$status]"
+        [[ -n "$last_char" ]] && echo "  Last: $last_char -> $last_score"
+    fi
+done
+
+echo ""
+echo "Total estimated: 1000 runs"
+total_done=$(cat /tmp/job-fair-*.log 2>/dev/null | grep -c "score=" || echo "0")
+echo "Total completed: $total_done"