@pennyfarthing/benchmark 10.2.0

package/scripts/solo-runner.sh

@@ -0,0 +1,344 @@
+#!/bin/bash
+# Solo Benchmark Runner
+# Executes a single agent benchmark with proper pipe syntax (NOT heredocs)
+# Usage: ./scripts/solo-runner.sh <theme:agent> <scenario> [output_dir] [--as <role>]
+#
+# Cross-role mode:
+#   ./scripts/solo-runner.sh shakespeare:prospero race-condition /tmp --as dev
+#   This runs Prospero (normally SM) as a dev on the scenario.
+
+set -e
+
+SPEC="$1"
+SCENARIO="$2"
+OUTPUT_DIR="${3:-/tmp/solo-results}"
+ROLE_OVERRIDE=""
+
+# Check for --as flag (can be in position 4 or after output_dir)
+if [[ "$4" == "--as" && -n "$5" ]]; then
+    ROLE_OVERRIDE="$5"
+elif [[ "$3" == "--as" && -n "$4" ]]; then
+    # Handle case where output_dir is omitted: theme:agent scenario --as role
+    OUTPUT_DIR="/tmp/solo-results"
+    ROLE_OVERRIDE="$4"
+fi
+
+if [[ -z "$SPEC" || -z "$SCENARIO" ]]; then
+    echo "Usage: $0 <theme:agent> <scenario> [output_dir] [--as <role>]" >&2
+    echo "" >&2
+    echo "Cross-role mode:" >&2
+    echo "  $0 shakespeare:prospero scenario /tmp --as dev" >&2
+    exit 1
+fi
+
+# Valid roles for --as validation
+VALID_ROLES="sm dev reviewer architect tea pm orchestrator tech-writer ux-designer devops"
+
+if [[ -n "$ROLE_OVERRIDE" ]]; then
+    if ! echo "$VALID_ROLES" | grep -qw "$ROLE_OVERRIDE"; then
+        echo "Error: Invalid role '$ROLE_OVERRIDE'. Must be one of: $VALID_ROLES" >&2
+        exit 1
+    fi
+fi
+
+# Parse spec
+THEME="${SPEC%%:*}"
+AGENT="${SPEC##*:}"
+
+if [[ "$THEME" == "$AGENT" ]]; then
+    echo "Error: Invalid spec format. Expected theme:agent, got: $SPEC" >&2
+    exit 1
+fi
+
+# Find paths
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)"
+PERSONA_FILE="$PROJECT_DIR/pennyfarthing-dist/personas/themes/${THEME}.yaml"
+SCENARIO_FILE=$(find "$SCRIPT_DIR/../scenarios" -name "${SCENARIO}.yaml" 2>/dev/null | head -1)
+
+if [[ ! -f "$PERSONA_FILE" ]]; then
+    echo "Error: Theme not found: $PERSONA_FILE" >&2
+    exit 1
+fi
+
+if [[ -z "$SCENARIO_FILE" || ! -f "$SCENARIO_FILE" ]]; then
+    echo "Error: Scenario not found: $SCENARIO" >&2
+    exit 1
+fi
+
+# Function to find which role a character belongs to (case-insensitive search)
+find_character_in_theme() {
+    local theme_file="$1"
+    local query="$2"
+
+    # Search all agents for matching character (case-insensitive)
+    # Use (?i) prefix for case-insensitive matching in yq
+    yq -r ".agents | to_entries[] | select(.value.character | test(\"(?i)$query\")) | .key" "$theme_file" | head -1
+}
+
+# Determine source role and effective role
+CROSS_ROLE=false
+if [[ -n "$ROLE_OVERRIDE" ]]; then
+    # Cross-role mode: AGENT is a character name, not a role
+    CHARACTER_QUERY="$AGENT"
+    SOURCE_ROLE=$(find_character_in_theme "$PERSONA_FILE" "$CHARACTER_QUERY")
+
+    if [[ -z "$SOURCE_ROLE" ]]; then
+        echo "Error: Character '$CHARACTER_QUERY' not found in theme '$THEME'" >&2
+        echo "Available characters:" >&2
+        yq -r '.agents | to_entries[] | "  - \(.key): \(.value.character)"' "$PERSONA_FILE" >&2
+        exit 1
+    fi
+
+    EFFECTIVE_ROLE="$ROLE_OVERRIDE"
+    CROSS_ROLE=true
+    # Use SOURCE_ROLE for persona lookup
+    LOOKUP_ROLE="$SOURCE_ROLE"
+else
+    # Standard mode: AGENT is the role name
+    SOURCE_ROLE="$AGENT"
+    EFFECTIVE_ROLE="$AGENT"
+    LOOKUP_ROLE="$AGENT"
+fi
+
+# Create output directory and temp dir
+mkdir -p "$OUTPUT_DIR"
+TMPDIR=$(mktemp -d)
+trap "rm -rf $TMPDIR" EXIT
+
+# Generate run ID
+TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+RUN_ID=$(echo "$SPEC-$SCENARIO-$$" | md5sum | head -c 8)
+
+# Extract persona info using yq (use LOOKUP_ROLE for persona, EFFECTIVE_ROLE for task)
+CHARACTER=$(yq -r ".agents.${LOOKUP_ROLE}.character // \"Unknown\"" "$PERSONA_FILE")
+STYLE=$(yq -r ".agents.${LOOKUP_ROLE}.style // \"Professional\"" "$PERSONA_FILE")
+EXPERTISE=$(yq -r ".agents.${LOOKUP_ROLE}.expertise // \"Software development\"" "$PERSONA_FILE")
+CATCHPHRASES=$(yq -r ".agents.${LOOKUP_ROLE}.catchphrases // [] | .[]" "$PERSONA_FILE" 2>/dev/null | sed 's/^/  - /' | head -5)
+
+# Generate character slug for cross-role output paths
+CHARACTER_SLUG=$(echo "$CHARACTER" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g' | sed 's/--*/-/g' | sed 's/^-//' | sed 's/-$//')
+
+# Extract scenario info to files (avoids shell escaping issues)
+yq -r '.prompt' "$SCENARIO_FILE" > "$TMPDIR/prompt.txt"
+yq -r '.code // ""' "$SCENARIO_FILE" > "$TMPDIR/code.txt"
+
+# Build agent prompt file directly
+cat > "$TMPDIR/agent_prompt.txt" << AGENT_PROMPT_EOF
+You are ${CHARACTER}.
+
+**Style:** ${STYLE}
+**Expertise:** ${EXPERTISE}
+**Catchphrases:**
+${CATCHPHRASES}
+
+---
+
+## Challenge
+
+$(cat "$TMPDIR/prompt.txt")
+
+## Code
+
+\`\`\`go
+$(cat "$TMPDIR/code.txt")
+\`\`\`
+
+---
+
+Respond fully in character. Under 500 words.
+
+**IMPORTANT:** Provide your complete response directly. Do not attempt to use tools, read files, or make function calls.
+AGENT_PROMPT_EOF
+
+# Execute agent via claude CLI with PIPE SYNTAX
+AGENT_OUTPUT=$(cat "$TMPDIR/agent_prompt.txt" | claude -p --output-format json --tools "" 2>/dev/null || echo '{"error": "CLI failed"}')
+
+# Extract agent results
+echo "$AGENT_OUTPUT" | jq -r '.result // .error // "No response"' > "$TMPDIR/response.txt"
+INPUT_TOKENS=$(echo "$AGENT_OUTPUT" | jq -r '.usage.input_tokens // 0')
+OUTPUT_TOKENS=$(echo "$AGENT_OUTPUT" | jq -r '.usage.output_tokens // 0')
+RESPONSE_LENGTH=$(wc -c < "$TMPDIR/response.txt" | tr -d ' ')
+# Extract model usage for per-model token tracking
+echo "$AGENT_OUTPUT" | jq -c '.modelUsage // {}' > "$TMPDIR/agent_model_usage.json"
+AGENT_COST=$(echo "$AGENT_OUTPUT" | jq -r '.total_cost_usd // 0')
+
+# Validate response
+if [[ $RESPONSE_LENGTH -lt 100 ]]; then
+    echo "Error: Response too short ($RESPONSE_LENGTH chars)" >&2
+    cat "$TMPDIR/response.txt" >&2
+    exit 1
+fi
+
+# Save agent response using jq to properly escape the response
+jq -n \
+  --arg run_id "$RUN_ID" \
+  --arg spec "$SPEC" \
+  --arg theme "$THEME" \
+  --arg agent "$AGENT" \
+  --arg character "$CHARACTER" \
+  --arg character_slug "$CHARACTER_SLUG" \
+  --arg source_role "$SOURCE_ROLE" \
+  --arg effective_role "$EFFECTIVE_ROLE" \
+  --argjson cross_role "$CROSS_ROLE" \
+  --arg scenario "$SCENARIO" \
+  --arg timestamp "$TIMESTAMP" \
+  --rawfile response "$TMPDIR/response.txt" \
+  --argjson response_length "$RESPONSE_LENGTH" \
+  --argjson input_tokens "$INPUT_TOKENS" \
+  --argjson output_tokens "$OUTPUT_TOKENS" \
+  --slurpfile model_usage "$TMPDIR/agent_model_usage.json" \
+  --argjson cost_usd "$AGENT_COST" \
+  '{
+    run_id: $run_id,
+    spec: $spec,
+    theme: $theme,
+    agent: $agent,
+    character: $character,
+    character_slug: $character_slug,
+    source_role: $source_role,
+    effective_role: $effective_role,
+    cross_role: $cross_role,
+    scenario: $scenario,
+    timestamp: $timestamp,
+    response: $response,
+    response_length: $response_length,
+    input_tokens: $input_tokens,
+    output_tokens: $output_tokens,
+    model_usage: $model_usage[0],
+    cost_usd: $cost_usd
+  }' > "$OUTPUT_DIR/agent_${RUN_ID}.json"
+
+# Build judge prompt file
+JUDGE_TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+
+cat > "$TMPDIR/judge_prompt.txt" << JUDGE_PROMPT_EOF
+You are an impartial judge evaluating an AI agent's response.
+
+## Contestant
+- **${SPEC}** (${CHARACTER})
+
+## Challenge
+$(cat "$TMPDIR/prompt.txt")
+
+## Response
+$(cat "$TMPDIR/response.txt")
+
+## Evaluation
+
+Score 1-10 on each dimension:
+
+1. **Correctness (25%)** - Technical accuracy
+2. **Depth (25%)** - Thoroughness
+3. **Quality (25%)** - Clarity and actionability
+4. **Persona (25%)** - Character embodiment
+
+Formula: (correctness × 2.5) + (depth × 2.5) + (quality × 2.5) + (persona × 2.5) = WEIGHTED_TOTAL
+
+**IMPORTANT: Output your evaluation as JSON only. No markdown, no extra text.**
+
+{
+  "scores": {
+    "correctness": { "value": N, "reasoning": "..." },
+    "depth": { "value": N, "reasoning": "..." },
+    "quality": { "value": N, "reasoning": "..." },
+    "persona": { "value": N, "reasoning": "..." }
+  },
+  "weighted_total": NN.N,
+  "assessment": "2-3 sentence overall assessment"
+}
+JUDGE_PROMPT_EOF
+
+# Execute judge via pipe syntax
+JUDGE_OUTPUT=$(cat "$TMPDIR/judge_prompt.txt" | claude -p --output-format json --tools "" 2>/dev/null || echo '{"error": "Judge CLI failed"}')
+
+# Extract judge results
+echo "$JUDGE_OUTPUT" | jq -r '.result // .error // "No response"' > "$TMPDIR/judge_response.txt"
+JUDGE_INPUT_TOKENS=$(echo "$JUDGE_OUTPUT" | jq -r '.usage.input_tokens // 0')
+JUDGE_OUTPUT_TOKENS=$(echo "$JUDGE_OUTPUT" | jq -r '.usage.output_tokens // 0')
+echo "$JUDGE_OUTPUT" | jq -c '.modelUsage // {}' > "$TMPDIR/judge_model_usage.json"
+JUDGE_COST=$(echo "$JUDGE_OUTPUT" | jq -r '.total_cost_usd // 0')
+
+# Extract score from judge response - try JSON first, then regex fallback
+SCORE=$(cat "$TMPDIR/judge_response.txt" | jq -r '.weighted_total // empty' 2>/dev/null || true)
+if [[ -z "$SCORE" ]]; then
+    # Fallback: extract from text
+    SCORE=$(grep -oE '"weighted_total"[^0-9]*([0-9.]+)' "$TMPDIR/judge_response.txt" | grep -oE '[0-9.]+' | tail -1 || true)
+fi
+if [[ -z "$SCORE" ]]; then
+    # Last resort: look for any WEIGHTED_TOTAL pattern
+    SCORE=$(grep -oE 'WEIGHTED_TOTAL[^0-9]*([0-9.]+)' "$TMPDIR/judge_response.txt" | grep -oE '[0-9.]+' | tail -1 || true)
+fi
+
+if [[ -z "$SCORE" || "$SCORE" == "null" ]]; then
+    echo "Error: Could not extract score from judge response" >&2
+    cat "$TMPDIR/judge_response.txt" >&2
+    exit 1
+fi
+
+# Save judge response using jq
+jq -n \
+  --arg run_id "$RUN_ID" \
+  --arg spec "$SPEC" \
+  --arg scenario "$SCENARIO" \
+  --arg timestamp "$JUDGE_TIMESTAMP" \
+  --argjson score "$SCORE" \
+  --rawfile judge_response "$TMPDIR/judge_response.txt" \
+  --argjson input_tokens "$JUDGE_INPUT_TOKENS" \
+  --argjson output_tokens "$JUDGE_OUTPUT_TOKENS" \
+  --slurpfile model_usage "$TMPDIR/judge_model_usage.json" \
+  --argjson cost_usd "$JUDGE_COST" \
+  '{
+    run_id: $run_id,
+    spec: $spec,
+    scenario: $scenario,
+    timestamp: $timestamp,
+    score: $score,
+    judge_response: $judge_response,
+    input_tokens: $input_tokens,
+    output_tokens: $output_tokens,
+    model_usage: $model_usage[0],
+    cost_usd: $cost_usd
+  }' > "$OUTPUT_DIR/judge_${RUN_ID}.json"
+
+# Output summary JSON to stdout
+jq -n \
+  --argjson success true \
+  --arg run_id "$RUN_ID" \
+  --arg spec "$SPEC" \
+  --arg theme "$THEME" \
+  --arg character "$CHARACTER" \
+  --arg character_slug "$CHARACTER_SLUG" \
+  --arg source_role "$SOURCE_ROLE" \
+  --arg effective_role "$EFFECTIVE_ROLE" \
+  --argjson cross_role "$CROSS_ROLE" \
+  --arg scenario "$SCENARIO" \
+  --argjson score "$SCORE" \
+  --argjson agent_tokens "$((INPUT_TOKENS + OUTPUT_TOKENS))" \
+  --argjson judge_tokens "$((JUDGE_INPUT_TOKENS + JUDGE_OUTPUT_TOKENS))" \
+  --slurpfile agent_model_usage "$TMPDIR/agent_model_usage.json" \
+  --slurpfile judge_model_usage "$TMPDIR/judge_model_usage.json" \
+  --argjson agent_cost "$AGENT_COST" \
+  --argjson judge_cost "$JUDGE_COST" \
+  --arg agent_file "$OUTPUT_DIR/agent_${RUN_ID}.json" \
+  --arg judge_file "$OUTPUT_DIR/judge_${RUN_ID}.json" \
+  '{
+    success: $success,
+    run_id: $run_id,
+    spec: $spec,
+    theme: $theme,
+    character: $character,
+    character_slug: $character_slug,
+    source_role: $source_role,
+    effective_role: $effective_role,
+    cross_role: $cross_role,
+    scenario: $scenario,
+    score: $score,
+    agent_tokens: $agent_tokens,
+    judge_tokens: $judge_tokens,
+    agent_model_usage: $agent_model_usage[0],
+    judge_model_usage: $judge_model_usage[0],
+    total_cost_usd: ($agent_cost + $judge_cost),
+    agent_file: $agent_file,
+    judge_file: $judge_file
+  }'

package/scripts/test/ensure-swebench-data.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# ensure-swebench-data.sh - Downloads SWE-bench data if not present
+#
+# Usage: ensure-swebench-data.sh [--force]
+#
+# Downloads SWE-bench Verified dataset from HuggingFace to /tmp/swebench_all.json
+# This is a dependency for:
+#   - swebench-judge.py
+#   - ground-truth-judge.py
+#
+# Options:
+#   --force    Re-download even if file exists
+
+set -euo pipefail
+
+CACHE_PATH="/tmp/swebench_all.json"
+DATASET_URL="https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified/resolve/main/data/test.jsonl"
+
+force=false
+if [[ "${1:-}" == "--force" ]]; then
+    force=true
+fi
+
+# Check if already present
+if [[ -f "$CACHE_PATH" ]] && [[ "$force" == "false" ]]; then
+    echo "SWE-bench data already cached at $CACHE_PATH"
+    exit 0
+fi
+
+echo "Downloading SWE-bench Verified dataset..."
+
+# Download JSONL and convert to JSON array
+if command -v curl &>/dev/null; then
+    curl -sL "$DATASET_URL" | python3 -c "
+import json
+import sys
+lines = [json.loads(line) for line in sys.stdin if line.strip()]
+print(json.dumps(lines, indent=2))
+" > "$CACHE_PATH"
+elif command -v wget &>/dev/null; then
+    wget -qO- "$DATASET_URL" | python3 -c "
+import json
+import sys
+lines = [json.loads(line) for line in sys.stdin if line.strip()]
+print(json.dumps(lines, indent=2))
+" > "$CACHE_PATH"
+else
+    echo "Error: curl or wget required to download SWE-bench data"
+    exit 1
+fi
+
+# Verify download
+if [[ -f "$CACHE_PATH" ]]; then
+    count=$(python3 -c "import json; print(len(json.load(open('$CACHE_PATH'))))")
+    echo "Downloaded $count SWE-bench scenarios to $CACHE_PATH"
+else
+    echo "Error: Failed to download SWE-bench data"
+    exit 1
+fi

package/scripts/test/ground-truth-judge.py

@@ -0,0 +1,220 @@
+#!/usr/bin/env python3
+"""
+Ground-truth judge for SWE-bench scenarios.
+
+Compares Claude's proposed fix against the actual SWE-bench patch.
+Scores based on:
+- File identification (20%)
+- Function/location identification (20%)
+- Fix logic match (40%)
+- Completeness (20%)
+"""
+
+import json
+import re
+import sys
+from pathlib import Path
+from difflib import SequenceMatcher
+
+# Add parent to path for pennyfarthing_scripts imports
+sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
+
+from pennyfarthing_scripts.swebench import (
+    extract_patch_info,
+    extract_problem_keywords,
+    find_scenario,
+    get_meaningful_patterns,
+    load_swebench_data,
+)
+
+
+def score_response(response_text, ground_truth):
+    """Score a response against ground truth patch."""
+    patch_info = extract_patch_info(ground_truth['patch'])
+
+    scores = {
+        'file_identification': 0,
+        'location_identification': 0,
+        'fix_logic_match': 0,
+        'completeness': 0,
+        'details': {}
+    }
+
+    response_lower = response_text.lower()
+
+    # 1. FILE IDENTIFICATION (20 points)
+    files_found = 0
+    for f in patch_info.files:
+        # Check various forms of the filename
+        filename = Path(f).name
+        if filename.lower() in response_lower or f.lower() in response_lower:
+            files_found += 1
+
+    if patch_info.files:
+        file_score = (files_found / len(patch_info.files)) * 20
+        scores['file_identification'] = min(20, file_score)
+        scores['details']['files_expected'] = patch_info.files
+        scores['details']['files_found'] = files_found
+    else:
+        scores['file_identification'] = 20  # No specific file in patch
+
+    # 2. LOCATION IDENTIFICATION (20 points)
+    # Look for function/class names mentioned in the patch
+    locations_found = 0
+    for func in patch_info.functions:
+        # Extract the function/class name
+        func_match = re.search(r'(def|class)\s+(\w+)', func)
+        if func_match:
+            func_name = func_match.group(2)
+            if func_name.lower() in response_lower:
+                locations_found += 1
+        elif func.strip() and func.strip().split()[0] in response_lower:
+            locations_found += 1
+
+    if patch_info.functions:
+        loc_score = (locations_found / len(patch_info.functions)) * 20
+        scores['location_identification'] = min(20, loc_score)
+        scores['details']['locations_expected'] = patch_info.functions[:3]
+        scores['details']['locations_found'] = locations_found
+    else:
+        scores['location_identification'] = 10  # Partial credit
+
+    # 3. FIX LOGIC MATCH (40 points)
+    # Check if key code patterns from the fix appear in the response
+    meaningful_patterns = get_meaningful_patterns(patch_info.key_patterns)
+
+    patterns_found = 0
+    for pattern in meaningful_patterns:
+        if pattern.lower() in response_lower:
+            patterns_found += 1
+
+    if meaningful_patterns:
+        pattern_score = (patterns_found / len(meaningful_patterns)) * 20
+        scores['details']['patterns_expected'] = meaningful_patterns[:10]
+        scores['details']['patterns_found'] = patterns_found
+    else:
+        pattern_score = 10
+
+    # Check for actual code additions
+    additions_matched = 0
+    for addition in patch_info.additions[:5]:  # Check first 5 additions
+        # Normalize and check
+        addition_normalized = re.sub(r'\s+', ' ', addition.lower())
+        response_normalized = re.sub(r'\s+', ' ', response_lower)
+
+        # Use fuzzy matching
+        similarity = SequenceMatcher(None, addition_normalized, response_normalized).ratio()
+        if similarity > 0.6 or addition_normalized in response_normalized:
+            additions_matched += 1
+
+    if patch_info.additions:
+        addition_score = (additions_matched / min(5, len(patch_info.additions))) * 20
+        scores['details']['additions_matched'] = additions_matched
+    else:
+        addition_score = 10
+
+    scores['fix_logic_match'] = min(40, pattern_score + addition_score)
+
+    # 4. COMPLETENESS (20 points)
+    # Does the response have all the elements of a good fix?
+    completeness_score = 0
+
+    # Has code block?
+    if '```' in response_text:
+        completeness_score += 5
+
+    # Has test considerations?
+    if 'test' in response_lower:
+        completeness_score += 5
+
+    # Mentions the specific error/issue?
+    problem_keywords = extract_problem_keywords(ground_truth.get('problem_statement', ''))
+    keywords_found = sum(1 for kw in problem_keywords if kw.lower() in response_lower)
+    if problem_keywords:
+        completeness_score += min(5, (keywords_found / len(problem_keywords)) * 5)
+    else:
+        completeness_score += 2.5
+
+    # Has explanation of why fix works?
+    explanation_words = ['because', 'this fixes', 'this resolves', 'the issue', 'the problem', 'solution']
+    if any(word in response_lower for word in explanation_words):
+        completeness_score += 5
+
+    scores['completeness'] = min(20, completeness_score)
+
+    # Total
+    scores['total'] = round(
+        scores['file_identification'] +
+        scores['location_identification'] +
+        scores['fix_logic_match'] +
+        scores['completeness']
+    , 1)
+
+    return scores
+
+
+def main():
+    if len(sys.argv) < 3:
+        print("Usage: ground-truth-judge.py <scenario_name> <response_file>")
+        print("Example: ground-truth-judge.py flask-5014 run_20260102T134237Z.json")
+        sys.exit(1)
+
+    scenario_name = sys.argv[1]
+    response_file = sys.argv[2]
+
+    # Load SWE-bench data
+    swebench_data = load_swebench_data()
+
+    # Find scenario
+    scenario = find_scenario(swebench_data, scenario_name)
+    if not scenario:
+        print(f"Error: Scenario '{scenario_name}' not found in SWE-bench data")
+        sys.exit(1)
+
+    # Load response
+    with open(response_file, 'r') as f:
+        response_data = json.load(f)
+
+    response_text = response_data.get('result', '')
+    if not response_text:
+        print("Error: No 'result' field in response file")
+        sys.exit(1)
+
+    # Score
+    scores = score_response(response_text, scenario)
+
+    # Output
+    print(f"\n{'='*60}")
+    print(f"GROUND TRUTH EVALUATION: {scenario_name}")
+    print(f"{'='*60}")
+    print(f"\nScores:")
+    print(f"  File Identification:     {scores['file_identification']:5.1f}/20")
+    print(f"  Location Identification: {scores['location_identification']:5.1f}/20")
+    print(f"  Fix Logic Match:         {scores['fix_logic_match']:5.1f}/40")
+    print(f"  Completeness:            {scores['completeness']:5.1f}/20")
+    print(f"  {'─'*40}")
+    print(f"  TOTAL:                   {scores['total']:5.1f}/100")
+
+    print(f"\nDetails:")
+    for key, value in scores['details'].items():
+        print(f"  {key}: {value}")
+
+    # Output JSON for programmatic use
+    output = {
+        'scenario': scenario_name,
+        'instance_id': scenario.get('instance_id'),
+        'scores': scores,
+        'ground_truth_patch_preview': scenario.get('patch', '')[:300]
+    }
+
+    # Save judge output
+    output_path = response_file.replace('run_', 'gt_judge_')
+    with open(output_path, 'w') as f:
+        json.dump(output, f, indent=2)
+    print(f"\nSaved to: {output_path}")
+
+    return scores
+
+
+if __name__ == '__main__':
+    main()