@pennyfarthing/benchmark 10.2.0

package/scenarios/swe-bench/django-10973.yaml

@@ -0,0 +1,93 @@
+---
+# SWE-bench Verified Scenario
+# Source: https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified
+# Instance: django__django-10973
+
+name: django-10973
+title: "Use subprocess.run and PGPASSWORD for client in postgres backend"
+category: dev
+difficulty: medium  # SWE-bench: 15 min - 1 hour
+version: "1.0"
+
+source:
+  benchmark: swe-bench-verified
+  instance_id: django__django-10973
+  repo: django/django
+  base_commit: ddb293685235
+
+description: |
+  Real GitHub issue from django/django requiring code changes to resolve.
+  This is a human-validated problem from the SWE-bench Verified dataset.
+
+prompt: |
+  You are working on the django/django repository at commit ddb293685235.
+
+  A user has reported the following issue:
+
+  ---
+  Use subprocess.run and PGPASSWORD for client in postgres backend
+  Description
+	
+  ​subprocess.run was added in python 3.5 (which is the minimum version since Django 2.1). This function allows you to pass a custom environment for the subprocess.
+  Using this in django.db.backends.postgres.client to set PGPASSWORD simplifies the code and makes it more reliable.
+
+  ---
+
+  Analyze this issue and provide:
+  1. Root cause analysis - what is causing the bug?
+  2. Proposed fix - what code changes would resolve this?
+  3. Test considerations - how would you verify the fix works?
+
+  Provide your response with specific file paths and code changes.
+
+scoring:
+  # Adapted for SWE-bench bug-fix scenarios
+  categories:
+    - name: root_cause
+      weight: 30
+      description: "Correctly identifies the underlying cause of the bug"
+      criteria:
+        - id: IDENTIFIES_BUG_LOCATION
+          description: "Points to correct file(s) and function(s)"
+          points: 15
+        - id: EXPLAINS_WHY_BROKEN
+          description: "Explains why current code fails"
+          points: 15
+
+    - name: fix_quality
+      weight: 40
+      description: "Proposes a correct and complete fix"
+      criteria:
+        - id: FIX_ADDRESSES_ISSUE
+          description: "Fix would resolve the reported problem"
+          points: 20
+        - id: FIX_IS_MINIMAL
+          description: "Fix is appropriately scoped, not over-engineered"
+          points: 10
+        - id: FIX_SYNTAX_CORRECT
+          description: "Code changes are syntactically valid"
+          points: 10
+
+    - name: completeness
+      weight: 20
+      description: "Considers edge cases and testing"
+      criteria:
+        - id: EDGE_CASES
+          description: "Considers related scenarios that might break"
+          points: 10
+        - id: TEST_COVERAGE
+          description: "Suggests appropriate test cases"
+          points: 10
+
+    - name: persona
+      weight: 10
+      description: "Maintains character while solving"
+      criteria:
+        - id: IN_CHARACTER
+          description: "Response reflects persona traits"
+          points: 10
+
+# Metadata for full harness evaluation (optional)
+swebench_metadata:
+  fail_to_pass: ["test_accent (dbshell.test_postgresql.PostgreSqlDbshellCommandTestCase)", "test_basic (dbshell.test_postgresql.PostgreSqlDbshellCommandTestCase)", "test_column (dbshell.test_postgresql.PostgreSqlDbshellCommandTestCase)"]
+  environment_version: "3.0"

package/scenarios/swe-bench/flask-5014-reviewer.yaml

@@ -0,0 +1,145 @@
+---
+# SWE-bench Reviewer Scenario
+# Adapted from: pallets__flask-5014
+# Role: Code Reviewer evaluating a proposed fix
+
+name: flask-5014-reviewer
+title: "Review: Require non-empty name for Blueprints"
+category: reviewer
+difficulty: easy
+version: "1.0"
+
+source:
+  benchmark: swe-bench-verified
+  instance_id: pallets__flask-5014
+  repo: pallets/flask
+  base_commit: 7ee9ceb71e86
+  adapted_for: reviewer
+
+description: |
+  Code review scenario adapted from SWE-bench. The reviewer must evaluate
+  a proposed patch for correctness, completeness, and code quality.
+
+prompt: |
+  You are reviewing a pull request for the pallets/flask repository.
+
+  ## Issue Being Fixed
+
+  **Title:** Require a non-empty name for Blueprints
+
+  **Description:**
+  Things do not work correctly if a Blueprint is given an empty name (e.g. #4944).
+  It would be helpful if a `ValueError` was raised when trying to do that.
+
+  ## Proposed Patch
+
+  The developer has submitted the following changes:
+
+  **src/flask/blueprints.py** - In the Blueprint.__init__ method:
+  ```python
+  def __init__(
+      self,
+      name: str,
+      import_name: str,
+      ...
+  ) -> None:
+      # NEW: Validate non-empty name
+      if not name:
+          raise ValueError("'name' may not be empty.")
+
+      if "." in name:
+          raise ValueError("'name' may not contain a dot '.' character.")
+      ...
+  ```
+
+  **tests/test_blueprints.py** - New test added:
+  ```python
+  def test_empty_name_not_allowed(app, client):
+      with pytest.raises(ValueError):
+          flask.Blueprint("", __name__)
+  ```
+
+  **CHANGES.rst** - Added to changelog:
+  ```
+  - If a blueprint is created with an empty name it raises a ValueError.
+    :issue:`5010`
+  ```
+
+  ## Your Review Task
+
+  Evaluate this pull request and provide:
+  1. **Correctness** - Does the fix address the issue? Any bugs?
+  2. **Edge Cases** - What about whitespace-only names? None values?
+  3. **Test Coverage** - Are the tests sufficient?
+  4. **Code Quality** - Style, error message clarity, placement
+  5. **Verdict** - APPROVE, REQUEST_CHANGES, or COMMENT
+
+  Be thorough but fair. Identify real issues, not nitpicks.
+
+scoring:
+  categories:
+    - name: issue_detection
+      weight: 35
+      description: "Identifies legitimate issues with the patch"
+      criteria:
+        - id: WHITESPACE_EDGE_CASE
+          description: "Notes that whitespace-only names like '   ' would pass"
+          points: 15
+          severity: high
+        - id: NONE_HANDLING
+          description: "Considers behavior if name=None is passed"
+          points: 10
+          severity: medium
+        - id: ERROR_MESSAGE_QUALITY
+          description: "Comments on error message clarity/consistency"
+          points: 5
+          severity: low
+        - id: TEST_COVERAGE_GAP
+          description: "Notes missing edge case tests"
+          points: 5
+          severity: low
+
+    - name: review_quality
+      weight: 30
+      description: "Quality of the review feedback"
+      criteria:
+        - id: CONSTRUCTIVE_FEEDBACK
+          description: "Provides actionable suggestions, not just complaints"
+          points: 15
+        - id: CORRECT_ASSESSMENT
+          description: "Doesn't flag false positives or miss obvious issues"
+          points: 15
+
+    - name: verdict_appropriateness
+      weight: 20
+      description: "Appropriate review decision"
+      criteria:
+        - id: REASONABLE_VERDICT
+          description: "Verdict matches severity of issues found"
+          points: 10
+        - id: JUSTIFIED_DECISION
+          description: "Decision is well-reasoned"
+          points: 10
+
+    - name: persona
+      weight: 15
+      description: "Maintains reviewer character"
+      criteria:
+        - id: IN_CHARACTER
+          description: "Response reflects persona traits"
+          points: 15
+
+baseline_issues:
+  critical: []
+  high:
+    - id: WHITESPACE_EDGE_CASE
+      description: "Empty check 'if not name' passes for whitespace-only strings"
+      expected_fix: "Use 'if not name or not name.strip()' or similar"
+  medium:
+    - id: NONE_HANDLING
+      description: "If name=None, 'if not name' works but error message is misleading"
+  low:
+    - id: ERROR_MESSAGE_CONSISTENCY
+      description: "Error message style differs from existing dot check"
+    - id: MISSING_WHITESPACE_TEST
+      description: "No test for whitespace-only names"

package/scenarios/swe-bench/flask-5014-tea.yaml

@@ -0,0 +1,123 @@
+---
+# SWE-bench TEA Scenario
+# Adapted from: pallets__flask-5014
+# Role: Test Engineer writing failing tests BEFORE the fix
+
+name: flask-5014-tea
+title: "RED Phase: Write tests for empty Blueprint name validation"
+category: tea
+difficulty: easy
+version: "1.0"
+
+source:
+  benchmark: swe-bench-verified
+  instance_id: pallets__flask-5014
+  repo: pallets/flask
+  base_commit: 7ee9ceb71e86
+  adapted_for: tea
+
+description: |
+  TDD RED phase scenario adapted from SWE-bench. The test engineer must write
+  failing tests that specify the expected behavior BEFORE seeing any fix.
+
+prompt: |
+  You are a Test Engineer working on the pallets/flask repository.
+
+  ## Bug Report
+
+  **Title:** Require a non-empty name for Blueprints
+
+  **Description:**
+  Things do not work correctly if a Blueprint is given an empty name (e.g. #4944).
+  It would be helpful if a `ValueError` was raised when trying to do that.
+
+  **Current Behavior:** Creating `Blueprint("", __name__)` succeeds but causes
+  problems downstream when the blueprint is registered.
+
+  **Expected Behavior:** A `ValueError` should be raised immediately when
+  attempting to create a Blueprint with an empty name.
+
+  ## Your Task (RED Phase)
+
+  Write comprehensive failing tests that:
+  1. Specify the expected behavior (ValueError on empty name)
+  2. Cover edge cases (whitespace, None, etc.)
+  3. Will FAIL against the current codebase
+  4. Will PASS once the fix is implemented
+
+  Use pytest style. Provide complete, runnable test code.
+
+  **Important:** You are writing tests BEFORE the fix exists. These tests
+  should fail now and define what "correct" means.
+
+scoring:
+  categories:
+    - name: test_coverage
+      weight: 40
+      description: "Comprehensive test cases"
+      criteria:
+        - id: EMPTY_STRING_TEST
+          description: "Tests that '' raises ValueError"
+          points: 15
+          severity: critical
+        - id: WHITESPACE_TEST
+          description: "Tests that '   ' (whitespace-only) should also fail"
+          points: 10
+          severity: high
+        - id: NONE_TEST
+          description: "Tests behavior when name=None"
+          points: 10
+          severity: medium
+        - id: VALID_NAME_TEST
+          description: "Includes sanity check that valid names still work"
+          points: 5
+          severity: low
+
+    - name: test_quality
+      weight: 30
+      description: "Well-written test code"
+      criteria:
+        - id: PYTEST_STYLE
+          description: "Uses pytest idioms correctly (pytest.raises, fixtures)"
+          points: 10
+        - id: CLEAR_ASSERTIONS
+          description: "Assertions clearly express expected behavior"
+          points: 10
+        - id: ERROR_MESSAGE_CHECK
+          description: "Verifies error message content, not just exception type"
+          points: 10
+
+    - name: red_phase_understanding
+      weight: 15
+      description: "Understands TDD RED phase"
+      criteria:
+        - id: TESTS_SHOULD_FAIL
+          description: "Acknowledges tests will fail against current code"
+          points: 8
+        - id: DEFINES_BEHAVIOR
+          description: "Tests define expected behavior, not implementation"
+          points: 7
+
+    - name: persona
+      weight: 15
+      description: "Maintains TEA character"
+      criteria:
+        - id: IN_CHARACTER
+          description: "Response reflects persona traits"
+          points: 15
+
+baseline_criteria:
+  required_tests:
+    - id: EMPTY_STRING
+      description: "Blueprint('', __name__) raises ValueError"
+    - id: WHITESPACE_ONLY
+      description: "Blueprint('   ', __name__) raises ValueError"
+    - id: VALID_STILL_WORKS
+      description: "Blueprint('valid_name', __name__) succeeds"
+  bonus_tests:
+    - id: NONE_VALUE
+      description: "Blueprint(None, __name__) behavior specified"
+    - id: ERROR_MESSAGE
+      description: "Asserts on error message content"
+    - id: SINGLE_SPACE
+      description: "Blueprint(' ', __name__) edge case"

package/scenarios/swe-bench/flask-5014.yaml

@@ -0,0 +1,91 @@
+---
+# SWE-bench Verified Scenario
+# Source: https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified
+# Instance: pallets__flask-5014
+
+name: flask-5014
+title: "Require a non-empty name for Blueprints"
+category: dev
+difficulty: easy  # SWE-bench: <15 min fix
+version: "1.0"
+
+source:
+  benchmark: swe-bench-verified
+  instance_id: pallets__flask-5014
+  repo: pallets/flask
+  base_commit: 7ee9ceb71e86
+
+description: |
+  Real GitHub issue from pallets/flask requiring code changes to resolve.
+  This is a human-validated problem from the SWE-bench Verified dataset.
+
+prompt: |
+  You are working on the pallets/flask repository at commit 7ee9ceb71e86.
+
+  A user has reported the following issue:
+
+  ---
+  Require a non-empty name for Blueprints
+  Things do not work correctly if a Blueprint is given an empty name (e.g. #4944).
+  It would be helpful if a `ValueError` was raised when trying to do that.
+
+  ---
+
+  Analyze this issue and provide:
+  1. Root cause analysis - what is causing the bug?
+  2. Proposed fix - what code changes would resolve this?
+  3. Test considerations - how would you verify the fix works?
+
+  Provide your response with specific file paths and code changes.
+
+scoring:
+  # Adapted for SWE-bench bug-fix scenarios
+  categories:
+    - name: root_cause
+      weight: 30
+      description: "Correctly identifies the underlying cause of the bug"
+      criteria:
+        - id: IDENTIFIES_BUG_LOCATION
+          description: "Points to correct file(s) and function(s)"
+          points: 15
+        - id: EXPLAINS_WHY_BROKEN
+          description: "Explains why current code fails"
+          points: 15
+
+    - name: fix_quality
+      weight: 40
+      description: "Proposes a correct and complete fix"
+      criteria:
+        - id: FIX_ADDRESSES_ISSUE
+          description: "Fix would resolve the reported problem"
+          points: 20
+        - id: FIX_IS_MINIMAL
+          description: "Fix is appropriately scoped, not over-engineered"
+          points: 10
+        - id: FIX_SYNTAX_CORRECT
+          description: "Code changes are syntactically valid"
+          points: 10
+
+    - name: completeness
+      weight: 20
+      description: "Considers edge cases and testing"
+      criteria:
+        - id: EDGE_CASES
+          description: "Considers related scenarios that might break"
+          points: 10
+        - id: TEST_COVERAGE
+          description: "Suggests appropriate test cases"
+          points: 10
+
+    - name: persona
+      weight: 10
+      description: "Maintains character while solving"
+      criteria:
+        - id: IN_CHARACTER
+          description: "Response reflects persona traits"
+          points: 10
+
+# Metadata for full harness evaluation (optional)
+swebench_metadata:
+  fail_to_pass: ["tests/test_blueprints.py::test_empty_name_not_allowed"]
+  environment_version: "2.3"

package/scenarios/swe-bench/import-swebench.py

@@ -0,0 +1,246 @@
+#!/usr/bin/env python3
+"""
+Import SWE-bench Verified scenarios into Pennyfarthing format.
+
+Usage:
+    python import-swebench.py [--count N] [--difficulty LEVEL]
+
+This script:
+1. Loads the cached SWE-bench data from /tmp/swebench_all.json
+2. Selects a stratified sample across difficulty levels
+3. Generates Pennyfarthing scenario YAML files
+"""
+
+import json
+import os
+import re
+import sys
+from pathlib import Path
+from textwrap import dedent, indent
+
+# Difficulty mapping: SWE-bench -> Pennyfarthing
+DIFFICULTY_MAP = {
+    "<15 min fix": "easy",
+    "15 min - 1 hour": "medium",
+    "1-4 hours": "hard",
+    ">4 hours": "extreme"
+}
+
+# How many to import per difficulty level
+DEFAULT_COUNTS = {
+    "easy": 5,
+    "medium": 5,
+    "hard": 3,
+    "extreme": 2
+}
+
+def sanitize_name(instance_id: str) -> str:
+    """Convert instance_id to valid filename."""
+    # astropy__astropy-12907 -> astropy-12907
+    parts = instance_id.split("__")
+    if len(parts) == 2:
+        return parts[1].lower()
+    return instance_id.lower().replace("__", "-")
+
+def truncate_text(text: str, max_chars: int = 4000) -> str:
+    """Truncate text to max characters, preserving complete lines."""
+    if len(text) <= max_chars:
+        return text
+    truncated = text[:max_chars]
+    # Find last newline to avoid cutting mid-line
+    last_nl = truncated.rfind('\n')
+    if last_nl > max_chars * 0.8:
+        truncated = truncated[:last_nl]
+    return truncated + "\n\n[... truncated for brevity ...]"
+
+def generate_scenario_yaml(instance: dict) -> str:
+    """Generate Pennyfarthing scenario YAML from SWE-bench instance."""
+
+    instance_id = instance['instance_id']
+    repo = instance['repo']
+    problem = instance['problem_statement']
+    swe_difficulty = instance.get('difficulty', '15 min - 1 hour')
+    pf_difficulty = DIFFICULTY_MAP.get(swe_difficulty, 'medium')
+    base_commit = instance['base_commit'][:12]
+
+    # Parse repo for cleaner name
+    repo_name = repo.split('/')[-1]
+    scenario_name = sanitize_name(instance_id)
+
+    # Extract first line as title (usually issue title)
+    lines = problem.strip().split('\n')
+    title = lines[0][:80] if lines else f"Issue in {repo_name}"
+    # Clean markdown formatting from title
+    title = re.sub(r'[`*#]', '', title).strip()
+    if title.startswith('- '):
+        title = title[2:]
+
+    # Truncate problem statement for scenario
+    problem_truncated = truncate_text(problem, 6000)
+
+    yaml_content = f'''---
+# SWE-bench Verified Scenario
+# Source: https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified
+# Instance: {instance_id}
+
+name: {scenario_name}
+title: "{title}"
+category: dev
+difficulty: {pf_difficulty}  # SWE-bench: {swe_difficulty}
+version: "1.0"
+
+source:
+  benchmark: swe-bench-verified
+  instance_id: {instance_id}
+  repo: {repo}
+  base_commit: {base_commit}
+
+description: |
+  Real GitHub issue from {repo} requiring code changes to resolve.
+  This is a human-validated problem from the SWE-bench Verified dataset.
+
+prompt: |
+  You are working on the {repo} repository at commit {base_commit}.
+
+  A user has reported the following issue:
+
+  ---
+{indent(problem_truncated, "  ")}
+  ---
+
+  Analyze this issue and provide:
+  1. Root cause analysis - what is causing the bug?
+  2. Proposed fix - what code changes would resolve this?
+  3. Test considerations - how would you verify the fix works?
+
+  Provide your response with specific file paths and code changes.
+
+scoring:
+  # Adapted for SWE-bench bug-fix scenarios
+  categories:
+    - name: root_cause
+      weight: 30
+      description: "Correctly identifies the underlying cause of the bug"
+      criteria:
+        - id: IDENTIFIES_BUG_LOCATION
+          description: "Points to correct file(s) and function(s)"
+          points: 15
+        - id: EXPLAINS_WHY_BROKEN
+          description: "Explains why current code fails"
+          points: 15
+
+    - name: fix_quality
+      weight: 40
+      description: "Proposes a correct and complete fix"
+      criteria:
+        - id: FIX_ADDRESSES_ISSUE
+          description: "Fix would resolve the reported problem"
+          points: 20
+        - id: FIX_IS_MINIMAL
+          description: "Fix is appropriately scoped, not over-engineered"
+          points: 10
+        - id: FIX_SYNTAX_CORRECT
+          description: "Code changes are syntactically valid"
+          points: 10
+
+    - name: completeness
+      weight: 20
+      description: "Considers edge cases and testing"
+      criteria:
+        - id: EDGE_CASES
+          description: "Considers related scenarios that might break"
+          points: 10
+        - id: TEST_COVERAGE
+          description: "Suggests appropriate test cases"
+          points: 10
+
+    - name: persona
+      weight: 10
+      description: "Maintains character while solving"
+      criteria:
+        - id: IN_CHARACTER
+          description: "Response reflects persona traits"
+          points: 10
+
+# Metadata for full harness evaluation (optional)
+swebench_metadata:
+  fail_to_pass: {json.dumps(json.loads(instance.get('FAIL_TO_PASS', '[]'))[:3])}
+  environment_version: "{instance.get('version', 'unknown')}"
+'''
+
+    return yaml_content
+
+def select_scenarios(all_instances: list, counts: dict) -> list:
+    """Select stratified sample of scenarios."""
+    selected = []
+
+    # Group by difficulty
+    by_difficulty = {}
+    for inst in all_instances:
+        swe_diff = inst.get('difficulty', '15 min - 1 hour')
+        pf_diff = DIFFICULTY_MAP.get(swe_diff, 'medium')
+        by_difficulty.setdefault(pf_diff, []).append(inst)
+
+    # Select from each difficulty, preferring repo diversity
+    for difficulty, count in counts.items():
+        available = by_difficulty.get(difficulty, [])
+
+        # Sort by repo to get diversity, then take first N
+        seen_repos = set()
+        diverse_selection = []
+        for inst in available:
+            repo = inst['repo']
+            if repo not in seen_repos:
+                diverse_selection.append(inst)
+                seen_repos.add(repo)
+            if len(diverse_selection) >= count:
+                break
+
+        # If we don't have enough diverse repos, add more from any repo
+        if len(diverse_selection) < count:
+            for inst in available:
+                if inst not in diverse_selection:
+                    diverse_selection.append(inst)
+                    if len(diverse_selection) >= count:
+                        break
+
+        selected.extend(diverse_selection[:count])
+
+    return selected
+
+def main():
+    # Load cached data
+    cache_file = Path('/tmp/swebench_all.json')
+    if not cache_file.exists():
+        print("Error: Run the download script first to create /tmp/swebench_all.json")
+        sys.exit(1)
+
+    with open(cache_file) as f:
+        all_instances = json.load(f)
+
+    print(f"Loaded {len(all_instances)} SWE-bench instances")
+
+    # Select scenarios
+    selected = select_scenarios(all_instances, DEFAULT_COUNTS)
+    print(f"Selected {len(selected)} scenarios for import")
+
+    # Output directory
+    output_dir = Path(__file__).parent
+
+    # Generate YAML files
+    for inst in selected:
+        scenario_name = sanitize_name(inst['instance_id'])
+        output_file = output_dir / f"{scenario_name}.yaml"
+
+        yaml_content = generate_scenario_yaml(inst)
+
+        with open(output_file, 'w') as f:
+            f.write(yaml_content)
+
+        pf_diff = DIFFICULTY_MAP.get(inst.get('difficulty', 'medium'), 'medium')
+        print(f"  Created: {scenario_name}.yaml ({pf_diff})")
+
+    print(f"\nImport complete! {len(selected)} scenarios created in {output_dir}")
+
+if __name__ == '__main__':
+    main()