@pennyfarthing/benchmark 10.2.0

package/scenarios/debugging/simple-logic-error.yaml

@@ -0,0 +1,115 @@
+---
+# Scenario: Simple Logic Errors
+# Category: debugging
+# Difficulty: easy
+# Error Type Focus: reasoning (single-type)
+
+id: debug-003
+name: simple-logic-error
+title: "Logic Errors: When the Code Does What You Said, Not What You Meant"
+category: debugging
+difficulty: easy
+version: "1.0"
+
+description: |
+  A discount calculator with straightforward logic errors.
+  Tests detection of incorrect boolean logic and comparisons.
+
+purpose: |
+  This scenario measures detection of reasoning-level bugs - where the
+  logic itself is flawed. Analytical agents will trace through the logic.
+  Pattern-matching agents might miss subtle comparison errors.
+
+prompt: |
+  BUG REPORT
+
+  Service: discount-calculator
+  Severity: P2
+  Status: Customers getting wrong discount amounts
+
+  The discount calculator is applying incorrect discounts:
+  - Premium members sometimes don't get their discount
+  - Some orders qualify for multiple discounts when they shouldn't
+
+  Your task:
+  1. Trace through the logic and find the errors
+  2. Explain what the code does vs what it should do
+  3. Provide the corrected logic
+
+  There are 5 known issues. How many can you find?
+
+code:
+  language: python
+  filename: discount_calculator.py
+  content: |
+    class DiscountCalculator:
+        def calculate_discount(self, order_total, is_premium, items_count):
+            """Calculate discount based on order and membership."""
+            discount = 0
+
+            # Premium member discount (should be 10%)
+            # Bug: using assignment instead of comparison
+            if is_premium = True:
+                discount = 0.10
+
+            # Bulk discount (should apply for 10+ items)
+            # Bug: wrong comparison operator
+            if items_count > 10:  # Should be >= 10
+                discount = max(discount, 0.15)
+
+            # Large order discount (should apply for $100+)
+            # Bug: inverted condition
+            if order_total < 100:  # Should be >= 100
+                discount = max(discount, 0.20)
+
+            return discount
+
+        def is_eligible_for_free_shipping(self, order_total, distance_miles):
+            """Check if order qualifies for free shipping."""
+            # Bug: AND should be OR (either condition should qualify)
+            if order_total >= 50 and distance_miles <= 100:
+                return True
+            return False
+
+        def calculate_final_price(self, original_price, discount_percent):
+            """Apply discount and return final price."""
+            # Bug: adding discount instead of subtracting
+            discount_amount = original_price * discount_percent
+            final_price = original_price + discount_amount
+            return final_price
+
+baseline_issues:
+  critical:
+    - id: logic-001
+      location: "line 9"
+      description: "Using = (assignment) instead of == (comparison)"
+      error_type: reasoning
+  high:
+    - id: logic-002
+      location: "line 13"
+      description: "Using > instead of >= excludes boundary case of exactly 10 items"
+      error_type: reasoning
+    - id: logic-003
+      location: "line 18"
+      description: "Condition is inverted - applies discount for small orders instead of large"
+      error_type: reasoning
+  medium:
+    - id: logic-004
+      location: "line 25"
+      description: "Using AND instead of OR - both conditions required instead of either"
+      error_type: reasoning
+    - id: logic-005
+      location: "line 33"
+      description: "Adding discount instead of subtracting - increases price"
+      error_type: reasoning
+
+scoring:
+  detection:
+    weight: 50
+    criteria: "Finding all 5 logic errors"
+  fix_quality:
+    weight: 30
+    criteria: "Providing correct boolean/arithmetic fixes"
+  explanation:
+    weight: 20
+    criteria: "Explaining expected vs actual behavior"

package/scenarios/debugging/sql-injection.yaml

@@ -0,0 +1,163 @@
+---
+# Scenario: SQL Injection Vulnerabilities
+# Category: debugging
+# Difficulty: hard
+# Error Type Focus: reasoning (security analysis)
+
+id: debug-009
+name: sql-injection
+title: "SQL Injection: The Database Destroyer"
+category: debugging
+difficulty: hard
+version: "1.0"
+
+description: |
+  A product search API with multiple SQL injection vulnerabilities.
+  Tests detection of injection points and understanding of attack vectors.
+
+purpose: |
+  This scenario measures reasoning about security vulnerabilities. Agents
+  must understand how untrusted input flows to SQL queries and identify
+  all injection points, including subtle ones.
+
+prompt: |
+  SECURITY AUDIT
+
+  Service: product-search
+  Severity: P0
+  Status: SQL injection vulnerabilities discovered
+
+  A security audit found the product search API is vulnerable:
+  - Direct string concatenation in queries
+  - Insufficient input sanitization
+  - Dynamic query building from user input
+
+  Your task:
+  1. Find all SQL injection vulnerabilities
+  2. Demonstrate how each could be exploited
+  3. Implement parameterized queries
+
+  There are 7 known issues. How many can you find?
+
+code:
+  language: python
+  filename: product_search.py
+  content: |
+    import sqlite3
+    from typing import List, Optional
+
+    class ProductSearch:
+        def __init__(self, db_path: str):
+            self.conn = sqlite3.connect(db_path)
+
+        def search_by_name(self, name: str) -> List[dict]:
+            """Search products by name."""
+            # Bug: Direct string concatenation
+            query = f"SELECT * FROM products WHERE name LIKE '%{name}%'"
+            cursor = self.conn.execute(query)
+            return [dict(row) for row in cursor.fetchall()]
+
+        def get_by_id(self, product_id: str) -> Optional[dict]:
+            """Get product by ID."""
+            # Bug: Even numeric IDs can be injected
+            query = f"SELECT * FROM products WHERE id = {product_id}"
+            cursor = self.conn.execute(query)
+            row = cursor.fetchone()
+            return dict(row) if row else None
+
+        def search_by_category(self, category: str, min_price: float) -> List[dict]:
+            """Search by category with minimum price."""
+            # Bug: Multiple injection points
+            query = f"""
+                SELECT * FROM products
+                WHERE category = '{category}'
+                AND price >= {min_price}
+            """
+            cursor = self.conn.execute(query)
+            return [dict(row) for row in cursor.fetchall()]
+
+        def advanced_search(self, filters: dict) -> List[dict]:
+            """Build dynamic query from filters."""
+            query = "SELECT * FROM products WHERE 1=1"
+
+            # Bug: Dynamic query building from untrusted input
+            for key, value in filters.items():
+                query += f" AND {key} = '{value}'"
+
+            cursor = self.conn.execute(query)
+            return [dict(row) for row in cursor.fetchall()]
+
+        def search_with_sort(self, search_term: str, sort_by: str, order: str) -> List[dict]:
+            """Search with custom sort."""
+            # Bug: ORDER BY injection (parameterization doesn't work here)
+            query = f"""
+                SELECT * FROM products
+                WHERE name LIKE '%{search_term}%'
+                ORDER BY {sort_by} {order}
+            """
+            cursor = self.conn.execute(query)
+            return [dict(row) for row in cursor.fetchall()]
+
+        def bulk_lookup(self, ids: List[str]) -> List[dict]:
+            """Look up multiple products by ID."""
+            # Bug: IN clause built from user input
+            id_list = ",".join(ids)
+            query = f"SELECT * FROM products WHERE id IN ({id_list})"
+            cursor = self.conn.execute(query)
+            return [dict(row) for row in cursor.fetchall()]
+
+        def authenticated_search(self, user_id: int, search_term: str) -> List[dict]:
+            """Search products user has access to."""
+            # Bug: Assuming user_id is safe because it's an int parameter
+            query = f"""
+                SELECT p.* FROM products p
+                JOIN user_products up ON p.id = up.product_id
+                WHERE up.user_id = {user_id}
+                AND p.name LIKE '%{search_term}%'
+            """
+            cursor = self.conn.execute(query)
+            return [dict(row) for row in cursor.fetchall()]
+
+baseline_issues:
+  critical:
+    - id: sqli-001
+      location: "line 12"
+      description: "LIKE clause with string concatenation - searchable injection"
+      error_type: reasoning
+    - id: sqli-002
+      location: "lines 38-41"
+      description: "Dynamic column names from user input - full query control"
+      error_type: reasoning
+  high:
+    - id: sqli-003
+      location: "line 19"
+      description: "Numeric ID not parameterized - type juggling bypass"
+      error_type: reasoning
+    - id: sqli-004
+      location: "lines 26-30"
+      description: "Multiple injection points in single query"
+      error_type: reasoning
+    - id: sqli-005
+      location: "lines 47-51"
+      description: "ORDER BY clause injection - can leak data via sorting"
+      error_type: reasoning
+  medium:
+    - id: sqli-006
+      location: "line 58"
+      description: "IN clause built from array - each element injectable"
+      error_type: reasoning
+    - id: sqli-007
+      location: "lines 65-70"
+      description: "User-controlled search term despite authenticated context"
+      error_type: reasoning
+
+scoring:
+  detection:
+    weight: 40
+    criteria: "Finding all 7 injection vulnerabilities"
+  fix_quality:
+    weight: 35
+    criteria: "Implementing parameterized queries correctly"
+  explanation:
+    weight: 25
+    criteria: "Demonstrating exploitation scenarios"