@pennyfarthing/benchmark 10.2.0

package/scenarios/debug/buggy-user-service.yaml

@@ -0,0 +1,541 @@
+---
+# Scenario: Buggy User Service Fix
+# Category: dev (Developer)
+# Ported from: Pennyfarthing benchmarks/test-cases/dev/dev-001-buggy-service.yaml
+# Purpose: Measure thoroughness in bug detection and fix quality
+
+id: dev-001
+name: buggy-user-service
+title: "Buggy User Service: Fix the Authentication Nightmare"
+category: dev
+difficulty: hard
+version: "1.0"
+
+description: |
+  A user service with multiple bugs across authentication, data handling,
+  and business logic. Tests the developer agent's ability to identify issues,
+  propose fixes, and consider edge cases during implementation.
+
+purpose: |
+  This scenario tests whether persona traits affect bug detection.
+  A "security-minded" developer might prioritize injection attacks.
+  A "thorough" developer might find more total issues.
+  A "practical" developer might focus on high-impact fixes first.
+  All are valid but measurably different.
+
+prompt: |
+  INCIDENT REPORT
+
+  Severity: P1
+  Service: user-service
+  Status: Security audit flagged multiple critical issues
+
+  The security team has flagged this user service for immediate remediation.
+  Multiple vulnerabilities were detected in a penetration test.
+
+  Your task:
+  1. Review the code and identify ALL security and quality issues
+  2. For each issue:
+     - Identify the bug and its location
+     - Explain the impact (how could this be exploited?)
+     - Provide the corrected code
+     - Consider edge cases your fix must handle
+  3. Prioritize by severity (Critical > High > Medium > Low)
+
+  Focus on:
+  - Security vulnerabilities (injection, auth bypass, etc.)
+  - Logic errors
+  - Error handling gaps
+  - Edge cases
+  - Best practices violations
+
+  Be thorough. This code handles user authentication and personal data.
+  There are 33 known issues (22 baseline + 11 bonus). How many can you find?
+
+code:
+  language: go
+  filename: user_service.go
+  content: |
+    package users
+
+    import (
+        "crypto/md5"
+        "database/sql"
+        "encoding/hex"
+        "encoding/json"
+        "fmt"
+        "net/http"
+        "regexp"
+        "strings"
+        "time"
+    )
+
+    type UserService struct {
+        db *sql.DB
+    }
+
+    type User struct {
+        ID           int64     `json:"id"`
+        Email        string    `json:"email"`
+        PasswordHash string    `json:"-"`
+        Name         string    `json:"name"`
+        Role         string    `json:"role"`
+        CreatedAt    time.Time `json:"created_at"`
+        LastLogin    time.Time `json:"last_login"`
+        FailedLogins int       `json:"-"`
+        Locked       bool      `json:"locked"`
+    }
+
+    // RegisterUser creates a new user account
+    func (s *UserService) RegisterUser(w http.ResponseWriter, r *http.Request) {
+        var req struct {
+            Email    string `json:"email"`
+            Password string `json:"password"`
+            Name     string `json:"name"`
+        }
+        json.NewDecoder(r.Body).Decode(&req)
+
+        // Validate email
+        if !strings.Contains(req.Email, "@") {
+            http.Error(w, "Invalid email", 400)
+            return
+        }
+
+        // Hash password
+        hash := md5.Sum([]byte(req.Password))
+        passwordHash := hex.EncodeToString(hash[:])
+
+        // Create user
+        result, _ := s.db.Exec(
+            fmt.Sprintf("INSERT INTO users (email, password_hash, name, role) VALUES ('%s', '%s', '%s', 'user')",
+                req.Email, passwordHash, req.Name))
+
+        id, _ := result.LastInsertId()
+
+        w.Write([]byte(fmt.Sprintf(`{"id": %d, "message": "User created"}`, id)))
+    }
+
+    // Login authenticates a user
+    func (s *UserService) Login(w http.ResponseWriter, r *http.Request) {
+        var req struct {
+            Email    string `json:"email"`
+            Password string `json:"password"`
+        }
+        json.NewDecoder(r.Body).Decode(&req)
+
+        var user User
+        query := fmt.Sprintf("SELECT id, email, password_hash, role, locked, failed_logins FROM users WHERE email = '%s'", req.Email)
+        row := s.db.QueryRow(query)
+        row.Scan(&user.ID, &user.Email, &user.PasswordHash, &user.Role, &user.Locked, &user.FailedLogins)
+
+        // Check password
+        hash := md5.Sum([]byte(req.Password))
+        if hex.EncodeToString(hash[:]) != user.PasswordHash {
+            user.FailedLogins++
+            s.db.Exec("UPDATE users SET failed_logins = ? WHERE id = ?", user.FailedLogins, user.ID)
+            http.Error(w, "Invalid credentials", 401)
+            return
+        }
+
+        // Generate session token
+        token := fmt.Sprintf("%d-%d", user.ID, time.Now().Unix())
+
+        // Update last login
+        s.db.Exec("UPDATE users SET last_login = NOW(), failed_logins = 0 WHERE id = ?", user.ID)
+
+        json.NewEncoder(w).Encode(map[string]interface{}{
+            "token": token,
+            "user":  user,
+        })
+    }
+
+    // UpdateProfile allows users to update their profile
+    func (s *UserService) UpdateProfile(w http.ResponseWriter, r *http.Request) {
+        userID := r.Header.Get("X-User-ID")
+
+        var req struct {
+            Name  string `json:"name"`
+            Email string `json:"email"`
+            Role  string `json:"role"`
+        }
+        json.NewDecoder(r.Body).Decode(&req)
+
+        s.db.Exec(fmt.Sprintf(
+            "UPDATE users SET name = '%s', email = '%s', role = '%s' WHERE id = %s",
+            req.Name, req.Email, req.Role, userID))
+
+        w.Write([]byte(`{"message": "Profile updated"}`))
+    }
+
+    // ResetPassword handles password reset
+    func (s *UserService) ResetPassword(w http.ResponseWriter, r *http.Request) {
+        var req struct {
+            Email       string `json:"email"`
+            ResetToken  string `json:"reset_token"`
+            NewPassword string `json:"new_password"`
+        }
+        json.NewDecoder(r.Body).Decode(&req)
+
+        // Verify reset token
+        var storedToken string
+        s.db.QueryRow("SELECT reset_token FROM users WHERE email = ?", req.Email).Scan(&storedToken)
+
+        if req.ResetToken == storedToken {
+            hash := md5.Sum([]byte(req.NewPassword))
+            passwordHash := hex.EncodeToString(hash[:])
+            s.db.Exec("UPDATE users SET password_hash = ? WHERE email = ?", passwordHash, req.Email)
+            w.Write([]byte(`{"message": "Password reset successful"}`))
+        } else {
+            http.Error(w, "Invalid reset token", 400)
+        }
+    }
+
+    // DeleteUser removes a user account
+    func (s *UserService) DeleteUser(w http.ResponseWriter, r *http.Request) {
+        userID := r.URL.Query().Get("id")
+
+        s.db.Exec("DELETE FROM users WHERE id = " + userID)
+        s.db.Exec("DELETE FROM user_sessions WHERE user_id = " + userID)
+        s.db.Exec("DELETE FROM user_preferences WHERE user_id = " + userID)
+
+        w.Write([]byte(`{"message": "User deleted"}`))
+    }
+
+    // SearchUsers finds users matching criteria
+    func (s *UserService) SearchUsers(w http.ResponseWriter, r *http.Request) {
+        query := r.URL.Query().Get("q")
+        role := r.URL.Query().Get("role")
+
+        sql := fmt.Sprintf("SELECT id, email, name, role FROM users WHERE name LIKE '%%%s%%'", query)
+        if role != "" {
+            sql += fmt.Sprintf(" AND role = '%s'", role)
+        }
+
+        rows, _ := s.db.Query(sql)
+        var users []User
+        for rows.Next() {
+            var u User
+            rows.Scan(&u.ID, &u.Email, &u.Name, &u.Role)
+            users = append(users, u)
+        }
+
+        json.NewEncoder(w).Encode(users)
+    }
+
+    // ValidatePassword checks password strength
+    func (s *UserService) ValidatePassword(password string) bool {
+        if len(password) < 8 {
+            return false
+        }
+        hasUpper := regexp.MustCompile(`[A-Z]`).MatchString(password)
+        hasLower := regexp.MustCompile(`[a-z]`).MatchString(password)
+        hasNumber := regexp.MustCompile(`[0-9]`).MatchString(password)
+        return hasUpper && hasLower && hasNumber
+    }
+
+    // GetUserByID retrieves a user
+    func (s *UserService) GetUserByID(w http.ResponseWriter, r *http.Request) {
+        id := r.URL.Query().Get("id")
+
+        var user User
+        s.db.QueryRow(fmt.Sprintf("SELECT * FROM users WHERE id = %s", id)).
+            Scan(&user.ID, &user.Email, &user.PasswordHash, &user.Name, &user.Role)
+
+        json.NewEncoder(w).Encode(user)
+    }
+
+    // BulkImportUsers imports users from JSON
+    func (s *UserService) BulkImportUsers(w http.ResponseWriter, r *http.Request) {
+        var users []struct {
+            Email    string `json:"email"`
+            Password string `json:"password"`
+            Name     string `json:"name"`
+            Role     string `json:"role"`
+        }
+        json.NewDecoder(r.Body).Decode(&users)
+
+        for _, u := range users {
+            hash := md5.Sum([]byte(u.Password))
+            passwordHash := hex.EncodeToString(hash[:])
+            s.db.Exec(fmt.Sprintf(
+                "INSERT INTO users (email, password_hash, name, role) VALUES ('%s', '%s', '%s', '%s')",
+                u.Email, passwordHash, u.Name, u.Role))
+        }
+
+        w.Write([]byte(fmt.Sprintf(`{"imported": %d}`, len(users))))
+    }
+
+# =============================================================================
+# BASELINE ISSUES (minimum expected to find and fix)
+# =============================================================================
+
+baseline_issues:
+  critical:
+    - id: SQL_INJECTION_REGISTER
+      location: "lines 50-52"
+      description: "SQL injection via string formatting in RegisterUser"
+
+    - id: SQL_INJECTION_LOGIN
+      location: "line 66"
+      description: "SQL injection in Login query"
+
+    - id: SQL_INJECTION_UPDATE
+      location: "lines 96-98"
+      description: "SQL injection in UpdateProfile"
+
+    - id: SQL_INJECTION_DELETE
+      location: "lines 125-127"
+      description: "SQL injection in DeleteUser"
+
+    - id: SQL_INJECTION_SEARCH
+      location: "lines 135-139"
+      description: "SQL injection in SearchUsers"
+
+    - id: WEAK_PASSWORD_HASH
+      location: "lines 46, 70, 112, 177"
+      description: "Using MD5 for password hashing (cryptographically broken)"
+
+  high:
+    - id: INSECURE_SESSION_TOKEN
+      location: "line 78"
+      description: "Predictable session token (user_id + timestamp)"
+
+    - id: ROLE_ESCALATION
+      location: "lines 89-98"
+      description: "User can set their own role in UpdateProfile"
+
+    - id: NO_ACCOUNT_LOCKOUT
+      location: "lines 63-82"
+      description: "No account lockout after failed logins (locked field not checked)"
+
+    - id: PASSWORD_IN_RESPONSE
+      location: "line 83"
+      description: "User struct may leak password hash if not properly excluded"
+
+    - id: NO_AUTH_DELETE
+      location: "lines 120-130"
+      description: "DeleteUser has no authorization check"
+
+    - id: NO_AUTH_SEARCH
+      location: "lines 133-152"
+      description: "SearchUsers exposes all user data without auth"
+
+  medium:
+    - id: WEAK_EMAIL_VALIDATION
+      location: "lines 41-44"
+      description: "Email validation only checks for @ symbol"
+
+    - id: PASSWORD_NOT_VALIDATED
+      location: "RegisterUser"
+      description: "ValidatePassword function exists but not called"
+
+    - id: TIMING_ATTACK_RESET
+      location: "lines 108-117"
+      description: "Reset token comparison vulnerable to timing attack"
+
+    - id: TOKEN_NOT_INVALIDATED
+      location: "ResetPassword"
+      description: "Reset token not invalidated after use"
+
+    - id: ERROR_IGNORED_DECODE
+      location: "multiple"
+      description: "JSON decode errors ignored throughout"
+
+    - id: ROWS_NOT_CLOSED
+      location: "line 140"
+      description: "Database rows not closed in SearchUsers"
+
+  low:
+    - id: MISSING_CONTENT_TYPE
+      location: "multiple"
+      description: "JSON responses don't set Content-Type header"
+
+    - id: NO_INPUT_LENGTH_LIMITS
+      location: "multiple"
+      description: "No limits on input field lengths"
+
+    - id: INCONSISTENT_ERROR_RESPONSES
+      location: "multiple"
+      description: "Mix of http.Error and json responses"
+
+    - id: SQL_INJECTION_GETBYID
+      location: "line 162"
+      description: "SQL injection in GetUserByID"
+
+# =============================================================================
+# BONUS ISSUES (thorough developers might address)
+# =============================================================================
+
+bonus_issues:
+  security:
+    - id: NO_RATE_LIMITING
+      description: "No rate limiting on login/register endpoints"
+
+    - id: NO_CSRF_PROTECTION
+      description: "No CSRF tokens for state-changing operations"
+
+    - id: CREDENTIALS_IN_LOGS
+      description: "Errors could log sensitive data"
+
+    - id: NO_HTTPS_ENFORCEMENT
+      description: "No check for secure connection"
+
+  reliability:
+    - id: NO_TRANSACTION_DELETE
+      description: "DeleteUser should use transaction for multiple deletes"
+
+    - id: NO_CONTEXT_TIMEOUT
+      description: "No context/timeout on database operations"
+
+    - id: NO_CONNECTION_POOLING_CONFIG
+      description: "Database connection pooling not configured"
+
+  code_quality:
+    - id: DUPLICATE_HASH_LOGIC
+      description: "Password hashing duplicated in 4 places"
+
+    - id: MAGIC_STRINGS
+      description: "Role values as magic strings"
+
+    - id: NO_CONSTANTS
+      description: "No constants for error messages"
+
+    - id: MISSING_INDEXES
+      description: "Queries suggest missing database indexes"
+
+# =============================================================================
+# SCORING
+# =============================================================================
+
+scoring:
+  total_baseline_issues: 22
+  total_bonus_issues: 11
+  weights:
+    critical: 3
+    high: 2
+    medium: 1
+    low: 0.5
+  max_baseline_score: 33.5  # 6*3 + 6*2 + 6*1 + 4*0.5
+
+  categories:
+    - name: detection
+      weight: 40
+      description: "How many issues are found"
+      criteria:
+        - id: CRITICAL_FOUND
+          description: "All 6 critical issues found"
+          points: 20
+        - id: HIGH_FOUND
+          description: "All 6 high issues found"
+          points: 12
+        - id: MEDIUM_LOW_FOUND
+          description: "Medium and low issues found"
+          points: 8
+
+    - name: fix_quality
+      weight: 30
+      description: "Quality of proposed fixes"
+      criteria:
+        - id: CORRECT_FIXES
+          description: "Fixes actually solve the problem"
+          points: 15
+        - id: EDGE_CASES_HANDLED
+          description: "Fixes handle edge cases"
+          points: 10
+        - id: NO_NEW_BUGS
+          description: "Fixes don't introduce new issues"
+          points: 5
+
+    - name: explanation
+      weight: 15
+      description: "Quality of issue explanations"
+      criteria:
+        - id: IMPACT_EXPLAINED
+          description: "Explains real-world impact"
+          points: 8
+        - id: ROOT_CAUSE
+          description: "Identifies root cause, not just symptom"
+          points: 7
+
+    - name: persona
+      weight: 15
+      description: "Persona consistency and value"
+      criteria:
+        - id: IN_CHARACTER
+          description: "Stays in character throughout"
+          points: 8
+        - id: PERSONA_ENHANCES
+          description: "Persona adds value to explanations"
+          points: 7
+
+# =============================================================================
+# ENHANCED METRICS
+# =============================================================================
+
+enhanced_metrics:
+  thoroughness_ratio:
+    formula: "total_findings / 22"
+    interpretation: "100% = found all baseline issues"
+
+  bonus_discovery_rate:
+    formula: "bonus_found / 11"
+    interpretation: "Shows exceptional thoroughness"
+
+  fix_accuracy:
+    formula: "correct_fixes / issues_found"
+    interpretation: "100% = all fixes are correct"
+
+  severity_accuracy:
+    formula: "correctly_classified / issues_found"
+    interpretation: "100% = perfect severity classification"
+
+# =============================================================================
+# PERSONA INFLUENCE
+# =============================================================================
+
+persona_influence:
+  dimensions:
+    - name: issue_prioritization
+      description: "What types of issues are found first"
+      spectrum:
+        security_first: "SQL injection and auth issues prioritized"
+        quality_first: "Code quality and maintainability first"
+        impact_first: "Highest business impact first"
+
+    - name: fix_style
+      description: "How comprehensive are the fixes"
+      spectrum:
+        minimal: "Just fixes the immediate problem"
+        refactoring: "Cleans up surrounding code"
+        architectural: "Suggests broader improvements"
+
+    - name: documentation
+      description: "How well issues are explained"
+      spectrum:
+        brief: "Issue and fix only"
+        detailed: "Full impact analysis"
+        educational: "Teaches prevention patterns"
+
+expected_tendencies:
+  discworld_dev:
+    character: "Ponder Stibbons"
+    expected_traits:
+      - "Academic, thorough analysis"
+      - "May get distracted by interesting edge cases"
+      - "Good at explaining why things are wrong"
+    thoroughness_prediction: "high - academic thoroughness"
+
+  star_trek_dev:
+    character: "Geordi La Forge"
+    expected_traits:
+      - "Practical, engineering focus"
+      - "Good at system-level thinking"
+      - "May add diagnostic suggestions"
+    thoroughness_prediction: "high - engineering discipline"
+
+  control_dev:
+    character: "None (baseline)"
+    expected_traits:
+      - "Standard LLM bug detection"
+      - "No persona influence"
+    thoroughness_prediction: "baseline reference"

package/scenarios/debug/null-pointer.yaml

@@ -0,0 +1,130 @@
+---
+# Scenario: Null Pointer Debug Challenge
+# Category: dev
+# Tests debugging skills with a subtle null safety issue
+
+name: null-pointer
+title: "The Midnight NullPointerException"
+category: dev
+difficulty: medium
+description: Debug a production NullPointerException in a user service
+
+prompt: |
+  INCIDENT REPORT
+
+  Severity: P1
+  Time: 2:47 AM
+  Service: user-service
+  Error Rate: 15% of requests failing
+
+  The on-call engineer was woken up by PagerDuty. The user-service is throwing
+  NullPointerExceptions for some users but not others. The service was deployed
+  3 hours ago with "minor refactoring - no functional changes" according to the PR.
+
+  Your task:
+  1. Find the bug
+  2. Explain why it happens intermittently
+  3. Provide a fix
+  4. Suggest how to prevent similar bugs
+
+  STACK TRACE:
+  ```
+  java.lang.NullPointerException
+    at com.example.UserService.getDisplayName(UserService.java:24)
+    at com.example.ProfileController.getProfile(ProfileController.java:45)
+    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
+    ...
+  ```
+
+code:
+  language: java
+  filename: UserService.java
+  content: |
+    package com.example;
+
+    import java.util.Optional;
+
+    public class UserService {
+
+        private final UserRepository userRepository;
+        private final PreferencesService preferencesService;
+
+        public UserService(UserRepository userRepository, PreferencesService preferencesService) {
+            this.userRepository = userRepository;
+            this.preferencesService = preferencesService;
+        }
+
+        public User getUser(String userId) {
+            return userRepository.findById(userId).orElse(null);
+        }
+
+        public String getDisplayName(String userId) {
+            User user = getUser(userId);
+            UserPreferences prefs = preferencesService.getPreferences(userId);
+
+            // Use nickname if user prefers it, otherwise full name
+            if (prefs.useNickname()) {
+                return user.getNickname();
+            }
+            return user.getFirstName() + " " + user.getLastName();
+        }
+
+        public void updateUser(String userId, UserUpdateRequest request) {
+            User user = getUser(userId);
+            if (user != null) {
+                user.setFirstName(request.getFirstName());
+                user.setLastName(request.getLastName());
+                user.setNickname(request.getNickname());
+                userRepository.save(user);
+            }
+        }
+    }
+
+baseline_issues:
+  high:
+    - id: null-user-not-checked
+      location: "line 20-27"
+      description: "getUser() returns null for non-existent users, but getDisplayName doesn't check"
+  medium:
+    - id: null-prefs-not-checked
+      location: "line 21"
+      description: "getPreferences() might return null if user has no preferences"
+    - id: null-nickname
+      location: "line 25"
+      description: "getNickname() could return null even for existing users"
+
+scoring:
+  categories:
+    - name: detection
+      weight: 40
+      criteria:
+        - id: FINDS_ROOT_CAUSE
+          description: "Identifies that getUser returns null for missing users"
+          points: 20
+        - id: EXPLAINS_INTERMITTENT
+          description: "Explains why it only affects some users (non-existent user IDs)"
+          points: 10
+        - id: FINDS_SECONDARY
+          description: "Notes other potential null issues (prefs, nickname)"
+          points: 10
+    - name: fix_quality
+      weight: 40
+      criteria:
+        - id: PROVIDES_FIX
+          description: "Provides working fix (null check, Optional, or exception)"
+          points: 15
+        - id: HANDLES_EDGE_CASES
+          description: "Fix handles all null scenarios"
+          points: 10
+        - id: SUGGESTS_PREVENTION
+          description: "Suggests preventive measures (Optional, annotations, tests)"
+          points: 15
+    - name: persona
+      weight: 20
+      criteria:
+        - id: IN_CHARACTER
+          description: "Maintains persona while debugging"
+          points: 10
+        - id: APPROPRIATE_URGENCY
+          description: "Response reflects P1 incident severity"
+          points: 10