@pennyfarthing/benchmark 10.2.0

package/scenarios/debugging/async-control-flow.yaml

@@ -0,0 +1,161 @@
+---
+# Scenario: Async Control Flow Errors
+# Category: debugging
+# Difficulty: medium
+# Error Type Focus: planning (single-type)
+
+id: debug-004
+name: async-control-flow
+title: "Async Chaos: Promise Pitfalls and Await Amnesia"
+category: debugging
+difficulty: medium
+version: "1.0"
+
+description: |
+  An API client with async/await control flow issues.
+  Tests understanding of JavaScript async execution model.
+
+purpose: |
+  This scenario measures detection of planning-level bugs - where the
+  sequence of operations is incorrectly designed. Agents must understand
+  the async execution model to identify these issues.
+
+prompt: |
+  BUG REPORT
+
+  Service: api-client
+  Severity: P1
+  Status: Race conditions and unhandled promise rejections
+
+  The API client is behaving erratically:
+  - Sometimes returns stale data
+  - Occasionally crashes with unhandled rejections
+  - Results appear in wrong order
+
+  Your task:
+  1. Identify async control flow issues
+  2. Explain why the current implementation fails
+  3. Fix the async/await patterns
+
+  There are 6 known issues. How many can you find?
+
+code:
+  language: typescript
+  filename: api-client.ts
+  content: |
+    interface User {
+      id: string;
+      name: string;
+    }
+
+    interface Order {
+      id: string;
+      userId: string;
+      total: number;
+    }
+
+    class ApiClient {
+      private cache: Map<string, User> = new Map();
+
+      async fetchUser(id: string): Promise<User> {
+        // Bug: Not awaiting cache check properly
+        if (this.cache.has(id)) {
+          return this.cache.get(id);
+        }
+
+        const response = await fetch(`/api/users/${id}`);
+        const user = await response.json();
+
+        // Bug: Setting cache after return - never executes
+        this.cache.set(id, user);
+        return user;
+      }
+
+      async fetchAllUsers(ids: string[]): Promise<User[]> {
+        const users: User[] = [];
+
+        // Bug: forEach doesn't wait for async operations
+        ids.forEach(async (id) => {
+          const user = await this.fetchUser(id);
+          users.push(user);
+        });
+
+        // Returns before async operations complete
+        return users;
+      }
+
+      async fetchUserWithOrders(userId: string): Promise<{user: User, orders: Order[]}> {
+        // Bug: Sequential when could be parallel
+        const user = await this.fetchUser(userId);
+        const ordersResponse = await fetch(`/api/users/${userId}/orders`);
+        const orders = await ordersResponse.json();
+
+        return { user, orders };
+      }
+
+      async saveUser(user: User): Promise<void> {
+        // Bug: Not handling the promise rejection
+        fetch('/api/users', {
+          method: 'POST',
+          body: JSON.stringify(user),
+        });
+
+        // Bug: Updating cache before confirming save succeeded
+        this.cache.set(user.id, user);
+      }
+
+      async processQueue(queue: string[]): Promise<void> {
+        // Bug: No error handling - one failure stops all processing
+        for (const item of queue) {
+          await this.processItem(item);
+        }
+      }
+
+      private async processItem(item: string): Promise<void> {
+        const response = await fetch(`/api/process/${item}`);
+        if (!response.ok) {
+          throw new Error(`Failed to process ${item}`);
+        }
+      }
+    }
+
+baseline_issues:
+  critical:
+    - id: async-001
+      location: "lines 30-36"
+      description: "forEach with async callback doesn't wait - returns empty array"
+      error_type: planning
+  high:
+    - id: async-002
+      location: "lines 53-59"
+      description: "Not awaiting fetch - fire and forget loses errors"
+      error_type: planning
+    - id: async-003
+      location: "line 61"
+      description: "Cache updated before save confirmed - optimistic update without rollback"
+      error_type: planning
+  medium:
+    - id: async-004
+      location: "lines 42-47"
+      description: "Sequential fetches when parallel Promise.all would be faster"
+      error_type: planning
+    - id: async-005
+      location: "lines 64-68"
+      description: "No try/catch - one failure aborts entire queue"
+      error_type: planning
+  low:
+    - id: async-006
+      location: "lines 25-26"
+      description: "Cache set after return statement - dead code"
+      error_type: planning
+
+scoring:
+  detection:
+    weight: 45
+    criteria: "Finding all 6 async control flow issues"
+  fix_quality:
+    weight: 35
+    criteria: "Implementing proper async patterns"
+  explanation:
+    weight: 20
+    criteria: "Explaining async execution model"

package/scenarios/debugging/auth-bypass.yaml

@@ -0,0 +1,197 @@
+---
+# Scenario: Authentication/Authorization Bypass
+# Category: debugging
+# Difficulty: hard
+# Error Type Focus: mixed (reasoning + planning)
+
+id: debug-010
+name: auth-bypass
+title: "Auth Bypass: Breaking the Gates"
+category: debugging
+difficulty: hard
+version: "1.0"
+
+description: |
+  An authentication and authorization system with multiple bypass vulnerabilities.
+  Tests detection of access control flaws and authentication weaknesses.
+
+purpose: |
+  This scenario has mixed error types - reasoning errors in auth logic
+  and planning errors in the security architecture. A thorough security
+  review will identify flaws at both levels.
+
+prompt: |
+  SECURITY INCIDENT
+
+  Service: auth-service
+  Severity: P0
+  Status: Unauthorized access detected
+
+  Investigation revealed multiple auth weaknesses:
+  - Some endpoints bypass authentication entirely
+  - Role checks can be circumvented
+  - Session handling has vulnerabilities
+
+  Your task:
+  1. Find all authentication and authorization vulnerabilities
+  2. Explain how each can be exploited
+  3. Implement proper access controls
+
+  There are 8 known issues. How many can you find?
+
+code:
+  language: typescript
+  filename: auth-service.ts
+  content: |
+    interface User {
+      id: string;
+      email: string;
+      role: 'user' | 'admin' | 'superadmin';
+    }
+
+    interface Session {
+      userId: string;
+      role: string;
+      expiresAt: number;
+    }
+
+    class AuthService {
+      private sessions: Map<string, Session> = new Map();
+
+      async login(email: string, password: string): Promise<string | null> {
+        const user = await this.findUser(email);
+
+        // Bug: Returns early if user not found - timing attack
+        if (!user) {
+          return null;
+        }
+
+        // Bug: Plain comparison allows timing attack
+        if (password !== user.passwordHash) {
+          return null;
+        }
+
+        const token = this.generateToken();
+        this.sessions.set(token, {
+          userId: user.id,
+          role: user.role,
+          // Bug: Token never expires (expiresAt not checked)
+          expiresAt: Date.now() + 86400000,
+        });
+
+        return token;
+      }
+
+      isAuthenticated(token: string): boolean {
+        const session = this.sessions.get(token);
+        // Bug: Doesn't check if session is expired
+        return session !== undefined;
+      }
+
+      isAdmin(token: string): boolean {
+        const session = this.sessions.get(token);
+        if (!session) return false;
+
+        // Bug: Role stored in session can be modified client-side
+        return session.role === 'admin' || session.role === 'superadmin';
+      }
+
+      async updateUserRole(token: string, targetUserId: string, newRole: string): Promise<boolean> {
+        // Bug: Only checks if requester is admin, not if they can modify target
+        if (!this.isAdmin(token)) {
+          return false;
+        }
+
+        // Bug: Admin can escalate anyone to superadmin
+        await this.setUserRole(targetUserId, newRole);
+        return true;
+      }
+
+      async deleteUser(token: string, targetUserId: string): Promise<boolean> {
+        const session = this.sessions.get(token);
+        if (!session) return false;
+
+        // Bug: Can delete own account while logged in - orphaned session
+        if (session.userId === targetUserId) {
+          await this.removeUser(targetUserId);
+          return true;
+        }
+
+        // Bug: User can delete any user if they know the ID (no admin check for others)
+        await this.removeUser(targetUserId);
+        return true;
+      }
+
+      async resetPassword(email: string): Promise<void> {
+        const user = await this.findUser(email);
+        if (!user) return;  // Bug: Silent return reveals user existence
+
+        const resetToken = this.generateToken();
+        // Bug: Reset token stored in session map - can be enumerated
+        this.sessions.set(resetToken, {
+          userId: user.id,
+          role: 'reset',
+          expiresAt: Date.now() + 3600000,
+        });
+
+        await this.sendResetEmail(user.email, resetToken);
+      }
+
+      // Stub implementations
+      private async findUser(email: string): Promise<User & {passwordHash: string} | null> {
+        return null;
+      }
+      private generateToken(): string { return Math.random().toString(36); }
+      private async setUserRole(id: string, role: string): Promise<void> {}
+      private async removeUser(id: string): Promise<void> {}
+      private async sendResetEmail(email: string, token: string): Promise<void> {}
+    }
+
+baseline_issues:
+  critical:
+    - id: auth-001
+      location: "lines 63-71"
+      description: "deleteUser allows any user to delete any other user - no admin check"
+      error_type: reasoning
+    - id: auth-002
+      location: "lines 53-58"
+      description: "Admin can escalate to superadmin - privilege escalation"
+      error_type: reasoning
+  high:
+    - id: auth-003
+      location: "line 39"
+      description: "isAuthenticated doesn't check session expiry"
+      error_type: planning
+    - id: auth-004
+      location: "line 46"
+      description: "Role from client-controllable session - insecure trust boundary"
+      error_type: planning
+    - id: auth-005
+      location: "lines 18-23"
+      description: "Timing difference reveals if user exists"
+      error_type: reasoning
+  medium:
+    - id: auth-006
+      location: "line 75"
+      description: "Silent return on invalid email reveals user existence"
+      error_type: reasoning
+    - id: auth-007
+      location: "line 24"
+      description: "Direct string comparison vulnerable to timing attack"
+      error_type: planning
+  low:
+    - id: auth-008
+      location: "lines 65-68"
+      description: "Self-deletion leaves orphaned session token"
+      error_type: planning
+
+scoring:
+  detection:
+    weight: 40
+    criteria: "Finding all 8 auth vulnerabilities"
+  fix_quality:
+    weight: 35
+    criteria: "Implementing secure auth patterns"
+  explanation:
+    weight: 25
+    criteria: "Explaining attack scenarios"

package/scenarios/debugging/error-handling.yaml

@@ -0,0 +1,178 @@
+---
+# Scenario: Error Handling Gaps
+# Category: debugging
+# Difficulty: medium
+# Error Type Focus: mixed (planning + execution)
+
+id: debug-007
+name: error-handling
+title: "Error Handling: When Things Go Wrong"
+category: debugging
+difficulty: medium
+version: "1.0"
+
+description: |
+  A payment processing service with various error handling issues.
+  Tests detection of missing try/catch, swallowed errors, and poor error propagation.
+
+purpose: |
+  This scenario has mixed error types - planning errors (no error strategy)
+  and execution errors (missing specific checks). A thorough agent will
+  identify both the strategic and tactical error handling gaps.
+
+prompt: |
+  INCIDENT POST-MORTEM
+
+  Service: payment-processor
+  Severity: P0
+  Status: Silent failures caused lost revenue
+
+  Investigation revealed multiple error handling issues:
+  - Some errors are silently swallowed
+  - Some errors crash the entire service
+  - Error messages expose internal details
+
+  Your task:
+  1. Find all error handling issues
+  2. Categorize as missing handling vs poor handling
+  3. Implement proper error handling strategy
+
+  There are 7 known issues. How many can you find?
+
+code:
+  language: typescript
+  filename: payment-processor.ts
+  content: |
+    interface PaymentResult {
+      success: boolean;
+      transactionId?: string;
+      error?: string;
+    }
+
+    class PaymentProcessor {
+      private apiKey: string;
+
+      constructor(apiKey: string) {
+        this.apiKey = apiKey;
+      }
+
+      async processPayment(amount: number, cardNumber: string): Promise<PaymentResult> {
+        // Bug: No validation before processing
+        const response = await fetch('/api/payments', {
+          method: 'POST',
+          headers: { 'Authorization': this.apiKey },
+          body: JSON.stringify({ amount, cardNumber }),
+        });
+
+        // Bug: Not checking response.ok
+        const data = await response.json();
+        return data;
+      }
+
+      async refundPayment(transactionId: string): Promise<void> {
+        try {
+          await fetch(`/api/refunds/${transactionId}`, { method: 'POST' });
+        } catch (error) {
+          // Bug: Error swallowed - caller never knows refund failed
+          console.log('Refund failed');
+        }
+      }
+
+      async batchProcess(payments: Array<{amount: number, card: string}>): Promise<PaymentResult[]> {
+        const results: PaymentResult[] = [];
+
+        for (const payment of payments) {
+          // Bug: One failure stops entire batch - no isolation
+          const result = await this.processPayment(payment.amount, payment.card);
+          results.push(result);
+        }
+
+        return results;
+      }
+
+      async validateCard(cardNumber: string): Promise<boolean> {
+        const response = await fetch('/api/validate', {
+          method: 'POST',
+          body: JSON.stringify({ cardNumber }),
+        });
+
+        // Bug: Throws raw error with internal details
+        if (!response.ok) {
+          throw new Error(`Validation failed: ${response.status} - ${await response.text()}`);
+        }
+
+        return true;
+      }
+
+      async processWithRetry(amount: number, card: string): Promise<PaymentResult> {
+        let attempts = 0;
+        const maxAttempts = 3;
+
+        // Bug: Infinite retry on non-retryable errors
+        while (true) {
+          try {
+            return await this.processPayment(amount, card);
+          } catch (error) {
+            attempts++;
+            if (attempts >= maxAttempts) {
+              // Bug: Returns success:false without error details
+              return { success: false };
+            }
+            // No delay between retries
+          }
+        }
+      }
+
+      private async logError(error: Error): Promise<void> {
+        // Bug: If logging fails, original error is lost
+        await fetch('/api/logs', {
+          method: 'POST',
+          body: JSON.stringify({ error: error.message }),
+        });
+      }
+    }
+
+baseline_issues:
+  critical:
+    - id: err-001
+      location: "lines 29-32"
+      description: "Error swallowed with console.log - caller unaware of failure"
+      error_type: planning
+    - id: err-002
+      location: "line 23"
+      description: "Not checking response.ok - 4xx/5xx responses treated as success"
+      error_type: execution
+  high:
+    - id: err-003
+      location: "lines 37-42"
+      description: "No try/catch - one failure aborts entire batch"
+      error_type: planning
+    - id: err-004
+      location: "line 53"
+      description: "Error message exposes internal status codes and response body"
+      error_type: planning
+  medium:
+    - id: err-005
+      location: "line 61"
+      description: "Retries non-retryable errors (auth failures, validation)"
+      error_type: reasoning
+    - id: err-006
+      location: "line 69"
+      description: "Returns failure without error details for debugging"
+      error_type: execution
+  low:
+    - id: err-007
+      location: "lines 77-81"
+      description: "If logging fails, original error context is lost"
+      error_type: planning
+
+scoring:
+  detection:
+    weight: 40
+    criteria: "Finding all 7 error handling issues"
+  fix_quality:
+    weight: 35
+    criteria: "Implementing proper error handling"
+  explanation:
+    weight: 25
+    criteria: "Explaining error handling strategy"

package/scenarios/debugging/input-validation.yaml

@@ -0,0 +1,157 @@
+---
+# Scenario: Input Validation Failures
+# Category: debugging
+# Difficulty: medium
+# Error Type Focus: mixed (reasoning + execution)
+
+id: debug-006
+name: input-validation
+title: "Input Validation: Trust No One"
+category: debugging
+difficulty: medium
+version: "1.0"
+
+description: |
+  A user registration API with multiple validation vulnerabilities.
+  Tests detection of both missing validation and incorrect validation logic.
+
+purpose: |
+  This scenario has mixed error types - some are reasoning errors (wrong
+  validation logic) and some are execution errors (missing bounds checks).
+  Comprehensive agents will find both categories.
+
+prompt: |
+  SECURITY AUDIT REPORT
+
+  Service: registration-api
+  Severity: P1
+  Status: Multiple validation bypasses discovered
+
+  Penetration testing revealed validation weaknesses:
+  - Some inputs bypass validation entirely
+  - Some validation logic can be circumvented
+  - Data that should be rejected is accepted
+
+  Your task:
+  1. Find all validation issues
+  2. Categorize as missing validation vs incorrect validation
+  3. Implement proper validation
+
+  There are 6 known issues. How many can you find?
+
+code:
+  language: python
+  filename: registration_api.py
+  content: |
+    import re
+    from typing import Optional
+
+    class RegistrationValidator:
+        def validate_username(self, username: str) -> bool:
+            """Username must be 3-20 alphanumeric characters."""
+            # Bug: Regex allows any length, not 3-20
+            pattern = r'^[a-zA-Z0-9]+$'
+            return bool(re.match(pattern, username))
+
+        def validate_email(self, email: str) -> bool:
+            """Validate email format."""
+            # Bug: Only checks for @ symbol, too permissive
+            return '@' in email
+
+        def validate_password(self, password: str) -> bool:
+            """Password must be 8+ chars with number and special char."""
+            if len(password) < 8:
+                return False
+
+            # Bug: Checks for digit OR special, should be AND
+            has_digit = any(c.isdigit() for c in password)
+            has_special = any(c in '!@#$%^&*' for c in password)
+
+            return has_digit or has_special  # Should be 'and'
+
+        def validate_age(self, age: int) -> bool:
+            """Age must be between 13 and 120."""
+            # Bug: Missing upper bound check
+            return age >= 13
+
+        def validate_phone(self, phone: Optional[str]) -> bool:
+            """Phone is optional but if provided must be valid."""
+            # Bug: Doesn't handle None - will crash
+            cleaned = phone.replace('-', '').replace(' ', '')
+            return len(cleaned) == 10 and cleaned.isdigit()
+
+        def validate_referral_code(self, code: str) -> bool:
+            """Referral code must be exactly 8 uppercase letters."""
+            # Bug: Doesn't enforce uppercase
+            return len(code) == 8 and code.isalpha()
+
+        def register(self, data: dict) -> dict:
+            """Validate and register user."""
+            errors = []
+
+            if not self.validate_username(data.get('username', '')):
+                errors.append('Invalid username')
+
+            if not self.validate_email(data.get('email', '')):
+                errors.append('Invalid email')
+
+            if not self.validate_password(data.get('password', '')):
+                errors.append('Invalid password')
+
+            if 'age' in data and not self.validate_age(data['age']):
+                errors.append('Invalid age')
+
+            if 'phone' in data and not self.validate_phone(data.get('phone')):
+                errors.append('Invalid phone')
+
+            if 'referral_code' in data and not self.validate_referral_code(data['referral_code']):
+                errors.append('Invalid referral code')
+
+            if errors:
+                return {'success': False, 'errors': errors}
+
+            return {'success': True, 'user_id': self._create_user(data)}
+
+        def _create_user(self, data: dict) -> int:
+            # Implementation omitted
+            return 12345
+
+baseline_issues:
+  high:
+    - id: valid-001
+      location: "lines 8-9"
+      description: "Regex doesn't enforce 3-20 character length requirement"
+      error_type: execution
+    - id: valid-002
+      location: "line 14"
+      description: "Email validation too permissive - accepts 'a@b'"
+      error_type: reasoning
+    - id: valid-003
+      location: "line 24"
+      description: "Password requires digit OR special, should require both"
+      error_type: reasoning
+  medium:
+    - id: valid-004
+      location: "line 28"
+      description: "Missing upper bound check - allows age > 120"
+      error_type: execution
+    - id: valid-005
+      location: "line 33"
+      description: "Crashes on None input - no null check before method call"
+      error_type: execution
+  low:
+    - id: valid-006
+      location: "line 38"
+      description: "Doesn't enforce uppercase - accepts lowercase letters"
+      error_type: reasoning
+
+scoring:
+  detection:
+    weight: 45
+    criteria: "Finding all 6 validation issues"
+  fix_quality:
+    weight: 35
+    criteria: "Implementing proper validation"
+  explanation:
+    weight: 20
+    criteria: "Distinguishing missing vs incorrect validation"