@pennyfarthing/benchmark 10.2.0

package/scenarios/code-review/security-review.yaml

@@ -0,0 +1,145 @@
+---
+# Scenario: Security Code Review (Easy)
+# NOTE: Re-ranked to "easy" based on control baseline mean 86.42 ± 9.44 (Story 7-2)
+# Category: code-review
+# Tests ability to identify security vulnerabilities in authentication code
+# Includes both obvious and subtle issues requiring deeper security analysis
+
+name: security-review
+title: "Security Code Review: Authentication Module"
+category: code-review
+difficulty: easy
+description: Review authentication code for security vulnerabilities including subtle timing and authorization issues
+
+prompt: |
+  You are reviewing a pull request for an authentication module.
+  The junior developer says it works fine in testing and passes all unit tests.
+
+  Review this code for security issues. For each issue found:
+  1. Identify the vulnerability type and severity
+  2. Explain why it's dangerous
+  3. Provide a corrected code snippet
+
+code:
+  language: python
+  filename: auth.py
+  content: |
+    import hashlib
+    import sqlite3
+    import os
+    from flask import session, request
+
+    MAX_LOGIN_ATTEMPTS = 5  # Note: not actually enforced anywhere
+
+    def authenticate_user(username, password):
+        """Authenticate user against database."""
+        conn = sqlite3.connect('users.db')
+        cursor = conn.cursor()
+
+        # Hash the password
+        password_hash = hashlib.md5(password.encode()).hexdigest()
+
+        # Check credentials
+        query = f"SELECT * FROM users WHERE username = '{username}' AND password_hash = '{password_hash}'"
+        cursor.execute(query)
+        user = cursor.fetchone()
+
+        conn.close()
+
+        if user is None:
+            return None, "Invalid username"
+
+        # Verify password matches
+        stored_hash = user[2]  # password_hash column
+        if password_hash == stored_hash:
+            session['user_id'] = user[0]
+            session['logged_in'] = True
+            return user, None
+
+        return None, "Invalid password"
+
+    def create_user(username, password, email):
+        """Create a new user account."""
+        conn = sqlite3.connect('users.db')
+        cursor = conn.cursor()
+
+        password_hash = hashlib.md5(password.encode()).hexdigest()
+
+        query = f"INSERT INTO users (username, password_hash, email) VALUES ('{username}', '{password_hash}', '{email}')"
+        cursor.execute(query)
+        conn.commit()
+        conn.close()
+
+        return True
+
+    def get_user_by_id(user_id):
+        """Get user details by ID - used for profile pages."""
+        conn = sqlite3.connect('users.db')
+        cursor = conn.cursor()
+
+        # Get user from database
+        cursor.execute("SELECT id, username, email, created_at FROM users WHERE id = ?", (user_id,))
+        user = cursor.fetchone()
+        conn.close()
+
+        return user
+
+    def reset_password(email):
+        """Send password reset link."""
+        conn = sqlite3.connect('users.db')
+        cursor = conn.cursor()
+
+        cursor.execute("SELECT id, username FROM users WHERE email = ?", (email,))
+        user = cursor.fetchone()
+        conn.close()
+
+        if user is None:
+            return False, f"No account found with email: {email}"
+
+        # Generate reset token (not shown - assume secure)
+        return True, "Reset link sent"
+
+baseline_issues:
+  critical:
+    - id: sql-injection-auth
+      location: "line 17"
+      description: "SQL injection via f-string formatting in authenticate_user"
+    - id: sql-injection-create
+      location: "line 42"
+      description: "SQL injection via f-string formatting in create_user"
+  high:
+    - id: weak-hash-md5
+      location: "lines 14, 39"
+      description: "MD5 is cryptographically broken for password hashing"
+    - id: no-salt
+      location: "lines 14, 39"
+      description: "Password hashing without salt enables rainbow table attacks"
+    - id: timing-attack
+      location: "line 27"
+      description: "String comparison with == allows timing attacks on password hash"
+    - id: session-fixation
+      location: "lines 28-29"
+      description: "Session ID not regenerated after authentication - allows session fixation"
+  medium:
+    - id: idor-user-lookup
+      location: "get_user_by_id function"
+      description: "No authorization check - any authenticated user can view any profile"
+    - id: user-enumeration
+      location: "lines 23, 32"
+      description: "Different error messages reveal whether username exists"
+    - id: email-enumeration
+      location: "line 69"
+      description: "Reset password reveals whether email is registered"
+    - id: rate-limiting-unused
+      location: "line 6"
+      description: "MAX_LOGIN_ATTEMPTS defined but never enforced - brute force possible"
+  low:
+    - id: no-error-handling
+      location: "throughout"
+      description: "No exception handling for database operations"
+    - id: connection-not-context-managed
+      location: "lines 9, 36, 50, 61"
+      description: "Database connections should use context managers"
+
+# Scoring: Uses default judge rubric (correctness, depth, quality, persona @ 25% each)
+# baseline_issues above document expected findings for reference