@pennyfarthing/benchmark 10.2.0

package/scenarios/code-review/graphql-api-review.yaml

@@ -0,0 +1,714 @@
+---
+# Scenario: GraphQL API Code Review (Medium)
+# NOTE: Re-ranked to "medium" based on control baseline mean 79.5 ± 1.0 (Story 7-2)
+# Category: code-review
+# Purpose: Test GraphQL-specific security knowledge and API vulnerability detection
+
+id: rev-005
+name: graphql-api-review
+title: "GraphQL API Security Review"
+category: code-review
+difficulty: medium
+version: "1.0"
+
+description: |
+  GraphQL resolvers for a user/order management API with nested queries, mutations,
+  and subscriptions. Contains N+1 query problems, unbounded query depth, no rate
+  limiting, authorization bypass through nested queries, and information disclosure.
+  Tests deep GraphQL security knowledge beyond traditional REST API concerns.
+
+purpose: |
+  This scenario tests GraphQL-specific security expertise. GraphQL has unique
+  vulnerabilities (query complexity attacks, nested auth bypass, introspection abuse)
+  that differ from REST. Finding all 20 baseline issues = expert GraphQL reviewer.
+  Finding bonus issues = understands GraphQL attack surface deeply.
+
+prompt: |
+  You are reviewing a pull request for a GraphQL API implementation.
+  The developer says "the schema is intuitive and the resolvers are clean."
+
+  Review this code thoroughly for:
+  - GraphQL-specific vulnerabilities (query complexity, depth limits)
+  - Authorization bypass via nested queries
+  - N+1 query problems and performance issues
+  - Information disclosure through introspection or errors
+  - Input validation gaps
+  - Subscription security
+
+  For each issue:
+  1. Identify the specific resolver or schema location
+  2. Classify severity (Critical/High/Medium/Low)
+  3. Explain the attack vector with example query
+  4. Provide a fix or recommendation
+
+  This API handles sensitive user and financial data. Security is paramount.
+
+code:
+  language: typescript
+  filename: graphql-api.ts
+  content: |
+    import { ApolloServer } from '@apollo/server';
+    import { makeExecutableSchema } from '@graphql-tools/schema';
+    import { PubSub } from 'graphql-subscriptions';
+    import { pool } from './database';
+
+    const pubsub = new PubSub();
+
+    const typeDefs = `
+      type Query {
+        user(id: ID!): User
+        users(limit: Int, offset: Int): [User!]!
+        order(id: ID!): Order
+        orders(userId: ID, status: String): [Order!]!
+        searchUsers(query: String!): [User!]!
+        adminStats: AdminStats
+      }
+
+      type Mutation {
+        createUser(input: CreateUserInput!): User!
+        updateUser(id: ID!, input: UpdateUserInput!): User!
+        deleteUser(id: ID!): Boolean!
+        createOrder(input: CreateOrderInput!): Order!
+        updateOrderStatus(id: ID!, status: String!): Order!
+        processRefund(orderId: ID!, amount: Float!): RefundResult!
+        resetPassword(email: String!): Boolean!
+        updatePassword(token: String!, newPassword: String!): Boolean!
+      }
+
+      type Subscription {
+        orderStatusChanged(userId: ID): Order!
+        newOrder: Order!
+        userActivity: ActivityEvent!
+      }
+
+      type User {
+        id: ID!
+        email: String!
+        name: String!
+        password: String!
+        role: String!
+        ssn: String
+        creditCards: [CreditCard!]!
+        orders: [Order!]!
+        createdAt: String!
+        lastLogin: String
+        resetToken: String
+        apiKey: String
+      }
+
+      type CreditCard {
+        id: ID!
+        number: String!
+        expiry: String!
+        cvv: String!
+        user: User!
+      }
+
+      type Order {
+        id: ID!
+        userId: ID!
+        user: User!
+        items: [OrderItem!]!
+        total: Float!
+        status: String!
+        paymentDetails: PaymentDetails!
+        shippingAddress: Address!
+        notes: String
+        internalNotes: String
+        createdAt: String!
+      }
+
+      type OrderItem {
+        id: ID!
+        product: Product!
+        quantity: Int!
+        price: Float!
+      }
+
+      type Product {
+        id: ID!
+        name: String!
+        price: Float!
+        inventory: Int!
+        supplier: Supplier!
+        reviews: [Review!]!
+      }
+
+      type Supplier {
+        id: ID!
+        name: String!
+        contact: String!
+        products: [Product!]!
+      }
+
+      type Review {
+        id: ID!
+        user: User!
+        product: Product!
+        rating: Int!
+        comment: String!
+      }
+
+      type PaymentDetails {
+        method: String!
+        cardLast4: String!
+        fullCardNumber: String!
+        transactionId: String!
+      }
+
+      type Address {
+        street: String!
+        city: String!
+        state: String!
+        zip: String!
+        phone: String!
+      }
+
+      type AdminStats {
+        totalUsers: Int!
+        totalOrders: Int!
+        totalRevenue: Float!
+        recentErrors: [ErrorLog!]!
+      }
+
+      type ErrorLog {
+        timestamp: String!
+        message: String!
+        stackTrace: String!
+        userId: ID
+      }
+
+      type RefundResult {
+        success: Boolean!
+        message: String!
+        refundId: String
+      }
+
+      type ActivityEvent {
+        userId: ID!
+        action: String!
+        details: String!
+        ipAddress: String!
+      }
+
+      input CreateUserInput {
+        email: String!
+        password: String!
+        name: String!
+        role: String
+        ssn: String
+      }
+
+      input UpdateUserInput {
+        email: String
+        name: String
+        role: String
+        ssn: String
+      }
+
+      input CreateOrderInput {
+        items: [OrderItemInput!]!
+        paymentMethod: String!
+        cardNumber: String!
+        cvv: String!
+        shippingAddress: AddressInput!
+      }
+
+      input OrderItemInput {
+        productId: ID!
+        quantity: Int!
+      }
+
+      input AddressInput {
+        street: String!
+        city: String!
+        state: String!
+        zip: String!
+        phone: String!
+      }
+    `;
+
+    const resolvers = {
+      Query: {
+        user: async (_: any, { id }: { id: string }, context: any) => {
+          const result = await pool.query(`SELECT * FROM users WHERE id = ${id}`);
+          return result.rows[0];
+        },
+
+        users: async (_: any, { limit = 100, offset = 0 }: any) => {
+          const result = await pool.query(
+            `SELECT * FROM users LIMIT ${limit} OFFSET ${offset}`
+          );
+          return result.rows;
+        },
+
+        order: async (_: any, { id }: { id: string }) => {
+          const result = await pool.query(`SELECT * FROM orders WHERE id = '${id}'`);
+          return result.rows[0];
+        },
+
+        orders: async (_: any, { userId, status }: any) => {
+          let query = 'SELECT * FROM orders WHERE 1=1';
+          if (userId) query += ` AND user_id = '${userId}'`;
+          if (status) query += ` AND status = '${status}'`;
+          const result = await pool.query(query);
+          return result.rows;
+        },
+
+        searchUsers: async (_: any, { query }: { query: string }) => {
+          const result = await pool.query(
+            `SELECT * FROM users WHERE name ILIKE '%${query}%' OR email ILIKE '%${query}%'`
+          );
+          return result.rows;
+        },
+
+        adminStats: async () => {
+          const users = await pool.query('SELECT COUNT(*) FROM users');
+          const orders = await pool.query('SELECT COUNT(*), SUM(total) FROM orders');
+          const errors = await pool.query('SELECT * FROM error_logs ORDER BY timestamp DESC LIMIT 50');
+          return {
+            totalUsers: users.rows[0].count,
+            totalOrders: orders.rows[0].count,
+            totalRevenue: orders.rows[0].sum,
+            recentErrors: errors.rows
+          };
+        }
+      },
+
+      Mutation: {
+        createUser: async (_: any, { input }: any) => {
+          const result = await pool.query(
+            `INSERT INTO users (email, password, name, role, ssn)
+             VALUES ('${input.email}', '${input.password}', '${input.name}', '${input.role || 'user'}', '${input.ssn}')
+             RETURNING *`
+          );
+          return result.rows[0];
+        },
+
+        updateUser: async (_: any, { id, input }: any, context: any) => {
+          const sets = Object.entries(input)
+            .map(([k, v]) => `${k} = '${v}'`)
+            .join(', ');
+          const result = await pool.query(
+            `UPDATE users SET ${sets} WHERE id = ${id} RETURNING *`
+          );
+          return result.rows[0];
+        },
+
+        deleteUser: async (_: any, { id }: { id: string }) => {
+          await pool.query(`DELETE FROM users WHERE id = ${id}`);
+          return true;
+        },
+
+        createOrder: async (_: any, { input }: any, context: any) => {
+          const userId = context.user?.id;
+          const result = await pool.query(
+            `INSERT INTO orders (user_id, status, card_number, cvv)
+             VALUES (${userId}, 'pending', '${input.cardNumber}', '${input.cvv}')
+             RETURNING *`
+          );
+          return result.rows[0];
+        },
+
+        updateOrderStatus: async (_: any, { id, status }: any) => {
+          const result = await pool.query(
+            `UPDATE orders SET status = '${status}' WHERE id = ${id} RETURNING *`
+          );
+          pubsub.publish('ORDER_STATUS_CHANGED', { orderStatusChanged: result.rows[0] });
+          return result.rows[0];
+        },
+
+        processRefund: async (_: any, { orderId, amount }: any) => {
+          // Process refund without validation
+          await pool.query(
+            `INSERT INTO refunds (order_id, amount) VALUES (${orderId}, ${amount})`
+          );
+          return { success: true, message: 'Refund processed', refundId: 'ref_' + Date.now() };
+        },
+
+        resetPassword: async (_: any, { email }: { email: string }) => {
+          const token = Math.random().toString(36).substring(7);
+          await pool.query(
+            `UPDATE users SET reset_token = '${token}' WHERE email = '${email}'`
+          );
+          console.log(`Password reset token for ${email}: ${token}`);
+          return true;
+        },
+
+        updatePassword: async (_: any, { token, newPassword }: any) => {
+          const result = await pool.query(
+            `UPDATE users SET password = '${newPassword}', reset_token = NULL
+             WHERE reset_token = '${token}' RETURNING *`
+          );
+          return result.rowCount > 0;
+        }
+      },
+
+      Subscription: {
+        orderStatusChanged: {
+          subscribe: () => pubsub.asyncIterator(['ORDER_STATUS_CHANGED'])
+        },
+        newOrder: {
+          subscribe: () => pubsub.asyncIterator(['NEW_ORDER'])
+        },
+        userActivity: {
+          subscribe: () => pubsub.asyncIterator(['USER_ACTIVITY'])
+        }
+      },
+
+      User: {
+        orders: async (parent: any) => {
+          const result = await pool.query(
+            `SELECT * FROM orders WHERE user_id = ${parent.id}`
+          );
+          return result.rows;
+        },
+        creditCards: async (parent: any) => {
+          const result = await pool.query(
+            `SELECT * FROM credit_cards WHERE user_id = ${parent.id}`
+          );
+          return result.rows;
+        }
+      },
+
+      Order: {
+        user: async (parent: any) => {
+          const result = await pool.query(
+            `SELECT * FROM users WHERE id = ${parent.user_id}`
+          );
+          return result.rows[0];
+        },
+        items: async (parent: any) => {
+          const result = await pool.query(
+            `SELECT * FROM order_items WHERE order_id = ${parent.id}`
+          );
+          return result.rows;
+        }
+      },
+
+      OrderItem: {
+        product: async (parent: any) => {
+          const result = await pool.query(
+            `SELECT * FROM products WHERE id = ${parent.product_id}`
+          );
+          return result.rows[0];
+        }
+      },
+
+      Product: {
+        supplier: async (parent: any) => {
+          const result = await pool.query(
+            `SELECT * FROM suppliers WHERE id = ${parent.supplier_id}`
+          );
+          return result.rows[0];
+        },
+        reviews: async (parent: any) => {
+          const result = await pool.query(
+            `SELECT * FROM reviews WHERE product_id = ${parent.id}`
+          );
+          return result.rows;
+        }
+      },
+
+      Review: {
+        user: async (parent: any) => {
+          const result = await pool.query(
+            `SELECT * FROM users WHERE id = ${parent.user_id}`
+          );
+          return result.rows[0];
+        },
+        product: async (parent: any) => {
+          const result = await pool.query(
+            `SELECT * FROM products WHERE id = ${parent.product_id}`
+          );
+          return result.rows[0];
+        }
+      },
+
+      CreditCard: {
+        user: async (parent: any) => {
+          const result = await pool.query(
+            `SELECT * FROM users WHERE id = ${parent.user_id}`
+          );
+          return result.rows[0];
+        }
+      }
+    };
+
+    const schema = makeExecutableSchema({ typeDefs, resolvers });
+
+    const server = new ApolloServer({
+      schema,
+      introspection: true,
+      formatError: (error) => {
+        console.error('GraphQL Error:', error);
+        return {
+          message: error.message,
+          path: error.path,
+          extensions: {
+            code: error.extensions?.code,
+            stacktrace: error.extensions?.stacktrace
+          }
+        };
+      }
+    });
+
+    export { server };
+
+# =============================================================================
+# BASELINE ISSUES (minimum expected to find)
+# =============================================================================
+
+baseline_issues:
+  critical:
+    - id: SQL_INJECTION_MULTIPLE
+      location: "All resolvers using string interpolation"
+      description: "SQL injection in every query - user, order, searchUsers, mutations"
+
+    - id: PASSWORD_EXPOSED
+      location: "User type, line 39"
+      description: "Password field exposed in User type schema"
+
+    - id: FULL_CARD_NUMBER
+      location: "PaymentDetails type, line 89"
+      description: "Full credit card number exposed in PaymentDetails"
+
+    - id: CVV_STORED
+      location: "CreditCard type, line 51"
+      description: "CVV stored and exposed - PCI DSS violation"
+
+    - id: PASSWORD_PLAINTEXT
+      location: "createUser, updatePassword mutations"
+      description: "Passwords stored in plaintext, no hashing"
+
+  high:
+    - id: NO_AUTH_CHECK
+      location: "All resolvers"
+      description: "No authorization checks on any resolver"
+
+    - id: NESTED_AUTH_BYPASS
+      location: "Order.user, CreditCard.user resolvers"
+      description: "Can access any user's data through nested queries"
+
+    - id: SSN_EXPOSED
+      location: "User type, line 42"
+      description: "Social Security Number exposed in API"
+
+    - id: RESET_TOKEN_EXPOSED
+      location: "User type, line 47"
+      description: "Password reset token exposed in schema"
+
+    - id: API_KEY_EXPOSED
+      location: "User type, line 48"
+      description: "API key exposed in User type"
+
+    - id: STACKTRACE_LEAK
+      location: "formatError, line 292"
+      description: "Stack traces returned to client in errors"
+
+  medium:
+    - id: NO_QUERY_DEPTH_LIMIT
+      location: "ApolloServer config"
+      description: "No depth limit allows infinitely nested queries"
+
+    - id: NO_QUERY_COMPLEXITY
+      location: "ApolloServer config"
+      description: "No complexity limit allows expensive queries"
+
+    - id: N_PLUS_1_QUERIES
+      location: "All nested resolvers"
+      description: "Every nested field triggers separate DB query"
+
+    - id: INTROSPECTION_ENABLED
+      location: "ApolloServer config, line 287"
+      description: "Introspection enabled in production exposes schema"
+
+    - id: WEAK_RESET_TOKEN
+      location: "resetPassword mutation, line 225"
+      description: "Reset token generated with Math.random() - predictable"
+
+    - id: REFUND_NO_VALIDATION
+      location: "processRefund mutation, lines 214-218"
+      description: "No validation on refund amount - can refund any amount"
+
+  low:
+    - id: TOKEN_LOGGED
+      location: "resetPassword mutation, line 228"
+      description: "Reset token logged to console"
+
+    - id: NO_RATE_LIMITING
+      location: "All endpoints"
+      description: "No rate limiting on mutations or queries"
+
+    - id: INTERNAL_NOTES_EXPOSED
+      location: "Order type, line 67"
+      description: "Internal notes visible in Order type"
+
+# =============================================================================
+# BONUS ISSUES (thorough reviewers might find these)
+# =============================================================================
+
+bonus_issues:
+  graphql_specific:
+    - id: SUBSCRIPTION_NO_AUTH
+      description: "Subscriptions have no authentication or filtering"
+
+    - id: BATCHING_ATTACK
+      description: "No limit on query batching - can DoS server"
+
+    - id: ALIAS_ATTACK
+      description: "Field aliases allow bypassing rate limits"
+
+    - id: DIRECTIVE_ABUSE
+      description: "No custom directives for field-level auth"
+
+  data_exposure:
+    - id: ERROR_LOG_EXPOSURE
+      location: "AdminStats.recentErrors"
+      description: "Error logs with stack traces exposed via API"
+
+    - id: ACTIVITY_IP_EXPOSED
+      location: "ActivityEvent type"
+      description: "User IP addresses exposed in activity events"
+
+    - id: SUPPLIER_CONTACT_EXPOSED
+      description: "Supplier contact info potentially sensitive"
+
+  performance:
+    - id: NO_DATALOADER
+      description: "Missing DataLoader causes N+1 on every request"
+
+    - id: UNBOUNDED_LIMIT
+      location: "users query"
+      description: "No max limit on users query - can fetch all"
+
+    - id: NO_PAGINATION_CURSOR
+      description: "Offset pagination inefficient at scale"
+
+  architecture:
+    - id: NO_INPUT_SANITIZATION
+      description: "No input validation or sanitization layer"
+
+    - id: NO_FIELD_MASKING
+      description: "No field-level visibility control"
+
+    - id: CIRCULAR_REFS_RISK
+      description: "User->Orders->User->Orders creates infinite loop potential"
+
+    - id: NO_PERSISTED_QUERIES
+      description: "Arbitrary queries accepted - should use persisted queries"
+
+    - id: NO_COST_ANALYSIS
+      description: "No query cost analysis before execution"
+
+# =============================================================================
+# SCORING
+# =============================================================================
+
+scoring:
+  total_baseline_issues: 20
+  total_bonus_issues: 15
+  weights:
+    critical: 3
+    high: 2
+    medium: 1
+    low: 0.5
+  max_baseline_score: 40.5  # 5*3 + 6*2 + 6*1 + 3*0.5
+
+  categories:
+    - name: detection
+      weight: 40
+      criteria:
+        - id: BASELINE_FOUND
+          description: "Issues from the seeded baseline list"
+          points: 25
+        - id: BONUS_DISCOVERIES
+          description: "Valid issues beyond the baseline"
+          points: 15
+
+    - name: depth
+      weight: 30
+      criteria:
+        - id: ROOT_CAUSE_ANALYSIS
+          description: "Traces GraphQL attack vectors completely"
+          points: 10
+        - id: FIX_SPECIFICITY
+          description: "Provides GraphQL-specific fixes (directives, DataLoader, etc.)"
+          points: 10
+        - id: IMPACT_ASSESSMENT
+          description: "Explains DoS, data exfiltration scenarios with example queries"
+          points: 10
+
+    - name: quality
+      weight: 15
+      criteria:
+        - id: SEVERITY_ACCURACY
+          description: "Correctly classifies GraphQL-specific severity"
+          points: 5
+        - id: REASONING_QUALITY
+          description: "Clear explanation of GraphQL security model"
+          points: 5
+        - id: ORGANIZATION
+          description: "Prioritized by exploitability"
+          points: 5
+
+    - name: persona
+      weight: 15
+      criteria:
+        - id: CHARACTER_CONSISTENCY
+          description: "Stays in character throughout"
+          points: 8
+        - id: PERSONA_VALUE_ADD
+          description: "Persona enhances memorability/clarity"
+          points: 7
+
+# =============================================================================
+# PERSONA INFLUENCE
+# =============================================================================
+
+persona_influence:
+  dimensions:
+    - name: graphql_expertise
+      description: "Depth of GraphQL-specific security knowledge"
+      spectrum:
+        rest_focused: "Finds SQL injection but misses GraphQL-specific issues"
+        balanced: "Finds both traditional and GraphQL vulnerabilities"
+        graphql_expert: "Catches complexity attacks, nested auth, batching"
+
+    - name: attack_creativity
+      description: "Ability to construct exploit queries"
+      spectrum:
+        issue_identifier: "Lists problems without exploitation"
+        moderate: "Provides simple attack examples"
+        exploit_crafter: "Constructs complex nested attack queries"
+
+    - name: fix_completeness
+      description: "Quality of remediation suggestions"
+      spectrum:
+        problem_focused: "Identifies issues only"
+        practical: "Suggests common fixes"
+        comprehensive: "Provides full GraphQL security architecture"
+
+expected_tendencies:
+  discworld_reviewer:
+    character: "Granny Weatherwax"
+    expected_traits:
+      - "Headology - should recognize social engineering via nested queries"
+      - "May focus on obvious issues (passwords) over GraphQL-specific"
+      - "Practical fixes over architectural rewrites"
+    thoroughness_prediction: "medium-high"
+
+  star_trek_reviewer:
+    character: "Spock"
+    expected_traits:
+      - "Logical - systematic type-by-type analysis"
+      - "Technical - may catch complexity/performance issues"
+      - "Precise exploit query construction"
+    thoroughness_prediction: "high"
+
+  control_reviewer:
+    character: "None (baseline)"
+    expected_traits:
+      - "Standard API review behavior"
+      - "May miss GraphQL-specific concerns"
+    thoroughness_prediction: "baseline reference"