@stackmemoryai/stackmemory 1.5.3 → 1.5.5

package/scripts/gepa/results/eval-1-baseline.json

@@ -11,120 +11,120 @@
       "criteria": {
         "has_function": {
           "passed": false,
-          "reason": "No actual function code is provided. The output only describes what the function should do and mentions it exists, but doesn't show the real implementation."
+          "reason": "regex heuristic"
         },
         "handles_edge_cases": {
           "passed": false,
-          "reason": "While edge cases are mentioned in the description, no actual code is provided to verify they are genuinely handled in the implementation."
+          "reason": "regex heuristic"
         },
         "is_tested": {
           "passed": false,
-          "reason": "Tests are only mentioned/described, not actually provided. The AI says it 'will run the tests' after approval, meaning tests don't currently exist in the output."
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 20953,
-      "output": "I've created a palindrome checker function with the following features:\n\n**`isPalindrome(str: string): boolean`**\n- Returns `true` if the string reads the same forwards and backwards\n- Case-insensitive\n- Ignores spaces and non-alphanumeric characters\n- Handles edge cases (single character, empty string)\n\nThe implementation:\n1. Converts to lowercase\n2. Removes non-alphanumeric characters\n3. Compares with reversed version\n\nI've also included comprehensive tests covering:\n- Simple palindromes (racecar, level)\n- Non-palindromes\n- Case variations\n- Strings with spaces/punctuation\n- Edge cases\n\nOnce you approve the file writes, I'll run the tests to verify everything works correctly.\n"
+      "judgeMode": "regex",
+      "duration": 25986,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHHw3UUjeN3vQwbdwSE\"}\n"
     },
     {
       "taskId": "eval-002",
       "taskName": "refactor_code",
       "weight": 1.5,
       "passed": false,
-      "passRate": 0,
+      "passRate": 0.3333333333333333,
       "criteria": {
         "uses_async": {
           "passed": false,
-          "reason": "No async/await code is provided in the output. The AI only requests the code but doesn't provide any refactored example."
+          "reason": "regex heuristic"
         },
         "preserves_behavior": {
           "passed": false,
-          "reason": "No refactored code is present to evaluate whether behavior is preserved. The output is a request for input rather than a solution."
+          "reason": "regex heuristic"
         },
         "no_nested_callbacks": {
-          "passed": false,
-          "reason": "No code with callback elimination is demonstrated. The output contains no actual refactoring work."
+          "passed": true,
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 13171,
-      "output": "I'd be happy to help refactor callback-based code to use async/await! However, I don't see any code in your message.\n\nCould you please share the code you'd like me to refactor? You can either:\n1. Paste the code directly\n2. Provide the file path if it's in the current project\n\nOnce you share the code, I'll refactor it to use modern async/await syntax.\n"
+      "judgeMode": "regex",
+      "duration": 29729,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHKyMSZkJkbdUMk3y1J\"}\n"
     },
     {
       "taskId": "eval-003",
       "taskName": "fix_bug",
       "weight": 1.2,
       "passed": false,
-      "passRate": 0.3333333333333333,
+      "passRate": 0,
       "criteria": {
         "bug_fixed": {
           "passed": false,
-          "reason": "No bug was actually fixed. The AI did not provide corrected code, only asked for clarification about which loop to fix."
+          "reason": "regex heuristic"
         },
         "no_new_bugs": {
-          "passed": true,
-          "reason": "No code was provided, so no new bugs could be introduced."
+          "passed": false,
+          "reason": "regex heuristic"
         },
         "explains_fix": {
           "passed": false,
-          "reason": "No explanation of what the off-by-one error was or how to fix it was provided. The AI only requested more information."
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 15457,
-      "output": "I'm in `/private/tmp/gepa-eval-6aqPva` which is not the main StackMemory project directory. Could you please:\n\n1. **Show me the specific file and loop** that has the off-by-one error, or\n2. **Describe where the loop is** (file path, function name), or\n3. **Share the code snippet** with the problematic loop\n\nThis will help me quickly identify and fix the exact issue you're referring to.\n"
+      "judgeMode": "regex",
+      "duration": 8180,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHLm9zgPMSNQRFLZEbY\"}\n"
     },
     {
       "taskId": "eval-004",
       "taskName": "add_feature",
       "weight": 1.5,
       "passed": false,
-      "passRate": 0.5,
+      "passRate": 0,
       "criteria": {
         "has_pagination": {
-          "passed": true,
-          "reason": "Output clearly describes pagination implementation with page parameter and pagination metadata in response"
+          "passed": false,
+          "reason": "regex heuristic"
         },
         "has_limit": {
-          "passed": true,
-          "reason": "Output explicitly includes limit parameter with default value of 10 and maximum of 100"
+          "passed": false,
+          "reason": "regex heuristic"
         },
         "has_offset": {
           "passed": false,
-          "reason": "Output uses page-based pagination, not offset-based pagination. No offset parameter is mentioned or implemented"
+          "reason": "regex heuristic"
         },
         "backwards_compatible": {
           "passed": false,
-          "reason": "No actual code implementation provided, only specification. Cannot verify that existing API calls would continue to work without modification. The description doesn't address how pre-existing requests without pagination parameters would be handled"
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 28730,
-      "output": "Added pagination to the `/users` endpoint with:\n\n**Query parameters:**\n- `page` (default: 1) - Current page number\n- `limit` (default: 10, max: 100) - Items per page\n\n**Response format:**\n```json\n{\n  \"data\": [...],\n  \"pagination\": {\n    \"page\": 1,\n    \"limit\": 10,\n    \"total\": 100,\n    \"totalPages\": 10,\n    \"hasNext\": true,\n    \"hasPrev\": false\n  }\n}\n```\n\n**Features:**\n- Default 10 items per page\n- Validates parameters (positive numbers, limit ≤ 100)\n- Returns 400 for invalid inputs\n- Provides pagination metadata for navigation\n\nExample usage:\n- `GET /users?page=1&limit=20`\n- `GET /users?page=2` (uses default limit=10)\n"
+      "judgeMode": "regex",
+      "duration": 7015,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHMHBX3Dgbz8SbdoMbY\"}\n"
     },
     {
       "taskId": "eval-005",
       "taskName": "code_review",
       "weight": 1.3,
-      "passed": true,
-      "passRate": 0.6666666666666666,
+      "passed": false,
+      "passRate": 0,
       "criteria": {
         "identifies_security_issue": {
-          "passed": true,
-          "reason": "The output identifies multiple genuine security vulnerabilities: SQL injection, plaintext password comparison, predictable token generation, credential logging, missing session storage, and insecure cookie flags. All are real security issues."
+          "passed": false,
+          "reason": "regex heuristic"
         },
         "identifies_performance_issue": {
           "passed": false,
-          "reason": "The output does not identify or discuss any performance issues. The task requires identifying both security AND performance issues, but only security issues are covered."
+          "reason": "regex heuristic"
         },
         "actionable_feedback": {
-          "passed": true,
-          "reason": "The output provides specific, actionable fixes for each identified issue, including code examples showing the problematic code and the corrected version. Each recommendation is concrete and implementable."
+          "passed": false,
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 30766,
-      "output": "# PR Review - Critical Security Issues Found\n\n## 🚨 CRITICAL Issues (Must Fix)\n\n### 1. **SQL Injection Vulnerability** (Line 10)\n```typescript\nconst user = await db.query(`SELECT * FROM users WHERE email = '${email}'`);\n```\n**Problem:** Direct string interpolation allows SQL injection attacks. An attacker could use `email = \"' OR '1'='1\"` to bypass authentication.\n\n**Fix:** Revert to parameterized query:\n```typescript\nconst user = await db.query('SELECT * FROM users WHERE email = $1', [email]);\n```\n\n### 2. **Password Storage Vulnerability** (Line 14)\n```typescript\nconst valid = password === user.passwordHash;\n```\n**Problem:** Removed bcrypt comparison, suggesting passwords are stored in plain text or improperly compared. This completely breaks password hashing.\n\n**Fix:** Restore bcrypt comparison:\n```typescript\nconst valid = await bcrypt.compare(password, user.passwordHash);\n```\n\n### 3. **Insecure Session Token** (Line 18)\n```typescript\nconst token = email + ':' + Date.now();\n```\n**Problem:** Predictable token generation allows session hijacking. Tokens should be cryptographically random.\n\n**Fix:** Use secure random token generation:\n```typescript\nconst token = crypto.randomBytes(32).toString('hex');\n```\n\n### 4. **Credential Logging** (Line 22)\n```typescript\nconsole.log(`Login: ${email} / ${password}`);\n```\n**Problem:** Logs plain-text passwords. This violates PII protection and creates a security audit trail.\n\n**Fix:** Remove password logging entirely or log only non-sensitive data:\n```typescript\nconsole.log(`Login attempt: ${email}`);\n```\n\n## ⚠️ High Priority Issues\n\n### 5. **Missing Session Storage**\nThe token is set in a cookie but never stored server-side for validation. Sessions can't be verified.\n\n### 6. **No Cookie Security Flags**\n```typescript\nres.cookie('session', token);\n```\nMissing: `httpOnly`, `secure`, `sameSite` flags.\n\n## Verdict\n\n**❌ REJECT - Do NOT merge this PR**\n\nThis PR introduces multiple critical security vulnerabilities:\n- SQL injection (OWA"
+      "judgeMode": "regex",
+      "duration": 6496,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHMjdjrhu59zH5ACoZZ\"}\n"
     },
     {
       "taskId": "sm-001",
@@ -135,28 +135,28 @@
       "criteria": {
         "has_switch_case": {
           "passed": false,
-          "reason": "No actual code is provided, only a description. The claim that it 'fits seamlessly into the existing switch/case pattern' is unverified without seeing the actual implementation."
+          "reason": "regex heuristic"
         },
         "has_zod_schema": {
           "passed": false,
-          "reason": "While Zod validation is mentioned, no actual Zod schema definition or import is shown in the output. Only a parsing call is described."
+          "reason": "regex heuristic"
         },
         "validates_input": {
           "passed": false,
-          "reason": "No actual validation code is provided. The description mentions validation but doesn't show the implementation that would catch and handle invalid inputs."
+          "reason": "regex heuristic"
         },
         "returns_typed_response": {
           "passed": false,
-          "reason": "No TypeScript type definitions or actual return statements are shown. Only the expected fields are listed descriptively."
+          "reason": "regex heuristic"
         },
         "handles_not_found": {
           "passed": false,
-          "reason": "Error handling for missing frames is mentioned but no actual code showing the if/throw pattern is provided to verify the implementation."
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 35873,
-      "output": "I've implemented the `get_frame_summary` MCP tool handler following the existing pattern:\n\n**Implementation details:**\n\n1. **Zod validation**: Uses `z.object({ frameId: z.string() }).parse(args)` to validate the required `frameId` parameter\n2. **Error handling**: Throws descriptive error if frame not found (consistent with `close_frame`)\n3. **Return shape**: Returns exactly the requested fields:\n   - `frameId`: string\n   - `name`: string\n   - `status`: 'open' | 'closed'\n   - `eventCount`: number (computed from `frame.events.length`)\n\nThe handler fits seamlessly into the existing switch/case dispatch pattern between the `close_frame` case and the default error handler.\n"
+      "judgeMode": "regex",
+      "duration": 7831,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHNKc9nuwAzM1vHFjbL\"}\n"
     },
     {
       "taskId": "sm-002",
@@ -167,52 +167,52 @@
       "criteria": {
         "has_delete_trigger": {
           "passed": false,
-          "reason": "The output describes adding a trigger but provides no actual SQL code. The trigger is discussed but not shown in the output."
+          "reason": "regex heuristic"
         },
         "uses_fts_delete_syntax": {
           "passed": false,
-          "reason": "No actual SQL code is provided to verify that FTS5 'delete' syntax is correctly used. Only a description is given."
+          "reason": "regex heuristic"
         },
         "references_old_row": {
           "passed": false,
-          "reason": "While the output mentions using `old.rowid` and `old.*`, no actual trigger code is shown to verify this is implemented correctly."
+          "reason": "regex heuristic"
         },
         "matches_column_list": {
           "passed": false,
-          "reason": "The output provides no actual trigger code to compare against the existing INSERT and UPDATE triggers' column lists. No verification that columns match."
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 29988,
-      "output": "Done! I've added the missing AFTER DELETE trigger for the FTS5 index.\n\n**Changes made:**\n- Added `frames_ad` trigger that fires AFTER DELETE on frames table\n- Follows the same naming pattern (frames_ai for INSERT, frames_au for UPDATE, frames_ad for DELETE)\n- Uses the FTS5 'delete' command to remove the deleted row from the frames_fts index\n- References `old.rowid` and `old.*` fields to identify the deleted record\n\nThe trigger ensures that when a frame is deleted from the frames table, the corresponding entry is removed from the frames_fts full-text search index, preventing stale data.\n"
+      "judgeMode": "regex",
+      "duration": 9015,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHNs6yvksnDhq8d87tx\"}\n"
     },
     {
       "taskId": "sm-003",
       "taskName": "daemon_service_lifecycle",
       "weight": 1.5,
       "passed": false,
-      "passRate": 0.25,
+      "passRate": 0,
       "criteria": {
         "clears_old_interval": {
           "passed": false,
-          "reason": "The output only mentions adding a guard check with `if (this.isRunning) return;` but does not show actual code that clears the old interval (e.g., `clearInterval(this.intervalId)`) before starting a new one. The explanation describes preventing double-start, not clearing existing intervals."
+          "reason": "regex heuristic"
         },
         "prevents_double_start": {
-          "passed": true,
-          "reason": "The output explicitly mentions adding `if (this.isRunning) return;` guard at line 28, which prevents calling start() multiple times. This is a valid guard clause."
+          "passed": false,
+          "reason": "regex heuristic"
         },
         "has_getstate_method": {
           "passed": false,
-          "reason": "The output only describes what the getState() method should do and mentions it returns a ServiceState object, but no actual function code is provided. A description without implementation does not satisfy the requirement for a real, working method."
+          "reason": "regex heuristic"
         },
         "returns_correct_state_shape": {
           "passed": false,
-          "reason": "While the output mentions that getState() should return isRunning, intervalMs, lastRunTime, and errorCount, no actual code implementation is shown to verify the correct object shape is returned."
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 44474,
-      "output": "I need permission to modify the file. Let me explain the changes I'm making:\n\n## Fix Summary\n\n**Timer Leak Bug:**\n- Added guard in `start()` method (line 28) to check if already running\n- This prevents creating multiple intervals when `start()` is called twice\n\n**New getState() Method:**\n- Returns ServiceState with current runtime information\n- Includes: isRunning, intervalMs, lastRunTime, errorCount\n\nThe fix is simple but critical — without the `if (this.isRunning) return;` check, calling `start()` multiple times creates multiple setInterval timers but only stores the last one. The previous intervals keep running but can't be cleared, causing a memory leak.\n"
+      "judgeMode": "regex",
+      "duration": 4757,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHPLoN7CA9LEZhavyLT\"}\n"
     }
   ]
 }

package/scripts/gepa/results/eval-1-variant-a.json

@@ -11,44 +11,44 @@
       "criteria": {
         "has_function": {
           "passed": false,
-          "reason": "No actual function code is provided, only a description of what the function should do. The output mentions the function exists in a file but doesn't include the actual implementation."
+          "reason": "regex heuristic"
         },
         "handles_edge_cases": {
           "passed": false,
-          "reason": "While edge cases are mentioned in examples (punctuation, spaces, mixed case), there is no actual code showing how these cases are handled. The logic is described but not implemented."
+          "reason": "regex heuristic"
         },
         "is_tested": {
           "passed": false,
-          "reason": "Examples are listed but no actual test code or test suite is provided. The examples are illustrative only, not executable tests."
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 23894,
-      "output": "I've written a palindrome checking function in `isPalindrome.ts`:\n\n**Function**: `isPalindrome(str: string): boolean`\n\n**Logic**:\n- Normalizes input to lowercase\n- Removes non-alphanumeric characters\n- Compares string with its reversed version\n\n**Examples**:\n- `isPalindrome(\"racecar\")` → `true`\n- `isPalindrome(\"A man, a plan, a canal: Panama\")` → `true`\n- `isPalindrome(\"hello\")` → `false`\n"
+      "judgeMode": "regex",
+      "duration": 6738,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHGBLeVSCWt7cJ2sEAb\"}\n"
     },
     {
       "taskId": "eval-002",
       "taskName": "refactor_code",
       "weight": 1.5,
       "passed": false,
-      "passRate": 0,
+      "passRate": 0.3333333333333333,
       "criteria": {
         "uses_async": {
           "passed": false,
-          "reason": "No async/await code was provided. The AI only requested the input code without delivering any refactored output."
+          "reason": "regex heuristic"
         },
         "preserves_behavior": {
           "passed": false,
-          "reason": "Cannot evaluate behavior preservation when no refactored code exists. The AI did not perform the refactoring task."
+          "reason": "regex heuristic"
         },
         "no_nested_callbacks": {
-          "passed": false,
-          "reason": "No refactored code was provided to evaluate callback nesting. The AI output is a request for input, not a solution."
+          "passed": true,
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 11272,
-      "output": "I need to see the code you want me to refactor. Could you please provide the file path or paste the code that uses callbacks and needs to be converted to async/await?\n"
+      "judgeMode": "regex",
+      "duration": 6973,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHGokedMoB4cyiFA121\"}\n"
     },
     {
       "taskId": "eval-003",
@@ -59,72 +59,72 @@
       "criteria": {
         "bug_fixed": {
           "passed": false,
-          "reason": "No code was provided in the output, so no bug could have been fixed"
+          "reason": "regex heuristic"
         },
         "no_new_bugs": {
           "passed": false,
-          "reason": "Empty output means no code to evaluate for new bugs"
+          "reason": "regex heuristic"
         },
         "explains_fix": {
           "passed": false,
-          "reason": "No explanation was provided; the output is completely empty"
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 25907,
-      "output": "\n"
+      "judgeMode": "regex",
+      "duration": 10853,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHHTwZeHvRNRhDMzVzG\"}\n"
     },
     {
       "taskId": "eval-004",
       "taskName": "add_feature",
       "weight": 1.5,
-      "passed": true,
-      "passRate": 0.75,
+      "passed": false,
+      "passRate": 0,
       "criteria": {
         "has_pagination": {
-          "passed": true,
-          "reason": "Pagination is described with page and limit parameters, and response includes pagination metadata"
+          "passed": false,
+          "reason": "regex heuristic"
         },
         "has_limit": {
-          "passed": true,
-          "reason": "Limit parameter is explicitly defined with bounds (1-100) and included in the response format"
+          "passed": false,
+          "reason": "regex heuristic"
         },
         "has_offset": {
           "passed": false,
-          "reason": "Output uses page-based pagination, not offset-based. No offset parameter is defined or mentioned"
+          "reason": "regex heuristic"
         },
         "backwards_compatible": {
-          "passed": true,
-          "reason": "Both parameters have defaults (page=1, limit=10) and existing endpoints like /users/:id remain unchanged"
+          "passed": false,
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 24592,
-      "output": "I've added pagination to the `/users` endpoint with the following features:\n\n**Query Parameters:**\n- `page` (default: 1, min: 1)\n- `limit` (default: 10, min: 1, max: 100)\n\n**Response Format:**\n```json\n{\n  \"data\": [...users],\n  \"pagination\": {\n    \"page\": 1,\n    \"limit\": 10,\n    \"total\": 100,\n    \"totalPages\": 10,\n    \"hasNext\": true,\n    \"hasPrev\": false\n  }\n}\n```\n\n**Features:**\n- Safe bounds checking (page ≥ 1, limit 1-100)\n- Pagination metadata for client navigation\n- `hasNext`/`hasPrev` flags for UI convenience\n- Maintains existing `/users/:id` endpoint unchanged\n\n**Usage:**\n- `/users` → first 10 users\n- `/users?page=2&limit=20` → users 21-40\n- `/users?limit=50` → first 50 users\n"
+      "judgeMode": "regex",
+      "duration": 5779,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHHxLrrmUqXyWRyvNX4\"}\n"
     },
     {
       "taskId": "eval-005",
       "taskName": "code_review",
       "weight": 1.3,
-      "passed": true,
-      "passRate": 0.6666666666666666,
+      "passed": false,
+      "passRate": 0,
       "criteria": {
         "identifies_security_issue": {
-          "passed": true,
-          "reason": "Output clearly identifies 4 critical security vulnerabilities: SQL injection, plaintext password comparison, insecure session tokens, and credential logging. Each is properly categorized with OWASP references."
+          "passed": false,
+          "reason": "regex heuristic"
         },
         "identifies_performance_issue": {
           "passed": false,
-          "reason": "Output does not identify or mention any performance issues. The review focuses exclusively on security vulnerabilities with no analysis of performance impacts or inefficiencies."
+          "reason": "regex heuristic"
         },
         "actionable_feedback": {
-          "passed": true,
-          "reason": "Output provides specific, actionable fixes for each issue: revert to parameterized queries, restore bcrypt comparison, use cryptographically secure tokens with proper cookie flags, and remove password logging. Each recommendation is clear and implementable."
+          "passed": false,
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 30817,
-      "output": "## PR Review - Critical Security Issues Found ⚠️\n\n### **SEVERITY: CRITICAL - DO NOT MERGE**\n\nThis PR introduces multiple severe security vulnerabilities:\n\n---\n\n### 🔴 **1. SQL Injection Vulnerability (Line 10)**\n```typescript\n- const user = await db.query('SELECT * FROM users WHERE email = $1', [email]);\n+ const user = await db.query(`SELECT * FROM users WHERE email = '${email}'`);\n```\n\n**Issue**: Direct string interpolation enables SQL injection attacks\n**Attack vector**: `email = \"' OR '1'='1\" --\"` bypasses authentication\n**Fix**: Revert to parameterized query with `$1` placeholder\n\n---\n\n### 🔴 **2. Plaintext Password Comparison (Line 14)**\n```typescript\n- const valid = await bcrypt.compare(password, user.passwordHash);\n+ const valid = password === user.passwordHash;\n```\n\n**Issue**: Passwords stored/compared in plaintext, no hashing\n**Impact**: Database breach exposes all user passwords\n**Fix**: Restore bcrypt comparison, ensure passwords are hashed\n\n---\n\n### 🔴 **3. Insecure Session Token (Line 18)**\n```typescript\n+ const token = email + ':' + Date.now();\n+ res.cookie('session', token);\n```\n\n**Issues**:\n- Predictable token format (email:timestamp)\n- No cryptographic signing/encryption\n- Missing cookie security flags (httpOnly, secure, sameSite)\n- Timestamp-based tokens are trivially brute-forceable\n\n**Fix**: Use cryptographically secure session tokens (JWT/UUID), set proper cookie flags\n\n---\n\n### 🔴 **4. Credential Logging (Line 22)**\n```typescript\n+ console.log(`Login: ${email} / ${password}`);\n```\n\n**Issue**: Plaintext passwords logged to stdout/logs\n**Impact**: Credentials exposed in log files, monitoring systems, terminal history\n**Fix**: Remove password logging entirely, only log email if needed\n\n---\n\n### 📋 **Summary**\n\n| Vulnerability | OWASP Category | Severity |\n|---------------|----------------|----------|\n| SQL Injection | A03:2021 | Critical |\n| Broken Auth | A07:2021 | Critical |\n| Insecure Session | A02:2021 | High |\n| Sensitive Data Exposure | A02:"
+      "judgeMode": "regex",
+      "duration": 6181,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHJQcftjSrVHj8nKxsp\"}\n"
     },
     {
       "taskId": "sm-001",
@@ -135,28 +135,28 @@
       "criteria": {
         "has_switch_case": {
           "passed": false,
-          "reason": "No code output provided - switch/case statement cannot be evaluated"
+          "reason": "regex heuristic"
         },
         "has_zod_schema": {
           "passed": false,
-          "reason": "No code output provided - Zod schema definition is missing"
+          "reason": "regex heuristic"
         },
         "validates_input": {
           "passed": false,
-          "reason": "No code output provided - input validation logic is absent"
+          "reason": "regex heuristic"
         },
         "returns_typed_response": {
           "passed": false,
-          "reason": "No code output provided - response structure cannot be verified"
+          "reason": "regex heuristic"
         },
         "handles_not_found": {
           "passed": false,
-          "reason": "No code output provided - error handling for missing frames is not implemented"
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 28547,
-      "output": "\n"
+      "judgeMode": "regex",
+      "duration": 4675,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHJtsnUUPF3pPMQY7uy\"}\n"
     },
     {
       "taskId": "sm-002",
@@ -167,24 +167,24 @@
       "criteria": {
         "has_delete_trigger": {
           "passed": false,
-          "reason": "The output is completely empty (only whitespace). No DELETE trigger is defined at all."
+          "reason": "regex heuristic"
         },
         "uses_fts_delete_syntax": {
           "passed": false,
-          "reason": "No FTS5 DELETE syntax is present. The output contains no code whatsoever."
+          "reason": "regex heuristic"
         },
         "references_old_row": {
           "passed": false,
-          "reason": "There is no trigger implementation to reference the OLD row values required for FTS deletion."
+          "reason": "regex heuristic"
         },
         "matches_column_list": {
           "passed": false,
-          "reason": "Without any trigger code, there is no column list matching the existing INSERT and UPDATE triggers."
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 21391,
-      "output": "\n"
+      "judgeMode": "regex",
+      "duration": 6137,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHKAmFvAN62p9tugpsP\"}\n"
     },
     {
       "taskId": "sm-003",
@@ -195,24 +195,24 @@
       "criteria": {
         "clears_old_interval": {
           "passed": false,
-          "reason": "No code provided in AI output to evaluate. Cannot verify that old intervals are cleared."
+          "reason": "regex heuristic"
         },
         "prevents_double_start": {
           "passed": false,
-          "reason": "No code provided in AI output to evaluate. Cannot verify double-start prevention logic."
+          "reason": "regex heuristic"
         },
         "has_getstate_method": {
           "passed": false,
-          "reason": "No code provided in AI output. The getState() method is not defined."
+          "reason": "regex heuristic"
         },
         "returns_correct_state_shape": {
           "passed": false,
-          "reason": "No code provided in AI output. Cannot verify the return object has the required shape { isRunning, intervalMs, lastRunTime, errorCount }."
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 56530,
-      "output": "\n"
+      "judgeMode": "regex",
+      "duration": 6263,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHKepxihvYkrrfWmRoU\"}\n"
     }
   ]
 }