@stackmemoryai/stackmemory 1.5.3 → 1.5.5

package/scripts/gepa/results/eval-1-variant-d.json

@@ -6,73 +6,73 @@
       "taskId": "eval-001",
       "taskName": "simple_function",
       "weight": 1,
-      "passed": true,
-      "passRate": 0.6666666666666666,
+      "passed": false,
+      "passRate": 0,
       "criteria": {
         "has_function": {
-          "passed": true,
-          "reason": "A complete, working function `isPalindrome` is defined in TypeScript with proper syntax, parameters, return type, and implementation."
+          "passed": false,
+          "reason": "regex heuristic"
         },
         "handles_edge_cases": {
-          "passed": true,
-          "reason": "The code handles edge cases: empty strings (returns true), mixed case (lowercase normalization), and special characters (regex removal of non-alphanumeric)."
+          "passed": false,
+          "reason": "regex heuristic"
         },
         "is_tested": {
           "passed": false,
-          "reason": "The output shows example outputs but provides no actual test cases, unit tests, or test function definitions. Examples are illustrative documentation, not executable tests."
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 26370,
-      "output": "I've written a palindrome checker function at `/private/tmp/gepa-eval-X4niM9/isPalindrome.ts`:\n\n```typescript\nexport function isPalindrome(str: string): boolean {\n  if (!str) return true;\n\n  const normalized = str.toLowerCase().replace(/[^a-z0-9]/g, '');\n  const reversed = normalized.split('').reverse().join('');\n\n  return normalized === reversed;\n}\n```\n\n**Features**:\n- Returns `true` for empty strings\n- Normalizes by converting to lowercase and removing non-alphanumeric characters\n- Compares the normalized string with its reverse\n\n**Examples**:\n- `isPalindrome(\"racecar\")` → `true`\n- `isPalindrome(\"A man, a plan, a canal: Panama\")` → `true`\n- `isPalindrome(\"hello\")` → `false`\n"
+      "judgeMode": "regex",
+      "duration": 6236,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHFLVHf4LRx9owtsUat\"}\n"
     },
     {
       "taskId": "eval-002",
       "taskName": "refactor_code",
       "weight": 1.5,
       "passed": false,
-      "passRate": 0,
+      "passRate": 0.3333333333333333,
       "criteria": {
         "uses_async": {
           "passed": false,
-          "reason": "No async/await code is provided; AI only requests the original code instead of demonstrating the refactored solution"
+          "reason": "regex heuristic"
         },
         "preserves_behavior": {
           "passed": false,
-          "reason": "No refactored code is shown, so behavior preservation cannot be verified"
+          "reason": "regex heuristic"
         },
         "no_nested_callbacks": {
-          "passed": false,
-          "reason": "No refactored code is demonstrated; the output is a request for input rather than a solution"
+          "passed": true,
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 8335,
-      "output": "I need to see the code you want to refactor. Could you please provide the file path or share the code that uses callbacks?\n"
+      "judgeMode": "regex",
+      "duration": 5342,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHFn1T5TkkCFboDp1aH\"}\n"
     },
     {
       "taskId": "eval-003",
       "taskName": "fix_bug",
       "weight": 1.2,
-      "passed": true,
-      "passRate": 0.6666666666666666,
+      "passed": false,
+      "passRate": 0,
       "criteria": {
         "bug_fixed": {
           "passed": false,
-          "reason": "The output only describes the fix but does not show the actual fixed code. No corrected code snippet is provided to verify the bug is actually fixed."
+          "reason": "regex heuristic"
         },
         "no_new_bugs": {
-          "passed": true,
-          "reason": "The suggested fix (changing `i <= items.length` to `i < items.length`) is a standard, correct solution for off-by-one errors and introduces no new bugs."
+          "passed": false,
+          "reason": "regex heuristic"
         },
         "explains_fix": {
-          "passed": true,
-          "reason": "The output clearly explains what was wrong (accessing undefined index 3 on a 3-element array) and why the fix works (using < instead of <=)."
+          "passed": false,
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 25447,
-      "output": "Fixed buggy-loop.js:6 - changed `i <= items.length` to `i < items.length` to prevent accessing `items[3]` (undefined) on a 3-element array.\n"
+      "judgeMode": "regex",
+      "duration": 4389,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHGFTCjb8PsbsYzRS1W\"}\n"
     },
     {
       "taskId": "eval-004",
@@ -83,116 +83,136 @@
       "criteria": {
         "has_pagination": {
           "passed": false,
-          "reason": "No code provided in output - empty code block"
+          "reason": "regex heuristic"
         },
         "has_limit": {
           "passed": false,
-          "reason": "No code provided in output - empty code block"
+          "reason": "regex heuristic"
         },
         "has_offset": {
           "passed": false,
-          "reason": "No code provided in output - empty code block"
+          "reason": "regex heuristic"
         },
         "backwards_compatible": {
           "passed": false,
-          "reason": "No code provided in output - empty code block"
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 20951,
-      "output": "\n"
+      "judgeMode": "regex",
+      "duration": 6872,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHGkbeXvwXwxysZTdhR\"}\n"
     },
     {
       "taskId": "eval-005",
       "taskName": "code_review",
       "weight": 1.3,
-      "passed": true,
-      "passRate": 0.6666666666666666,
+      "passed": false,
+      "passRate": 0,
       "criteria": {
         "identifies_security_issue": {
-          "passed": true,
-          "reason": "Output correctly identifies 5 legitimate security vulnerabilities: SQL injection, plaintext password comparison, sensitive data logging, insecure session tokens, and missing HttpOnly cookie flags with clear code examples and impact analysis"
+          "passed": false,
+          "reason": "regex heuristic"
         },
         "identifies_performance_issue": {
           "passed": false,
-          "reason": "Output does not identify any performance issues. The review focuses exclusively on security vulnerabilities with no mention of performance, scalability, or optimization concerns"
+          "reason": "regex heuristic"
         },
         "actionable_feedback": {
-          "passed": true,
-          "reason": "Output provides specific, actionable fixes for each issue including exact code examples (e.g., parameterized queries, bcrypt.compare, crypto.randomBytes, cookie options) that developers can immediately implement"
+          "passed": false,
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 29814,
-      "output": "## Critical Security Issues Found\n\nThis PR contains **MULTIPLE SEVERE SECURITY VULNERABILITIES** that must be rejected:\n\n### 🔴 CRITICAL: SQL Injection (Line 10)\n```typescript\nconst user = await db.query(`SELECT * FROM users WHERE email = '${email}'`);\n```\n**Issue**: Direct string interpolation enables SQL injection attacks  \n**Attack**: `email = \"' OR '1'='1' --\"` bypasses authentication  \n**Fix**: Revert to parameterized query: `db.query('SELECT * FROM users WHERE email = $1', [email])`\n\n### 🔴 CRITICAL: Plaintext Password Comparison (Line 14)\n```typescript\nconst valid = password === user.passwordHash;\n```\n**Issue**: Removes bcrypt password hashing, comparing plaintext to hash  \n**Impact**: Authentication always fails OR passwords stored in plaintext  \n**Fix**: Revert to `await bcrypt.compare(password, user.passwordHash)`\n\n### 🔴 CRITICAL: Sensitive Data Logging (Line 22)\n```typescript\nconsole.log(`Login: ${email} / ${password}`);\n```\n**Issue**: Logs plaintext passwords to console/logs  \n**Impact**: Password exposure in log files, GDPR/compliance violation  \n**Fix**: Remove this line entirely or log only: `console.log(\\`Login attempt: ${email}\\`)`\n\n### 🔴 HIGH: Insecure Session Token (Line 18)\n```typescript\nconst token = email + ':' + Date.now();\n```\n**Issue**: Predictable token generation (email + timestamp)  \n**Impact**: Session hijacking via token guessing  \n**Fix**: Use cryptographically secure random tokens: `crypto.randomBytes(32).toString('hex')`\n\n### 🟡 MEDIUM: Missing HttpOnly Cookie Flag\n```typescript\nres.cookie('session', token);\n```\n**Issue**: Cookie accessible via JavaScript (XSS vulnerability)  \n**Fix**: `res.cookie('session', token, { httpOnly: true, secure: true, sameSite: 'strict' })`\n\n## Recommendation\n\n**REJECT THIS PR IMMEDIATELY** - Contains multiple critical security vulnerabilities including SQL injection and plaintext password exposure. The \"quick fix\" comment on line 9 is a red flag for introducing security issues.\n\nAll changes should be *"
+      "judgeMode": "regex",
+      "duration": 10308,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHHXPvni9SXWYQD2ML2\"}\n"
     },
     {
       "taskId": "sm-001",
       "taskName": "add_mcp_tool_handler",
       "weight": 1.5,
-      "passed": true,
-      "passRate": 1,
+      "passed": false,
+      "passRate": 0,
       "criteria": {
         "has_switch_case": {
-          "passed": true,
-          "reason": "Code shows a case statement 'get_frame_summary' within a switch/case structure, following the dispatch pattern"
+          "passed": false,
+          "reason": "regex heuristic"
         },
         "has_zod_schema": {
-          "passed": true,
-          "reason": "Zod schema is defined inline: z.object({ frameId: z.string() }), validating the required frameId parameter"
+          "passed": false,
+          "reason": "regex heuristic"
         },
         "validates_input": {
-          "passed": true,
-          "reason": "Input validation occurs via .parse(args) which will throw if frameId is missing or not a string"
+          "passed": false,
+          "reason": "regex heuristic"
         },
         "returns_typed_response": {
-          "passed": true,
-          "reason": "Return statement explicitly returns an object with the four required properties: frameId, name, status, eventCount"
+          "passed": false,
+          "reason": "regex heuristic"
         },
         "handles_not_found": {
-          "passed": true,
-          "reason": "Code includes explicit null check (if (!frame)) with error thrown when frame is not found in the frames map"
+          "passed": false,
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 33763,
-      "output": "I've added the `get_frame_summary` handler to the MCP server dispatch pattern. The implementation:\n\n**Added case** (mcp-dispatch-stub.ts:32-42):\n```typescript\ncase 'get_frame_summary': {\n  const input = z.object({ frameId: z.string() }).parse(args);\n  const frame = frames.get(input.frameId);\n  if (!frame) throw new Error(`Frame not found: ${input.frameId}`);\n  return {\n    frameId: frame.id,\n    name: frame.name,\n    status: frame.status,\n    eventCount: frame.events.length\n  };\n}\n```\n\n**Features**:\n- ✓ Zod validation for required `frameId` string parameter\n- ✓ Follows existing switch/case dispatch pattern\n- ✓ Returns `{ frameId, name, status, eventCount }` as specified\n- ✓ Error handling for missing frames (consistent with `close_frame`)\n- ✓ Calculates `eventCount` from `frame.events.length`\n\nThe handler integrates seamlessly with the existing `start_frame` and `close_frame` tools.\n"
+      "judgeMode": "regex",
+      "duration": 6548,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHHpuPC36UFe4w8R6LA\"}\n"
     },
     {
       "taskId": "sm-002",
       "taskName": "fix_fts5_trigger_bug",
       "weight": 1.8,
-      "passed": true,
-      "passRate": 1,
+      "passed": false,
+      "passRate": 0,
       "criteria": {
         "has_delete_trigger": {
-          "passed": true,
-          "reason": "A DELETE trigger named 'frames_ad' is defined with AFTER DELETE ON frames syntax"
+          "passed": false,
+          "reason": "regex heuristic"
         },
         "uses_fts_delete_syntax": {
-          "passed": true,
-          "reason": "The trigger uses FTS5 special syntax with INSERT INTO frames_fts(frames_fts, ...) VALUES ('delete', ...) which is the correct way to delete from FTS index"
+          "passed": false,
+          "reason": "regex heuristic"
         },
         "references_old_row": {
-          "passed": true,
-          "reason": "The trigger references old.rowid, old.name, old.digest_text, and old.inputs, old.outputs to access the deleted row values"
+          "passed": false,
+          "reason": "regex heuristic"
         },
         "matches_column_list": {
-          "passed": true,
-          "reason": "The column list (rowid, name, digest_text, inputs, outputs) matches the FTS index structure referenced in the INSERT and UPDATE triggers"
+          "passed": false,
+          "reason": "regex heuristic"
         }
       },
-      "judgeMode": "llm",
-      "duration": 34003,
-      "output": "I've added the missing DELETE trigger following the exact pattern of the existing triggers:\n\n**Pattern analysis:**\n- INSERT trigger (`frames_ai`): Adds new row to FTS index\n- UPDATE trigger (`frames_au`): Deletes old row, inserts new row\n- DELETE trigger (`frames_ad`): Deletes row from FTS index\n\n**Implementation:**\n```sql\nCREATE TRIGGER IF NOT EXISTS frames_ad AFTER DELETE ON frames BEGIN\n  INSERT INTO frames_fts(frames_fts, rowid, name, digest_text, inputs, outputs)\n  VALUES ('delete', old.rowid, old.name, old.digest_text, old.inputs, old.outputs);\nEND;\n```\n\nThe trigger uses the FTS5 special command syntax where `INSERT INTO frames_fts(frames_fts, ...)` with `'delete'` as the first value removes the entry from the full-text index.\n"
+      "judgeMode": "regex",
+      "duration": 6290,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHJHsrxx3PXJcuPnJio\"}\n"
     },
     {
       "taskId": "sm-003",
       "taskName": "daemon_service_lifecycle",
       "weight": 1.5,
       "passed": false,
-      "duration": 120042,
-      "error": "claude timed out after 120000ms"
+      "passRate": 0,
+      "criteria": {
+        "clears_old_interval": {
+          "passed": false,
+          "reason": "regex heuristic"
+        },
+        "prevents_double_start": {
+          "passed": false,
+          "reason": "regex heuristic"
+        },
+        "has_getstate_method": {
+          "passed": false,
+          "reason": "regex heuristic"
+        },
+        "returns_correct_state_shape": {
+          "passed": false,
+          "reason": "regex heuristic"
+        }
+      },
+      "judgeMode": "regex",
+      "duration": 6657,
+      "output": "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You have reached your specified API usage limits. You will regain access on 2026-04-01 at 00:00 UTC.\"},\"request_id\":\"req_011CYrHJmYFr5sCJFhW9h9Qm\"}\n"
     }
   ]
 }

package/scripts/gepa/state.json

@@ -1,14 +1,14 @@
 {
   "currentGeneration": 1,
-  "bestVariant": "variant-d",
-  "bestScore": 0.4985250737463126,
-  "targetPath": "/Users/jwu/Dev/stackmemory/CLAUDE.md",
+  "bestVariant": "baseline",
+  "bestScore": 0.04424778761061947,
+  "targetPath": "/Users/jwu/Dev/rize/sol/CLAUDE.md",
   "history": [
     {
       "generation": 0,
       "variant": "baseline",
       "action": "init",
-      "timestamp": "2026-02-18T22:59:36.437Z"
+      "timestamp": "2026-03-08T20:55:25.777Z"
     },
     {
       "generation": 1,
@@ -19,35 +19,19 @@
         "variant-c",
         "variant-d"
       ],
-      "timestamp": "2026-02-18T23:03:13.997Z"
+      "timestamp": "2026-03-08T20:56:32.495Z"
     },
     {
       "generation": 1,
       "action": "select",
       "scores": [
-        {
-          "variant": "variant-d",
-          "score": 0.4985250737463126
-        },
-        {
-          "variant": "variant-b",
-          "score": 0.3938053097345133
-        },
-        {
-          "variant": "variant-c",
-          "score": 0.23230088495575218
-        },
         {
           "variant": "baseline",
-          "score": 0.21165191740412978
-        },
-        {
-          "variant": "variant-a",
-          "score": 0.1762536873156342
+          "score": 0.04424778761061947
         }
       ],
-      "best": "variant-d",
-      "timestamp": "2026-02-18T23:23:18.544Z"
+      "best": "baseline",
+      "timestamp": "2026-03-08T20:58:11.649Z"
     }
   ]
 }