@stackmemoryai/stackmemory 1.6.0 → 1.6.2

package/scripts/gepa/results/eval-1-baseline.json

@@ -1,218 +0,0 @@
-{
-  "variant": "baseline",
-  "generation": 1,
-  "results": [
-    {
-      "taskId": "eval-001",
-      "taskName": "simple_function",
-      "weight": 1,
-      "passed": false,
-      "passRate": 0.3333333333333333,
-      "criteria": {
-        "has_function": {
-          "passed": true,
-          "reason": "A properly defined function `is_palindrome` with type hints and a working implementation is present."
-        },
-        "handles_edge_cases": {
-          "passed": false,
-          "reason": "The function only handles spaces and case sensitivity, but does not handle punctuation, special characters, or numbers. A true palindrome checker should ignore these. For example, 'A man, a plan, a canal: Panama' would fail."
-        },
-        "is_tested": {
-          "passed": false,
-          "reason": "No test cases or assertions are provided to demonstrate that the function works correctly."
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 9807,
-      "output": "```python\ndef is_palindrome(s: str) -> bool:\n    s = s.lower().replace(\" \", \"\")\n    return s == s[::-1]\n```\n"
-    },
-    {
-      "taskId": "eval-002",
-      "taskName": "refactor_code",
-      "weight": 1.5,
-      "passed": false,
-      "passRate": 0,
-      "criteria": {
-        "uses_async": {
-          "passed": false,
-          "reason": "No async/await code was provided in the output. The AI only requested the code to be refactored without delivering any refactored code."
-        },
-        "preserves_behavior": {
-          "passed": false,
-          "reason": "No refactored code exists to evaluate whether behavior is preserved. The output is a request for input, not a solution."
-        },
-        "no_nested_callbacks": {
-          "passed": false,
-          "reason": "No code transformation was performed. The output does not demonstrate elimination of nested callbacks in any actual code."
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 11792,
-      "output": "I don't see any specific code provided. Could you share the code you'd like refactored, or point me to the file path?\n"
-    },
-    {
-      "taskId": "eval-003",
-      "taskName": "fix_bug",
-      "weight": 1.2,
-      "passed": false,
-      "passRate": 0.3333333333333333,
-      "criteria": {
-        "bug_fixed": {
-          "passed": false,
-          "reason": "No code was provided by the user, so no bug could be fixed. The AI correctly identified the missing input but did not fix any bug."
-        },
-        "no_new_bugs": {
-          "passed": true,
-          "reason": "The AI did not write any code, so no new bugs were introduced."
-        },
-        "explains_fix": {
-          "passed": false,
-          "reason": "No explanation of a fix was provided because no code was available to analyze or fix."
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 12640,
-      "output": "I don't see any code in your message. Could you share the loop you'd like me to fix?\n"
-    },
-    {
-      "taskId": "eval-004",
-      "taskName": "add_feature",
-      "weight": 1.5,
-      "passed": false,
-      "passRate": 0,
-      "criteria": {
-        "has_pagination": {
-          "passed": false,
-          "reason": "No pagination implementation provided. AI only asks for clarification instead of delivering pagination code."
-        },
-        "has_limit": {
-          "passed": false,
-          "reason": "No limit parameter implemented in any code. The output contains no actual code implementation."
-        },
-        "has_offset": {
-          "passed": false,
-          "reason": "No offset parameter implemented in any code. The output contains no actual code implementation."
-        },
-        "backwards_compatible": {
-          "passed": false,
-          "reason": "No implementation provided to evaluate backwards compatibility. The output is a question rather than a solution."
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 10855,
-      "output": "Which API endpoint would you like to add pagination to? Please share the file path or endpoint name.\n"
-    },
-    {
-      "taskId": "eval-005",
-      "taskName": "code_review",
-      "weight": 1.3,
-      "passed": false,
-      "passRate": 0,
-      "criteria": {
-        "identifies_security_issue": {
-          "passed": false,
-          "reason": "No security issues are identified. The output is a request for clarification rather than an actual review."
-        },
-        "identifies_performance_issue": {
-          "passed": false,
-          "reason": "No performance issues are identified. The output does not analyze any code."
-        },
-        "actionable_feedback": {
-          "passed": false,
-          "reason": "No actionable feedback is provided. The output only requests missing information instead of providing a review."
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 26949,
-      "output": "I need a PR number or URL to review. Which PR would you like me to review?\n"
-    },
-    {
-      "taskId": "sm-001",
-      "taskName": "add_mcp_tool_handler",
-      "weight": 1.5,
-      "passed": true,
-      "passRate": 1,
-      "criteria": {
-        "has_switch_case": {
-          "passed": true,
-          "reason": "The output shows a proper switch case statement with 'get_frame_summary' as the case label, following the switch/case dispatch pattern requested."
-        },
-        "has_zod_schema": {
-          "passed": true,
-          "reason": "Zod schema is defined with `z.object({ frameId: z.string() }).parse(args)` which validates the input object."
-        },
-        "validates_input": {
-          "passed": true,
-          "reason": "Input validation is performed via Zod's `.parse()` method, which will throw if the input doesn't match the schema (frameId must be a string)."
-        },
-        "returns_typed_response": {
-          "passed": true,
-          "reason": "The handler returns an object with the exact shape specified: `{ frameId, name, status, eventCount }` with appropriate values extracted from the frame object."
-        },
-        "handles_not_found": {
-          "passed": true,
-          "reason": "The code includes explicit error handling with `if (!frame) throw new Error(...)` when the frame doesn't exist, consistent with the close_frame pattern mentioned."
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 97214,
-      "output": "Please approve the file write permission and I'll apply the change. The handler follows the exact same pattern as `close_frame`:\n\n```ts\ncase 'get_frame_summary': {\n  const input = z.object({ frameId: z.string() }).parse(args);\n  const frame = frames.get(input.frameId);\n  if (!frame) throw new Error(`Frame not found: ${input.frameId}`);\n  return { frameId: frame.id, name: frame.name, status: frame.status, eventCount: frame.events.length };\n}\n```\n\n- Zod validates `frameId` as a required string\n- Throws if the frame doesn't exist (consistent with `close_frame`)\n- Returns `{ frameId, name, status, eventCount }` where `eventCount` is derived from `frame.events.length`\n"
-    },
-    {
-      "taskId": "sm-002",
-      "taskName": "fix_fts5_trigger_bug",
-      "weight": 1.8,
-      "passed": true,
-      "passRate": 1,
-      "criteria": {
-        "has_delete_trigger": {
-          "passed": true,
-          "reason": "A DELETE trigger named 'frames_ad' is explicitly defined with CREATE TRIGGER IF NOT EXISTS syntax, targeting the frames table with AFTER DELETE event."
-        },
-        "uses_fts_delete_syntax": {
-          "passed": true,
-          "reason": "The trigger uses the correct FTS5 delete pattern: INSERT INTO frames_fts with the special 'delete' command as the first value, which is the standard FTS5 mechanism for removing indexed rows."
-        },
-        "references_old_row": {
-          "passed": true,
-          "reason": "The trigger references the OLD row context (old.rowid, old.name, old.digest_text, old.inputs, old.outputs) which is the correct way to access deleted row values in an AFTER DELETE trigger."
-        },
-        "matches_column_list": {
-          "passed": true,
-          "reason": "The column list in the INSERT statement (frames_fts, rowid, name, digest_text, inputs, outputs) matches the pattern used in the existing INSERT and UPDATE triggers, maintaining consistency across all three triggers."
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 28917,
-      "output": "The edit needs your approval. The trigger to add follows the same FTS5 \"delete\" command pattern used in the UPDATE trigger — issuing a special `INSERT ... VALUES ('delete', ...)` to remove the old row from the index:\n\n```sql\n-- Sync trigger: DELETE\nCREATE TRIGGER IF NOT EXISTS frames_ad AFTER DELETE ON frames BEGIN\n  INSERT INTO frames_fts(frames_fts, rowid, name, digest_text, inputs, outputs)\n  VALUES ('delete', old.rowid, old.name, old.digest_text, old.inputs, old.outputs);\nEND;\n```\n\nPlease approve the file write (or let me know if you'd like to apply it differently).\n"
-    },
-    {
-      "taskId": "sm-003",
-      "taskName": "daemon_service_lifecycle",
-      "weight": 1.5,
-      "passed": true,
-      "passRate": 1,
-      "criteria": {
-        "clears_old_interval": {
-          "passed": true,
-          "reason": "The fix adds `if (this.isRunning) this.stop();` at the start of `start()`, which calls `clearInterval` on the old `intervalId` before creating a new one."
-        },
-        "prevents_double_start": {
-          "passed": true,
-          "reason": "The guard `if (this.isRunning) this.stop();` prevents double-start by stopping any existing interval before starting a new one, ensuring only one interval runs at a time."
-        },
-        "has_getstate_method": {
-          "passed": true,
-          "reason": "A `getState()` method is defined that returns an object with the required properties. The code is syntactically correct and callable."
-        },
-        "returns_correct_state_shape": {
-          "passed": true,
-          "reason": "The `getState()` method returns an object with all four required properties: `isRunning`, `intervalMs`, `lastRunTime`, and `errorCount` with appropriate values derived from instance state."
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 44838,
-      "output": "It seems write permissions aren't granted. Here's the full fix to apply:\n\n**Bug fix** — add a guard at the top of `start()` so any existing interval is cleared before a new one is created:\n\n```diff\n-  // BUG: No guard against double-start — calling start() twice\n-  // creates two intervals but only stores the second one.\n-  // The first interval leaks and keeps running forever.\n   start(): void {\n+    if (this.isRunning) this.stop(); // clear any existing interval before starting\n     this.isRunning = true;\n```\n\n**Remove the now-stale bug comment** from `updateConfig` (it's no longer accurate once `start()` self-clears).\n\n**Add `getState()`** after `updateConfig`:\n\n```ts\ngetState(): ServiceState {\n  return {\n    isRunning: this.isRunning,\n    intervalMs: this.config.interval * 60 * 1000,\n    lastRunTime: this.lastRunTime,\n    errorCount: this.errorCount,\n  };\n}\n```\n\n**Why this works:**\n- `start()` calling `stop()` first ensures `clearInterval` is always called on the old `intervalId` before overwriting it — no leak possible.\n- `updateConfig` already called `stop()` before `start()`, but the root bug was that `start()` itself had no guard. Now both paths are safe.\n- `getState()` reads the private fields directly; `intervalMs` is derived from config the same way `start()` computes it, keeping them in sync.\n"
-    }
-  ]
-}

package/scripts/gepa/results/eval-1-variant-a.json

@@ -1,218 +0,0 @@
-{
-  "variant": "variant-a",
-  "generation": 1,
-  "results": [
-    {
-      "taskId": "eval-001",
-      "taskName": "simple_function",
-      "weight": 1,
-      "passed": false,
-      "passRate": 0.3333333333333333,
-      "criteria": {
-        "has_function": {
-          "passed": true,
-          "reason": "A complete, working function named is_palindrome is defined with proper syntax and type hints"
-        },
-        "handles_edge_cases": {
-          "passed": false,
-          "reason": "The function only handles spaces and case sensitivity, but does not handle punctuation, special characters, or empty strings in a robust way"
-        },
-        "is_tested": {
-          "passed": false,
-          "reason": "No test cases or assertions are provided to verify the function works correctly"
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 12084,
-      "output": "```python\ndef is_palindrome(s: str) -> bool:\n    s = s.lower().replace(\" \", \"\")\n    return s == s[::-1]\n```\n"
-    },
-    {
-      "taskId": "eval-002",
-      "taskName": "refactor_code",
-      "weight": 1.5,
-      "passed": false,
-      "passRate": 0,
-      "criteria": {
-        "uses_async": {
-          "passed": false,
-          "reason": "No async/await code is present in the output. The AI only requested the code to be refactored without providing any refactored implementation."
-        },
-        "preserves_behavior": {
-          "passed": false,
-          "reason": "No refactored code is provided, so behavior cannot be verified or preserved. The output is merely a request for input."
-        },
-        "no_nested_callbacks": {
-          "passed": false,
-          "reason": "No code transformation was attempted. The output does not demonstrate removal of nested callbacks since no refactored code was delivered."
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 19736,
-      "output": "Please share the code you'd like me to refactor.\n"
-    },
-    {
-      "taskId": "eval-003",
-      "taskName": "fix_bug",
-      "weight": 1.2,
-      "passed": false,
-      "passRate": 0,
-      "criteria": {
-        "bug_fixed": {
-          "passed": false,
-          "reason": "No code was provided to fix, so no bug was actually corrected"
-        },
-        "no_new_bugs": {
-          "passed": false,
-          "reason": "Cannot evaluate without code being analyzed or modified"
-        },
-        "explains_fix": {
-          "passed": false,
-          "reason": "No fix was explained because the AI correctly identified that no code was provided to work with"
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 16159,
-      "output": "I don't see any code shared yet. Could you paste the loop you'd like me to fix?\n"
-    },
-    {
-      "taskId": "eval-004",
-      "taskName": "add_feature",
-      "weight": 1.5,
-      "passed": false,
-      "passRate": 0,
-      "criteria": {
-        "has_pagination": {
-          "passed": false,
-          "reason": "No pagination implementation provided. AI only asks for clarification instead of delivering working code."
-        },
-        "has_limit": {
-          "passed": false,
-          "reason": "No limit parameter implemented in any code. Only a question asking for more information."
-        },
-        "has_offset": {
-          "passed": false,
-          "reason": "No offset parameter implemented in any code. Only a question asking for more information."
-        },
-        "backwards_compatible": {
-          "passed": false,
-          "reason": "No code provided to evaluate backwards compatibility. The response is a request for clarification, not a solution."
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 27626,
-      "output": "Which API endpoint do you want to add pagination to? Please share the file path or endpoint name so I can look at the current implementation.\n"
-    },
-    {
-      "taskId": "eval-005",
-      "taskName": "code_review",
-      "weight": 1.3,
-      "passed": true,
-      "passRate": 0.6666666666666666,
-      "criteria": {
-        "identifies_security_issue": {
-          "passed": true,
-          "reason": "The output identifies five distinct security issues: SQL injection, plaintext password comparison, weak session tokens, insecure cookie flags, and plaintext password logging. These are all legitimate security vulnerabilities."
-        },
-        "identifies_performance_issue": {
-          "passed": false,
-          "reason": "The output does not identify any performance issues. It mentions that the PR comment claims 'direct string interpolation for faster queries' but explicitly refutes this claim as incorrect. It does not identify or discuss any actual performance problems in the code."
-        },
-        "actionable_feedback": {
-          "passed": true,
-          "reason": "The output provides specific, actionable recommendations for each issue: revert SQL to parameterized queries, use bcrypt.compare instead of direct comparison, use crypto.randomBytes for tokens, add specific cookie flags (httpOnly, secure, sameSite), and remove password logging. Each recommendation includes concrete code examples or clear guidance on what to change."
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 37583,
-      "output": "This PR has **critical security issues** — do not merge.\n\n## Issues Found\n\n### 1. SQL Injection (Critical)\n```diff\n- const user = await db.query('SELECT * FROM users WHERE email = $1', [email]);\n+ const user = await db.query(`SELECT * FROM users WHERE email = '${email}'`);\n```\nString interpolation into a SQL query allows any user to inject arbitrary SQL. The original parameterized query was correct — revert this immediately.\n\n### 2. Plaintext Password Comparison (Critical)\n```diff\n- const valid = await bcrypt.compare(password, user.passwordHash);\n+ const valid = password === user.passwordHash;\n```\nThis compares the raw password string directly against what should be a bcrypt hash. This means either (a) authentication is completely broken, or (b) passwords are stored in plaintext. Either way, revert to `bcrypt.compare`.\n\n### 3. Weak/Predictable Session Token (High)\n```js\nconst token = email + ':' + Date.now();\n```\nTokens built from `email + timestamp` are trivially guessable and forgeable. Use a cryptographically secure random value (e.g., `crypto.randomBytes(32).toString('hex')`).\n\n### 4. Session Cookie Missing Security Flags (High)\n```js\nres.cookie('session', token);\n```\nNo `httpOnly`, `secure`, or `sameSite` flags. This exposes the cookie to XSS and CSRF attacks. Should be at minimum:\n```js\nres.cookie('session', token, { httpOnly: true, secure: true, sameSite: 'strict' });\n```\n\n### 5. Password Logged in Plaintext (High)\n```js\nconsole.log(`Login: ${email} / ${password}`);\n```\nPasswords are being written to logs. Remove immediately — this violates basic security hygiene and likely compliance requirements (GDPR, SOC2, etc.).\n\n---\n\n**Summary**: This PR degrades security at every step — SQL injection, broken auth, weak tokens, insecure cookies, and leaked credentials. The comment \"Quick fix: direct string interpolation for faster queries\" is incorrect; parameterized queries have negligible overhead. **Reject and revert all changes.**\n"
-    },
-    {
-      "taskId": "sm-001",
-      "taskName": "add_mcp_tool_handler",
-      "weight": 1.5,
-      "passed": true,
-      "passRate": 1,
-      "criteria": {
-        "has_switch_case": {
-          "passed": true,
-          "reason": "A switch case for 'get_frame_summary' is clearly defined with proper case syntax"
-        },
-        "has_zod_schema": {
-          "passed": true,
-          "reason": "Zod schema is present: z.object({ frameId: z.string() }) validates the required string input"
-        },
-        "validates_input": {
-          "passed": true,
-          "reason": "Input validation occurs via z.object().parse(args), which will throw if validation fails"
-        },
-        "returns_typed_response": {
-          "passed": true,
-          "reason": "Returns an object with all four required fields: frameId, name, status, eventCount in the correct shape"
-        },
-        "handles_not_found": {
-          "passed": true,
-          "reason": "Includes explicit check for missing frame with frames.get() followed by error throw with descriptive message"
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 30073,
-      "output": "Please approve the file write permission. The edit adds this case to the switch:\n\n```ts\ncase 'get_frame_summary': {\n  const input = z.object({ frameId: z.string() }).parse(args);\n  const frame = frames.get(input.frameId);\n  if (!frame) throw new Error(`Frame not found: ${input.frameId}`);\n  return { frameId: frame.id, name: frame.name, status: frame.status, eventCount: frame.events.length };\n}\n```\n\nIt follows the same pattern as `close_frame`: Zod parse → lookup → error if missing → return the requested shape.\n"
-    },
-    {
-      "taskId": "sm-002",
-      "taskName": "fix_fts5_trigger_bug",
-      "weight": 1.8,
-      "passed": false,
-      "passRate": 0,
-      "criteria": {
-        "has_delete_trigger": {
-          "passed": false,
-          "reason": "No DELETE trigger is actually created in the output; only a request for approval is made"
-        },
-        "uses_fts_delete_syntax": {
-          "passed": false,
-          "reason": "No FTS5 DELETE syntax (INSERT INTO frames_fts) is present in the output"
-        },
-        "references_old_row": {
-          "passed": false,
-          "reason": "No OLD row references are used; the output contains no trigger code"
-        },
-        "matches_column_list": {
-          "passed": false,
-          "reason": "No column list comparison exists; the output is just a request without implementation"
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 24722,
-      "output": "Please approve the write to `fts5-triggers.sql` so I can add the DELETE trigger.\n"
-    },
-    {
-      "taskId": "sm-003",
-      "taskName": "daemon_service_lifecycle",
-      "weight": 1.5,
-      "passed": true,
-      "passRate": 1,
-      "criteria": {
-        "clears_old_interval": {
-          "passed": true,
-          "reason": "The fix adds `if (this.isRunning) this.stop()` at the start of `start()`, which calls `clearInterval()` before creating a new interval. This directly addresses the timer leak."
-        },
-        "prevents_double_start": {
-          "passed": true,
-          "reason": "The guard `if (this.isRunning) this.stop()` prevents double-start by stopping any existing interval before starting a new one, whether called directly or via `updateConfig()`."
-        },
-        "has_getstate_method": {
-          "passed": true,
-          "reason": "A `getState()` method is defined in the output that returns the required ServiceState object with all four properties."
-        },
-        "returns_correct_state_shape": {
-          "passed": true,
-          "reason": "The `getState()` method returns an object with the exact shape requested: `isRunning`, `intervalMs`, `lastRunTime`, and `errorCount`. The intervalMs calculation (`this.config.interval * 60 * 1000`) correctly converts minutes to milliseconds."
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 69511,
-      "output": "It looks like write permissions aren't granted for this directory. Here are the two changes needed:\n\n**Fix 1 — Guard `start()` against double-start** (`leaky-service.ts:28`):\n```ts\nstart(): void {\n  if (this.isRunning) this.stop(); // clear any existing interval before starting\n  this.isRunning = true;\n  // ... rest unchanged\n}\n```\n\nThis ensures `stop()` (which calls `clearInterval`) always runs before a new interval is created, preventing leaks from both direct double-calls and `updateConfig()` re-entry.\n\n**Fix 2 — Remove stale comment from `updateConfig()`** (the bug comment is now inaccurate since `start()` handles it).\n\n**Fix 3 — Add `getState()`** (replace the TODO comment):\n```ts\ngetState(): ServiceState {\n  return {\n    isRunning: this.isRunning,\n    intervalMs: this.config.interval * 60 * 1000,\n    lastRunTime: this.lastRunTime,\n    errorCount: this.errorCount,\n  };\n}\n```\n\nThe root cause was that `start()` unconditionally set `this.isRunning = true` and created a new interval without clearing the previous one. The fix moves responsibility to `start()` itself — if already running, stop first — rather than relying on every caller to avoid double-starts.\n"
-    }
-  ]
-}

package/scripts/gepa/results/eval-1-variant-b.json

@@ -1,218 +0,0 @@
-{
-  "variant": "variant-b",
-  "generation": 1,
-  "results": [
-    {
-      "taskId": "eval-001",
-      "taskName": "simple_function",
-      "weight": 1,
-      "passed": false,
-      "passRate": 0.3333333333333333,
-      "criteria": {
-        "has_function": {
-          "passed": true,
-          "reason": "A real, working function named is_palindrome is defined with proper syntax, type hints, and implementation"
-        },
-        "handles_edge_cases": {
-          "passed": false,
-          "reason": "The function only handles lowercase conversion and space removal, but doesn't handle punctuation, special characters, or empty strings. A complete edge case handler would address these scenarios"
-        },
-        "is_tested": {
-          "passed": false,
-          "reason": "No test cases or assertions are provided in the output to verify the function works correctly"
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 58252,
-      "output": "```python\ndef is_palindrome(s: str) -> bool:\n    s = s.lower().replace(\" \", \"\")\n    return s == s[::-1]\n```\n"
-    },
-    {
-      "taskId": "eval-002",
-      "taskName": "refactor_code",
-      "weight": 1.5,
-      "passed": false,
-      "passRate": 0,
-      "criteria": {
-        "uses_async": {
-          "passed": false,
-          "reason": "No async/await code is present in the output. The AI only requests the source code without providing any refactored implementation."
-        },
-        "preserves_behavior": {
-          "passed": false,
-          "reason": "No refactored code is provided, so behavior cannot be preserved or verified."
-        },
-        "no_nested_callbacks": {
-          "passed": false,
-          "reason": "No code transformation is shown. The AI did not perform the refactoring task and therefore did not eliminate nested callbacks."
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 20841,
-      "output": "Please share the code you'd like me to refactor.\n"
-    },
-    {
-      "taskId": "eval-003",
-      "taskName": "fix_bug",
-      "weight": 1.2,
-      "passed": false,
-      "passRate": 0.3333333333333333,
-      "criteria": {
-        "bug_fixed": {
-          "passed": false,
-          "reason": "No code was provided to the AI, so no bug could be fixed. The AI correctly identified this but did not fix anything."
-        },
-        "no_new_bugs": {
-          "passed": true,
-          "reason": "The AI did not write any code, so no new bugs were introduced."
-        },
-        "explains_fix": {
-          "passed": false,
-          "reason": "No explanation of a fix was provided because the AI was waiting for code input. A generic explanation of off-by-one errors was not given."
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 35783,
-      "output": "I don't see any code or file provided to fix. Could you share the code with the off-by-one error?\n"
-    },
-    {
-      "taskId": "eval-004",
-      "taskName": "add_feature",
-      "weight": 1.5,
-      "passed": false,
-      "passRate": 0,
-      "criteria": {
-        "has_pagination": {
-          "passed": false,
-          "reason": "No pagination code is implemented; AI only asked a clarifying question"
-        },
-        "has_limit": {
-          "passed": false,
-          "reason": "No limit parameter is defined or implemented"
-        },
-        "has_offset": {
-          "passed": false,
-          "reason": "No offset parameter is defined or implemented"
-        },
-        "backwards_compatible": {
-          "passed": false,
-          "reason": "No implementation exists to maintain backwards compatibility"
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 73593,
-      "output": "Which API endpoint should I add pagination to? Please share the file path or endpoint name.\n"
-    },
-    {
-      "taskId": "eval-005",
-      "taskName": "code_review",
-      "weight": 1.3,
-      "passed": false,
-      "passRate": 0,
-      "criteria": {
-        "identifies_security_issue": {
-          "passed": false,
-          "reason": "The AI output does not identify any security issues. It only requests clarification on which PR to review."
-        },
-        "identifies_performance_issue": {
-          "passed": false,
-          "reason": "The AI output does not identify any performance issues. It only requests clarification on which PR to review."
-        },
-        "actionable_feedback": {
-          "passed": false,
-          "reason": "The AI output provides no actionable feedback about code. It only asks for a PR number or URL without performing any actual review."
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 20273,
-      "output": "I need a PR number or URL to review. Could you provide the PR you'd like me to review?\n"
-    },
-    {
-      "taskId": "sm-001",
-      "taskName": "add_mcp_tool_handler",
-      "weight": 1.5,
-      "passed": true,
-      "passRate": 1,
-      "criteria": {
-        "has_switch_case": {
-          "passed": true,
-          "reason": "Code includes a `case 'get_frame_summary':` statement, which is a real switch/case dispatch pattern matching the requirement"
-        },
-        "has_zod_schema": {
-          "passed": true,
-          "reason": "Code includes `z.object({ frameId: z.string() }).parse(args)`, which is a real Zod schema definition and validation"
-        },
-        "validates_input": {
-          "passed": true,
-          "reason": "Zod schema validation is executed with `.parse(args)`, which will throw if validation fails, properly validating the required string frameId"
-        },
-        "returns_typed_response": {
-          "passed": true,
-          "reason": "Returns an object with all four required fields: frameId, name, status, and eventCount with appropriate types derived from frame object properties"
-        },
-        "handles_not_found": {
-          "passed": true,
-          "reason": "Code includes null check `if (!frame)` with explicit error throw, matching the error handling pattern shown in the example"
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 28089,
-      "output": "The edit was denied. Here's the handler to add in place of the `// TODO` comment:\n\n```ts\ncase 'get_frame_summary': {\n  const input = z.object({ frameId: z.string() }).parse(args);\n  const frame = frames.get(input.frameId);\n  if (!frame) throw new Error(`Frame not found: ${input.frameId}`);\n  return {\n    frameId: frame.id,\n    name: frame.name,\n    status: frame.status,\n    eventCount: frame.events.length,\n  };\n}\n```\n\n**What it does:**\n- Zod validates `frameId` as a required string (line 18 pattern)\n- Looks up the frame, throws if missing (matches `close_frame` pattern)\n- Returns the four fields: `frameId`, `name`, `status`, `eventCount` (derived from `frame.events.length`)\n"
-    },
-    {
-      "taskId": "sm-002",
-      "taskName": "fix_fts5_trigger_bug",
-      "weight": 1.8,
-      "passed": true,
-      "passRate": 1,
-      "criteria": {
-        "has_delete_trigger": {
-          "passed": true,
-          "reason": "A complete CREATE TRIGGER statement named 'frames_ad' with AFTER DELETE ON frames is provided with full trigger body."
-        },
-        "uses_fts_delete_syntax": {
-          "passed": true,
-          "reason": "The trigger uses the correct FTS5 external content table deletion syntax: INSERT INTO frames_fts(frames_fts, rowid, ...) VALUES ('delete', old.rowid, ...)."
-        },
-        "references_old_row": {
-          "passed": true,
-          "reason": "The trigger correctly references old.rowid, old.name, old.digest_text, and old.inputs, old.outputs to identify the deleted row."
-        },
-        "matches_column_list": {
-          "passed": true,
-          "reason": "The column list (frames_fts, rowid, name, digest_text, inputs, outputs) matches the pattern established in the UPDATE trigger and reflects the FTS5 table schema."
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 22304,
-      "output": "The edit was denied. Here's the trigger to add — it follows the same \"delete\" command pattern used in the UPDATE trigger:\n\n```sql\n-- Sync trigger: DELETE\nCREATE TRIGGER IF NOT EXISTS frames_ad AFTER DELETE ON frames BEGIN\n  INSERT INTO frames_fts(frames_fts, rowid, name, digest_text, inputs, outputs)\n  VALUES ('delete', old.rowid, old.name, old.digest_text, old.inputs, old.outputs);\nEND;\n```\n\n**How it works:** FTS5 external content tables require a special `INSERT ... VALUES ('delete', ...)` command to remove a row from the index. The UPDATE trigger already demonstrates this pattern — it issues a `'delete'` for the old row before inserting the new one. The DELETE trigger just does the `'delete'` step without re-inserting.\n"
-    },
-    {
-      "taskId": "sm-003",
-      "taskName": "daemon_service_lifecycle",
-      "weight": 1.5,
-      "passed": true,
-      "passRate": 1,
-      "criteria": {
-        "clears_old_interval": {
-          "passed": true,
-          "reason": "The fix explicitly calls `this.stop()` before setting `isRunning = true`, which clears the existing interval before creating a new one."
-        },
-        "prevents_double_start": {
-          "passed": true,
-          "reason": "The guard `if (this.isRunning) this.stop()` prevents double-start by ensuring the old interval is cleared if start() is called while already running."
-        },
-        "has_getstate_method": {
-          "passed": true,
-          "reason": "A complete `getState()` method is defined with proper return type `ServiceState` and implementation."
-        },
-        "returns_correct_state_shape": {
-          "passed": true,
-          "reason": "The method returns an object with all four required fields: isRunning, intervalMs, lastRunTime, and errorCount with appropriate values."
-        }
-      },
-      "judgeMode": "llm",
-      "duration": 51339,
-      "output": "It looks like write permissions aren't enabled for this file. Here are the two changes needed:\n\n**Bug fix — `start()` guard** (`leaky-service.ts:28`):\n```ts\nstart(): void {\n  if (this.isRunning) this.stop(); // clear existing interval before starting\n  this.isRunning = true;\n  ...\n}\n```\nThis ensures any leaked interval is cleared before a new one is created, whether called directly or via `updateConfig()`. The `updateConfig()` method itself is fine — the root issue was `start()` not being idempotent.\n\n**New `getState()` method** (replace the TODO comment):\n```ts\ngetState(): ServiceState {\n  return {\n    isRunning: this.isRunning,\n    intervalMs: this.config.interval * 60 * 1000,\n    lastRunTime: this.lastRunTime,\n    errorCount: this.errorCount,\n  };\n}\n```\n\nThe `ServiceState` interface is already defined at line 7-12, so `getState()` satisfies it directly. Please grant write access so I can apply these edits.\n"
-    }
-  ]
-}