@stackmemoryai/stackmemory 1.5.8 → 1.5.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cli/commands/orchestrate.js +251 -110
- package/dist/src/cli/commands/orchestrator.js +82 -11
- package/package.json +1 -1
- package/scripts/gepa/.before-optimize.md +0 -32
- package/scripts/gepa/generations/gen-000/baseline.md +0 -32
- package/scripts/gepa/generations/gen-001/baseline.md +0 -32
- package/scripts/gepa/generations/gen-001/variant-a.md +107 -1
- package/scripts/gepa/generations/gen-001/variant-b.md +216 -1
- package/scripts/gepa/generations/gen-001/variant-c.md +83 -1
- package/scripts/gepa/generations/gen-001/variant-d.md +90 -1
- package/scripts/gepa/results/eval-1-baseline.json +78 -78
- package/scripts/gepa/results/eval-1-variant-a.json +74 -74
- package/scripts/gepa/results/eval-1-variant-b.json +78 -78
- package/scripts/gepa/results/eval-1-variant-c.json +72 -72
- package/scripts/gepa/results/eval-1-variant-d.json +80 -80
- package/scripts/gepa/state.json +17 -5
|
@@ -7,72 +7,72 @@
|
|
|
7
7
|
"taskName": "simple_function",
|
|
8
8
|
"weight": 1,
|
|
9
9
|
"passed": false,
|
|
10
|
-
"passRate": 0,
|
|
10
|
+
"passRate": 0.3333333333333333,
|
|
11
11
|
"criteria": {
|
|
12
12
|
"has_function": {
|
|
13
|
-
"passed":
|
|
14
|
-
"reason": "
|
|
13
|
+
"passed": true,
|
|
14
|
+
"reason": "A properly defined function `is_palindrome` with type hints and a working implementation is present."
|
|
15
15
|
},
|
|
16
16
|
"handles_edge_cases": {
|
|
17
17
|
"passed": false,
|
|
18
|
-
"reason": "
|
|
18
|
+
"reason": "The function only handles spaces and case sensitivity, but does not handle punctuation, special characters, or numbers. A true palindrome checker should ignore these. For example, 'A man, a plan, a canal: Panama' would fail."
|
|
19
19
|
},
|
|
20
20
|
"is_tested": {
|
|
21
21
|
"passed": false,
|
|
22
|
-
"reason": "
|
|
22
|
+
"reason": "No test cases or assertions are provided to demonstrate that the function works correctly."
|
|
23
23
|
}
|
|
24
24
|
},
|
|
25
|
-
"judgeMode": "
|
|
26
|
-
"duration":
|
|
27
|
-
"output": "
|
|
25
|
+
"judgeMode": "llm",
|
|
26
|
+
"duration": 9807,
|
|
27
|
+
"output": "```python\ndef is_palindrome(s: str) -> bool:\n s = s.lower().replace(\" \", \"\")\n return s == s[::-1]\n```\n"
|
|
28
28
|
},
|
|
29
29
|
{
|
|
30
30
|
"taskId": "eval-002",
|
|
31
31
|
"taskName": "refactor_code",
|
|
32
32
|
"weight": 1.5,
|
|
33
33
|
"passed": false,
|
|
34
|
-
"passRate": 0
|
|
34
|
+
"passRate": 0,
|
|
35
35
|
"criteria": {
|
|
36
36
|
"uses_async": {
|
|
37
37
|
"passed": false,
|
|
38
|
-
"reason": "
|
|
38
|
+
"reason": "No async/await code was provided in the output. The AI only requested the code to be refactored without delivering any refactored code."
|
|
39
39
|
},
|
|
40
40
|
"preserves_behavior": {
|
|
41
41
|
"passed": false,
|
|
42
|
-
"reason": "
|
|
42
|
+
"reason": "No refactored code exists to evaluate whether behavior is preserved. The output is a request for input, not a solution."
|
|
43
43
|
},
|
|
44
44
|
"no_nested_callbacks": {
|
|
45
|
-
"passed":
|
|
46
|
-
"reason": "
|
|
45
|
+
"passed": false,
|
|
46
|
+
"reason": "No code transformation was performed. The output does not demonstrate elimination of nested callbacks in any actual code."
|
|
47
47
|
}
|
|
48
48
|
},
|
|
49
|
-
"judgeMode": "
|
|
50
|
-
"duration":
|
|
51
|
-
"output": "
|
|
49
|
+
"judgeMode": "llm",
|
|
50
|
+
"duration": 11792,
|
|
51
|
+
"output": "I don't see any specific code provided. Could you share the code you'd like refactored, or point me to the file path?\n"
|
|
52
52
|
},
|
|
53
53
|
{
|
|
54
54
|
"taskId": "eval-003",
|
|
55
55
|
"taskName": "fix_bug",
|
|
56
56
|
"weight": 1.2,
|
|
57
57
|
"passed": false,
|
|
58
|
-
"passRate": 0,
|
|
58
|
+
"passRate": 0.3333333333333333,
|
|
59
59
|
"criteria": {
|
|
60
60
|
"bug_fixed": {
|
|
61
61
|
"passed": false,
|
|
62
|
-
"reason": "
|
|
62
|
+
"reason": "No code was provided by the user, so no bug could be fixed. The AI correctly identified the missing input but did not fix any bug."
|
|
63
63
|
},
|
|
64
64
|
"no_new_bugs": {
|
|
65
|
-
"passed":
|
|
66
|
-
"reason": "
|
|
65
|
+
"passed": true,
|
|
66
|
+
"reason": "The AI did not write any code, so no new bugs were introduced."
|
|
67
67
|
},
|
|
68
68
|
"explains_fix": {
|
|
69
69
|
"passed": false,
|
|
70
|
-
"reason": "
|
|
70
|
+
"reason": "No explanation of a fix was provided because no code was available to analyze or fix."
|
|
71
71
|
}
|
|
72
72
|
},
|
|
73
|
-
"judgeMode": "
|
|
74
|
-
"duration":
|
|
75
|
-
"output": "
|
|
73
|
+
"judgeMode": "llm",
|
|
74
|
+
"duration": 12640,
|
|
75
|
+
"output": "I don't see any code in your message. Could you share the loop you'd like me to fix?\n"
|
|
76
76
|
},
|
|
77
77
|
{
|
|
78
78
|
"taskId": "eval-004",
|
|
@@ -83,24 +83,24 @@
|
|
|
83
83
|
"criteria": {
|
|
84
84
|
"has_pagination": {
|
|
85
85
|
"passed": false,
|
|
86
|
-
"reason": "
|
|
86
|
+
"reason": "No pagination implementation provided. AI only asks for clarification instead of delivering pagination code."
|
|
87
87
|
},
|
|
88
88
|
"has_limit": {
|
|
89
89
|
"passed": false,
|
|
90
|
-
"reason": "
|
|
90
|
+
"reason": "No limit parameter implemented in any code. The output contains no actual code implementation."
|
|
91
91
|
},
|
|
92
92
|
"has_offset": {
|
|
93
93
|
"passed": false,
|
|
94
|
-
"reason": "
|
|
94
|
+
"reason": "No offset parameter implemented in any code. The output contains no actual code implementation."
|
|
95
95
|
},
|
|
96
96
|
"backwards_compatible": {
|
|
97
97
|
"passed": false,
|
|
98
|
-
"reason": "
|
|
98
|
+
"reason": "No implementation provided to evaluate backwards compatibility. The output is a question rather than a solution."
|
|
99
99
|
}
|
|
100
100
|
},
|
|
101
|
-
"judgeMode": "
|
|
102
|
-
"duration":
|
|
103
|
-
"output": "API
|
|
101
|
+
"judgeMode": "llm",
|
|
102
|
+
"duration": 10855,
|
|
103
|
+
"output": "Which API endpoint would you like to add pagination to? Please share the file path or endpoint name.\n"
|
|
104
104
|
},
|
|
105
105
|
{
|
|
106
106
|
"taskId": "eval-005",
|
|
@@ -111,108 +111,108 @@
|
|
|
111
111
|
"criteria": {
|
|
112
112
|
"identifies_security_issue": {
|
|
113
113
|
"passed": false,
|
|
114
|
-
"reason": "
|
|
114
|
+
"reason": "No security issues are identified. The output is a request for clarification rather than an actual review."
|
|
115
115
|
},
|
|
116
116
|
"identifies_performance_issue": {
|
|
117
117
|
"passed": false,
|
|
118
|
-
"reason": "
|
|
118
|
+
"reason": "No performance issues are identified. The output does not analyze any code."
|
|
119
119
|
},
|
|
120
120
|
"actionable_feedback": {
|
|
121
121
|
"passed": false,
|
|
122
|
-
"reason": "
|
|
122
|
+
"reason": "No actionable feedback is provided. The output only requests missing information instead of providing a review."
|
|
123
123
|
}
|
|
124
124
|
},
|
|
125
|
-
"judgeMode": "
|
|
126
|
-
"duration":
|
|
127
|
-
"output": "
|
|
125
|
+
"judgeMode": "llm",
|
|
126
|
+
"duration": 26949,
|
|
127
|
+
"output": "I need a PR number or URL to review. Which PR would you like me to review?\n"
|
|
128
128
|
},
|
|
129
129
|
{
|
|
130
130
|
"taskId": "sm-001",
|
|
131
131
|
"taskName": "add_mcp_tool_handler",
|
|
132
132
|
"weight": 1.5,
|
|
133
|
-
"passed":
|
|
134
|
-
"passRate":
|
|
133
|
+
"passed": true,
|
|
134
|
+
"passRate": 1,
|
|
135
135
|
"criteria": {
|
|
136
136
|
"has_switch_case": {
|
|
137
|
-
"passed":
|
|
138
|
-
"reason": "
|
|
137
|
+
"passed": true,
|
|
138
|
+
"reason": "The output shows a proper switch case statement with 'get_frame_summary' as the case label, following the switch/case dispatch pattern requested."
|
|
139
139
|
},
|
|
140
140
|
"has_zod_schema": {
|
|
141
|
-
"passed":
|
|
142
|
-
"reason": "
|
|
141
|
+
"passed": true,
|
|
142
|
+
"reason": "Zod schema is defined with `z.object({ frameId: z.string() }).parse(args)` which validates the input object."
|
|
143
143
|
},
|
|
144
144
|
"validates_input": {
|
|
145
|
-
"passed":
|
|
146
|
-
"reason": "
|
|
145
|
+
"passed": true,
|
|
146
|
+
"reason": "Input validation is performed via Zod's `.parse()` method, which will throw if the input doesn't match the schema (frameId must be a string)."
|
|
147
147
|
},
|
|
148
148
|
"returns_typed_response": {
|
|
149
|
-
"passed":
|
|
150
|
-
"reason": "
|
|
149
|
+
"passed": true,
|
|
150
|
+
"reason": "The handler returns an object with the exact shape specified: `{ frameId, name, status, eventCount }` with appropriate values extracted from the frame object."
|
|
151
151
|
},
|
|
152
152
|
"handles_not_found": {
|
|
153
|
-
"passed":
|
|
154
|
-
"reason": "
|
|
153
|
+
"passed": true,
|
|
154
|
+
"reason": "The code includes explicit error handling with `if (!frame) throw new Error(...)` when the frame doesn't exist, consistent with the close_frame pattern mentioned."
|
|
155
155
|
}
|
|
156
156
|
},
|
|
157
|
-
"judgeMode": "
|
|
158
|
-
"duration":
|
|
159
|
-
"output": "
|
|
157
|
+
"judgeMode": "llm",
|
|
158
|
+
"duration": 97214,
|
|
159
|
+
"output": "Please approve the file write permission and I'll apply the change. The handler follows the exact same pattern as `close_frame`:\n\n```ts\ncase 'get_frame_summary': {\n const input = z.object({ frameId: z.string() }).parse(args);\n const frame = frames.get(input.frameId);\n if (!frame) throw new Error(`Frame not found: ${input.frameId}`);\n return { frameId: frame.id, name: frame.name, status: frame.status, eventCount: frame.events.length };\n}\n```\n\n- Zod validates `frameId` as a required string\n- Throws if the frame doesn't exist (consistent with `close_frame`)\n- Returns `{ frameId, name, status, eventCount }` where `eventCount` is derived from `frame.events.length`\n"
|
|
160
160
|
},
|
|
161
161
|
{
|
|
162
162
|
"taskId": "sm-002",
|
|
163
163
|
"taskName": "fix_fts5_trigger_bug",
|
|
164
164
|
"weight": 1.8,
|
|
165
|
-
"passed":
|
|
166
|
-
"passRate":
|
|
165
|
+
"passed": true,
|
|
166
|
+
"passRate": 1,
|
|
167
167
|
"criteria": {
|
|
168
168
|
"has_delete_trigger": {
|
|
169
|
-
"passed":
|
|
170
|
-
"reason": "
|
|
169
|
+
"passed": true,
|
|
170
|
+
"reason": "A DELETE trigger named 'frames_ad' is explicitly defined with CREATE TRIGGER IF NOT EXISTS syntax, targeting the frames table with AFTER DELETE event."
|
|
171
171
|
},
|
|
172
172
|
"uses_fts_delete_syntax": {
|
|
173
|
-
"passed":
|
|
174
|
-
"reason": "
|
|
173
|
+
"passed": true,
|
|
174
|
+
"reason": "The trigger uses the correct FTS5 delete pattern: INSERT INTO frames_fts with the special 'delete' command as the first value, which is the standard FTS5 mechanism for removing indexed rows."
|
|
175
175
|
},
|
|
176
176
|
"references_old_row": {
|
|
177
|
-
"passed":
|
|
178
|
-
"reason": "
|
|
177
|
+
"passed": true,
|
|
178
|
+
"reason": "The trigger references the OLD row context (old.rowid, old.name, old.digest_text, old.inputs, old.outputs) which is the correct way to access deleted row values in an AFTER DELETE trigger."
|
|
179
179
|
},
|
|
180
180
|
"matches_column_list": {
|
|
181
|
-
"passed":
|
|
182
|
-
"reason": "
|
|
181
|
+
"passed": true,
|
|
182
|
+
"reason": "The column list in the INSERT statement (frames_fts, rowid, name, digest_text, inputs, outputs) matches the pattern used in the existing INSERT and UPDATE triggers, maintaining consistency across all three triggers."
|
|
183
183
|
}
|
|
184
184
|
},
|
|
185
|
-
"judgeMode": "
|
|
186
|
-
"duration":
|
|
187
|
-
"output": "
|
|
185
|
+
"judgeMode": "llm",
|
|
186
|
+
"duration": 28917,
|
|
187
|
+
"output": "The edit needs your approval. The trigger to add follows the same FTS5 \"delete\" command pattern used in the UPDATE trigger — issuing a special `INSERT ... VALUES ('delete', ...)` to remove the old row from the index:\n\n```sql\n-- Sync trigger: DELETE\nCREATE TRIGGER IF NOT EXISTS frames_ad AFTER DELETE ON frames BEGIN\n INSERT INTO frames_fts(frames_fts, rowid, name, digest_text, inputs, outputs)\n VALUES ('delete', old.rowid, old.name, old.digest_text, old.inputs, old.outputs);\nEND;\n```\n\nPlease approve the file write (or let me know if you'd like to apply it differently).\n"
|
|
188
188
|
},
|
|
189
189
|
{
|
|
190
190
|
"taskId": "sm-003",
|
|
191
191
|
"taskName": "daemon_service_lifecycle",
|
|
192
192
|
"weight": 1.5,
|
|
193
|
-
"passed":
|
|
194
|
-
"passRate":
|
|
193
|
+
"passed": true,
|
|
194
|
+
"passRate": 1,
|
|
195
195
|
"criteria": {
|
|
196
196
|
"clears_old_interval": {
|
|
197
|
-
"passed":
|
|
198
|
-
"reason": "
|
|
197
|
+
"passed": true,
|
|
198
|
+
"reason": "The fix adds `if (this.isRunning) this.stop();` at the start of `start()`, which calls `clearInterval` on the old `intervalId` before creating a new one."
|
|
199
199
|
},
|
|
200
200
|
"prevents_double_start": {
|
|
201
|
-
"passed":
|
|
202
|
-
"reason": "
|
|
201
|
+
"passed": true,
|
|
202
|
+
"reason": "The guard `if (this.isRunning) this.stop();` prevents double-start by stopping any existing interval before starting a new one, ensuring only one interval runs at a time."
|
|
203
203
|
},
|
|
204
204
|
"has_getstate_method": {
|
|
205
|
-
"passed":
|
|
206
|
-
"reason": "
|
|
205
|
+
"passed": true,
|
|
206
|
+
"reason": "A `getState()` method is defined that returns an object with the required properties. The code is syntactically correct and callable."
|
|
207
207
|
},
|
|
208
208
|
"returns_correct_state_shape": {
|
|
209
|
-
"passed":
|
|
210
|
-
"reason": "
|
|
209
|
+
"passed": true,
|
|
210
|
+
"reason": "The `getState()` method returns an object with all four required properties: `isRunning`, `intervalMs`, `lastRunTime`, and `errorCount` with appropriate values derived from instance state."
|
|
211
211
|
}
|
|
212
212
|
},
|
|
213
|
-
"judgeMode": "
|
|
214
|
-
"duration":
|
|
215
|
-
"output": "
|
|
213
|
+
"judgeMode": "llm",
|
|
214
|
+
"duration": 44838,
|
|
215
|
+
"output": "It seems write permissions aren't granted. Here's the full fix to apply:\n\n**Bug fix** — add a guard at the top of `start()` so any existing interval is cleared before a new one is created:\n\n```diff\n- // BUG: No guard against double-start — calling start() twice\n- // creates two intervals but only stores the second one.\n- // The first interval leaks and keeps running forever.\n start(): void {\n+ if (this.isRunning) this.stop(); // clear any existing interval before starting\n this.isRunning = true;\n```\n\n**Remove the now-stale bug comment** from `updateConfig` (it's no longer accurate once `start()` self-clears).\n\n**Add `getState()`** after `updateConfig`:\n\n```ts\ngetState(): ServiceState {\n return {\n isRunning: this.isRunning,\n intervalMs: this.config.interval * 60 * 1000,\n lastRunTime: this.lastRunTime,\n errorCount: this.errorCount,\n };\n}\n```\n\n**Why this works:**\n- `start()` calling `stop()` first ensures `clearInterval` is always called on the old `intervalId` before overwriting it — no leak possible.\n- `updateConfig` already called `stop()` before `start()`, but the root bug was that `start()` itself had no guard. Now both paths are safe.\n- `getState()` reads the private fields directly; `intervalMs` is derived from config the same way `start()` computes it, keeping them in sync.\n"
|
|
216
216
|
}
|
|
217
217
|
]
|
|
218
218
|
}
|
|
@@ -7,48 +7,48 @@
|
|
|
7
7
|
"taskName": "simple_function",
|
|
8
8
|
"weight": 1,
|
|
9
9
|
"passed": false,
|
|
10
|
-
"passRate": 0,
|
|
10
|
+
"passRate": 0.3333333333333333,
|
|
11
11
|
"criteria": {
|
|
12
12
|
"has_function": {
|
|
13
|
-
"passed":
|
|
14
|
-
"reason": "
|
|
13
|
+
"passed": true,
|
|
14
|
+
"reason": "A complete, working function named is_palindrome is defined with proper syntax and type hints"
|
|
15
15
|
},
|
|
16
16
|
"handles_edge_cases": {
|
|
17
17
|
"passed": false,
|
|
18
|
-
"reason": "
|
|
18
|
+
"reason": "The function only handles spaces and case sensitivity, but does not handle punctuation, special characters, or empty strings in a robust way"
|
|
19
19
|
},
|
|
20
20
|
"is_tested": {
|
|
21
21
|
"passed": false,
|
|
22
|
-
"reason": "
|
|
22
|
+
"reason": "No test cases or assertions are provided to verify the function works correctly"
|
|
23
23
|
}
|
|
24
24
|
},
|
|
25
|
-
"judgeMode": "
|
|
26
|
-
"duration":
|
|
27
|
-
"output": "
|
|
25
|
+
"judgeMode": "llm",
|
|
26
|
+
"duration": 12084,
|
|
27
|
+
"output": "```python\ndef is_palindrome(s: str) -> bool:\n s = s.lower().replace(\" \", \"\")\n return s == s[::-1]\n```\n"
|
|
28
28
|
},
|
|
29
29
|
{
|
|
30
30
|
"taskId": "eval-002",
|
|
31
31
|
"taskName": "refactor_code",
|
|
32
32
|
"weight": 1.5,
|
|
33
33
|
"passed": false,
|
|
34
|
-
"passRate": 0
|
|
34
|
+
"passRate": 0,
|
|
35
35
|
"criteria": {
|
|
36
36
|
"uses_async": {
|
|
37
37
|
"passed": false,
|
|
38
|
-
"reason": "
|
|
38
|
+
"reason": "No async/await code is present in the output. The AI only requested the code to be refactored without providing any refactored implementation."
|
|
39
39
|
},
|
|
40
40
|
"preserves_behavior": {
|
|
41
41
|
"passed": false,
|
|
42
|
-
"reason": "
|
|
42
|
+
"reason": "No refactored code is provided, so behavior cannot be verified or preserved. The output is merely a request for input."
|
|
43
43
|
},
|
|
44
44
|
"no_nested_callbacks": {
|
|
45
|
-
"passed":
|
|
46
|
-
"reason": "
|
|
45
|
+
"passed": false,
|
|
46
|
+
"reason": "No code transformation was attempted. The output does not demonstrate removal of nested callbacks since no refactored code was delivered."
|
|
47
47
|
}
|
|
48
48
|
},
|
|
49
|
-
"judgeMode": "
|
|
50
|
-
"duration":
|
|
51
|
-
"output": "
|
|
49
|
+
"judgeMode": "llm",
|
|
50
|
+
"duration": 19736,
|
|
51
|
+
"output": "Please share the code you'd like me to refactor.\n"
|
|
52
52
|
},
|
|
53
53
|
{
|
|
54
54
|
"taskId": "eval-003",
|
|
@@ -59,20 +59,20 @@
|
|
|
59
59
|
"criteria": {
|
|
60
60
|
"bug_fixed": {
|
|
61
61
|
"passed": false,
|
|
62
|
-
"reason": "
|
|
62
|
+
"reason": "No code was provided to fix, so no bug was actually corrected"
|
|
63
63
|
},
|
|
64
64
|
"no_new_bugs": {
|
|
65
65
|
"passed": false,
|
|
66
|
-
"reason": "
|
|
66
|
+
"reason": "Cannot evaluate without code being analyzed or modified"
|
|
67
67
|
},
|
|
68
68
|
"explains_fix": {
|
|
69
69
|
"passed": false,
|
|
70
|
-
"reason": "
|
|
70
|
+
"reason": "No fix was explained because the AI correctly identified that no code was provided to work with"
|
|
71
71
|
}
|
|
72
72
|
},
|
|
73
|
-
"judgeMode": "
|
|
74
|
-
"duration":
|
|
75
|
-
"output": "
|
|
73
|
+
"judgeMode": "llm",
|
|
74
|
+
"duration": 16159,
|
|
75
|
+
"output": "I don't see any code shared yet. Could you paste the loop you'd like me to fix?\n"
|
|
76
76
|
},
|
|
77
77
|
{
|
|
78
78
|
"taskId": "eval-004",
|
|
@@ -83,80 +83,80 @@
|
|
|
83
83
|
"criteria": {
|
|
84
84
|
"has_pagination": {
|
|
85
85
|
"passed": false,
|
|
86
|
-
"reason": "
|
|
86
|
+
"reason": "No pagination implementation provided. AI only asks for clarification instead of delivering working code."
|
|
87
87
|
},
|
|
88
88
|
"has_limit": {
|
|
89
89
|
"passed": false,
|
|
90
|
-
"reason": "
|
|
90
|
+
"reason": "No limit parameter implemented in any code. Only a question asking for more information."
|
|
91
91
|
},
|
|
92
92
|
"has_offset": {
|
|
93
93
|
"passed": false,
|
|
94
|
-
"reason": "
|
|
94
|
+
"reason": "No offset parameter implemented in any code. Only a question asking for more information."
|
|
95
95
|
},
|
|
96
96
|
"backwards_compatible": {
|
|
97
97
|
"passed": false,
|
|
98
|
-
"reason": "
|
|
98
|
+
"reason": "No code provided to evaluate backwards compatibility. The response is a request for clarification, not a solution."
|
|
99
99
|
}
|
|
100
100
|
},
|
|
101
|
-
"judgeMode": "
|
|
102
|
-
"duration":
|
|
103
|
-
"output": "API
|
|
101
|
+
"judgeMode": "llm",
|
|
102
|
+
"duration": 27626,
|
|
103
|
+
"output": "Which API endpoint do you want to add pagination to? Please share the file path or endpoint name so I can look at the current implementation.\n"
|
|
104
104
|
},
|
|
105
105
|
{
|
|
106
106
|
"taskId": "eval-005",
|
|
107
107
|
"taskName": "code_review",
|
|
108
108
|
"weight": 1.3,
|
|
109
|
-
"passed":
|
|
110
|
-
"passRate": 0,
|
|
109
|
+
"passed": true,
|
|
110
|
+
"passRate": 0.6666666666666666,
|
|
111
111
|
"criteria": {
|
|
112
112
|
"identifies_security_issue": {
|
|
113
|
-
"passed":
|
|
114
|
-
"reason": "
|
|
113
|
+
"passed": true,
|
|
114
|
+
"reason": "The output identifies five distinct security issues: SQL injection, plaintext password comparison, weak session tokens, insecure cookie flags, and plaintext password logging. These are all legitimate security vulnerabilities."
|
|
115
115
|
},
|
|
116
116
|
"identifies_performance_issue": {
|
|
117
117
|
"passed": false,
|
|
118
|
-
"reason": "
|
|
118
|
+
"reason": "The output does not identify any performance issues. It mentions that the PR comment claims 'direct string interpolation for faster queries' but explicitly refutes this claim as incorrect. It does not identify or discuss any actual performance problems in the code."
|
|
119
119
|
},
|
|
120
120
|
"actionable_feedback": {
|
|
121
|
-
"passed":
|
|
122
|
-
"reason": "
|
|
121
|
+
"passed": true,
|
|
122
|
+
"reason": "The output provides specific, actionable recommendations for each issue: revert SQL to parameterized queries, use bcrypt.compare instead of direct comparison, use crypto.randomBytes for tokens, add specific cookie flags (httpOnly, secure, sameSite), and remove password logging. Each recommendation includes concrete code examples or clear guidance on what to change."
|
|
123
123
|
}
|
|
124
124
|
},
|
|
125
|
-
"judgeMode": "
|
|
126
|
-
"duration":
|
|
127
|
-
"output": "
|
|
125
|
+
"judgeMode": "llm",
|
|
126
|
+
"duration": 37583,
|
|
127
|
+
"output": "This PR has **critical security issues** — do not merge.\n\n## Issues Found\n\n### 1. SQL Injection (Critical)\n```diff\n- const user = await db.query('SELECT * FROM users WHERE email = $1', [email]);\n+ const user = await db.query(`SELECT * FROM users WHERE email = '${email}'`);\n```\nString interpolation into a SQL query allows any user to inject arbitrary SQL. The original parameterized query was correct — revert this immediately.\n\n### 2. Plaintext Password Comparison (Critical)\n```diff\n- const valid = await bcrypt.compare(password, user.passwordHash);\n+ const valid = password === user.passwordHash;\n```\nThis compares the raw password string directly against what should be a bcrypt hash. This means either (a) authentication is completely broken, or (b) passwords are stored in plaintext. Either way, revert to `bcrypt.compare`.\n\n### 3. Weak/Predictable Session Token (High)\n```js\nconst token = email + ':' + Date.now();\n```\nTokens built from `email + timestamp` are trivially guessable and forgeable. Use a cryptographically secure random value (e.g., `crypto.randomBytes(32).toString('hex')`).\n\n### 4. Session Cookie Missing Security Flags (High)\n```js\nres.cookie('session', token);\n```\nNo `httpOnly`, `secure`, or `sameSite` flags. This exposes the cookie to XSS and CSRF attacks. Should be at minimum:\n```js\nres.cookie('session', token, { httpOnly: true, secure: true, sameSite: 'strict' });\n```\n\n### 5. Password Logged in Plaintext (High)\n```js\nconsole.log(`Login: ${email} / ${password}`);\n```\nPasswords are being written to logs. Remove immediately — this violates basic security hygiene and likely compliance requirements (GDPR, SOC2, etc.).\n\n---\n\n**Summary**: This PR degrades security at every step — SQL injection, broken auth, weak tokens, insecure cookies, and leaked credentials. The comment \"Quick fix: direct string interpolation for faster queries\" is incorrect; parameterized queries have negligible overhead. **Reject and revert all changes.**\n"
|
|
128
128
|
},
|
|
129
129
|
{
|
|
130
130
|
"taskId": "sm-001",
|
|
131
131
|
"taskName": "add_mcp_tool_handler",
|
|
132
132
|
"weight": 1.5,
|
|
133
|
-
"passed":
|
|
134
|
-
"passRate":
|
|
133
|
+
"passed": true,
|
|
134
|
+
"passRate": 1,
|
|
135
135
|
"criteria": {
|
|
136
136
|
"has_switch_case": {
|
|
137
|
-
"passed":
|
|
138
|
-
"reason": "
|
|
137
|
+
"passed": true,
|
|
138
|
+
"reason": "A switch case for 'get_frame_summary' is clearly defined with proper case syntax"
|
|
139
139
|
},
|
|
140
140
|
"has_zod_schema": {
|
|
141
|
-
"passed":
|
|
142
|
-
"reason": "
|
|
141
|
+
"passed": true,
|
|
142
|
+
"reason": "Zod schema is present: z.object({ frameId: z.string() }) validates the required string input"
|
|
143
143
|
},
|
|
144
144
|
"validates_input": {
|
|
145
|
-
"passed":
|
|
146
|
-
"reason": "
|
|
145
|
+
"passed": true,
|
|
146
|
+
"reason": "Input validation occurs via z.object().parse(args), which will throw if validation fails"
|
|
147
147
|
},
|
|
148
148
|
"returns_typed_response": {
|
|
149
|
-
"passed":
|
|
150
|
-
"reason": "
|
|
149
|
+
"passed": true,
|
|
150
|
+
"reason": "Returns an object with all four required fields: frameId, name, status, eventCount in the correct shape"
|
|
151
151
|
},
|
|
152
152
|
"handles_not_found": {
|
|
153
|
-
"passed":
|
|
154
|
-
"reason": "
|
|
153
|
+
"passed": true,
|
|
154
|
+
"reason": "Includes explicit check for missing frame with frames.get() followed by error throw with descriptive message"
|
|
155
155
|
}
|
|
156
156
|
},
|
|
157
|
-
"judgeMode": "
|
|
158
|
-
"duration":
|
|
159
|
-
"output": "
|
|
157
|
+
"judgeMode": "llm",
|
|
158
|
+
"duration": 30073,
|
|
159
|
+
"output": "Please approve the file write permission. The edit adds this case to the switch:\n\n```ts\ncase 'get_frame_summary': {\n const input = z.object({ frameId: z.string() }).parse(args);\n const frame = frames.get(input.frameId);\n if (!frame) throw new Error(`Frame not found: ${input.frameId}`);\n return { frameId: frame.id, name: frame.name, status: frame.status, eventCount: frame.events.length };\n}\n```\n\nIt follows the same pattern as `close_frame`: Zod parse → lookup → error if missing → return the requested shape.\n"
|
|
160
160
|
},
|
|
161
161
|
{
|
|
162
162
|
"taskId": "sm-002",
|
|
@@ -167,52 +167,52 @@
|
|
|
167
167
|
"criteria": {
|
|
168
168
|
"has_delete_trigger": {
|
|
169
169
|
"passed": false,
|
|
170
|
-
"reason": "
|
|
170
|
+
"reason": "No DELETE trigger is actually created in the output; only a request for approval is made"
|
|
171
171
|
},
|
|
172
172
|
"uses_fts_delete_syntax": {
|
|
173
173
|
"passed": false,
|
|
174
|
-
"reason": "
|
|
174
|
+
"reason": "No FTS5 DELETE syntax (INSERT INTO frames_fts) is present in the output"
|
|
175
175
|
},
|
|
176
176
|
"references_old_row": {
|
|
177
177
|
"passed": false,
|
|
178
|
-
"reason": "
|
|
178
|
+
"reason": "No OLD row references are used; the output contains no trigger code"
|
|
179
179
|
},
|
|
180
180
|
"matches_column_list": {
|
|
181
181
|
"passed": false,
|
|
182
|
-
"reason": "
|
|
182
|
+
"reason": "No column list comparison exists; the output is just a request without implementation"
|
|
183
183
|
}
|
|
184
184
|
},
|
|
185
|
-
"judgeMode": "
|
|
186
|
-
"duration":
|
|
187
|
-
"output": "
|
|
185
|
+
"judgeMode": "llm",
|
|
186
|
+
"duration": 24722,
|
|
187
|
+
"output": "Please approve the write to `fts5-triggers.sql` so I can add the DELETE trigger.\n"
|
|
188
188
|
},
|
|
189
189
|
{
|
|
190
190
|
"taskId": "sm-003",
|
|
191
191
|
"taskName": "daemon_service_lifecycle",
|
|
192
192
|
"weight": 1.5,
|
|
193
|
-
"passed":
|
|
194
|
-
"passRate":
|
|
193
|
+
"passed": true,
|
|
194
|
+
"passRate": 1,
|
|
195
195
|
"criteria": {
|
|
196
196
|
"clears_old_interval": {
|
|
197
|
-
"passed":
|
|
198
|
-
"reason": "
|
|
197
|
+
"passed": true,
|
|
198
|
+
"reason": "The fix adds `if (this.isRunning) this.stop()` at the start of `start()`, which calls `clearInterval()` before creating a new interval. This directly addresses the timer leak."
|
|
199
199
|
},
|
|
200
200
|
"prevents_double_start": {
|
|
201
|
-
"passed":
|
|
202
|
-
"reason": "
|
|
201
|
+
"passed": true,
|
|
202
|
+
"reason": "The guard `if (this.isRunning) this.stop()` prevents double-start by stopping any existing interval before starting a new one, whether called directly or via `updateConfig()`."
|
|
203
203
|
},
|
|
204
204
|
"has_getstate_method": {
|
|
205
|
-
"passed":
|
|
206
|
-
"reason": "
|
|
205
|
+
"passed": true,
|
|
206
|
+
"reason": "A `getState()` method is defined in the output that returns the required ServiceState object with all four properties."
|
|
207
207
|
},
|
|
208
208
|
"returns_correct_state_shape": {
|
|
209
|
-
"passed":
|
|
210
|
-
"reason": "
|
|
209
|
+
"passed": true,
|
|
210
|
+
"reason": "The `getState()` method returns an object with the exact shape requested: `isRunning`, `intervalMs`, `lastRunTime`, and `errorCount`. The intervalMs calculation (`this.config.interval * 60 * 1000`) correctly converts minutes to milliseconds."
|
|
211
211
|
}
|
|
212
212
|
},
|
|
213
|
-
"judgeMode": "
|
|
214
|
-
"duration":
|
|
215
|
-
"output": "
|
|
213
|
+
"judgeMode": "llm",
|
|
214
|
+
"duration": 69511,
|
|
215
|
+
"output": "It looks like write permissions aren't granted for this directory. Here are the two changes needed:\n\n**Fix 1 — Guard `start()` against double-start** (`leaky-service.ts:28`):\n```ts\nstart(): void {\n if (this.isRunning) this.stop(); // clear any existing interval before starting\n this.isRunning = true;\n // ... rest unchanged\n}\n```\n\nThis ensures `stop()` (which calls `clearInterval`) always runs before a new interval is created, preventing leaks from both direct double-calls and `updateConfig()` re-entry.\n\n**Fix 2 — Remove stale comment from `updateConfig()`** (the bug comment is now inaccurate since `start()` handles it).\n\n**Fix 3 — Add `getState()`** (replace the TODO comment):\n```ts\ngetState(): ServiceState {\n return {\n isRunning: this.isRunning,\n intervalMs: this.config.interval * 60 * 1000,\n lastRunTime: this.lastRunTime,\n errorCount: this.errorCount,\n };\n}\n```\n\nThe root cause was that `start()` unconditionally set `this.isRunning = true` and created a new interval without clearing the previous one. The fix moves responsibility to `start()` itself — if already running, stop first — rather than relying on every caller to avoid double-starts.\n"
|
|
216
216
|
}
|
|
217
217
|
]
|
|
218
218
|
}
|