@stackmemoryai/stackmemory 1.5.8 → 1.5.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cli/commands/orchestrate.js +251 -110
- package/dist/src/cli/commands/orchestrator.js +82 -11
- package/package.json +1 -1
- package/scripts/gepa/.before-optimize.md +0 -32
- package/scripts/gepa/generations/gen-000/baseline.md +0 -32
- package/scripts/gepa/generations/gen-001/baseline.md +0 -32
- package/scripts/gepa/generations/gen-001/variant-a.md +107 -1
- package/scripts/gepa/generations/gen-001/variant-b.md +216 -1
- package/scripts/gepa/generations/gen-001/variant-c.md +83 -1
- package/scripts/gepa/generations/gen-001/variant-d.md +90 -1
- package/scripts/gepa/results/eval-1-baseline.json +78 -78
- package/scripts/gepa/results/eval-1-variant-a.json +74 -74
- package/scripts/gepa/results/eval-1-variant-b.json +78 -78
- package/scripts/gepa/results/eval-1-variant-c.json +72 -72
- package/scripts/gepa/results/eval-1-variant-d.json +80 -80
- package/scripts/gepa/state.json +17 -5
|
@@ -7,72 +7,72 @@
|
|
|
7
7
|
"taskName": "simple_function",
|
|
8
8
|
"weight": 1,
|
|
9
9
|
"passed": false,
|
|
10
|
-
"passRate": 0,
|
|
10
|
+
"passRate": 0.3333333333333333,
|
|
11
11
|
"criteria": {
|
|
12
12
|
"has_function": {
|
|
13
|
-
"passed":
|
|
14
|
-
"reason": "
|
|
13
|
+
"passed": true,
|
|
14
|
+
"reason": "A real, working function named is_palindrome is defined with proper syntax, type hints, and implementation"
|
|
15
15
|
},
|
|
16
16
|
"handles_edge_cases": {
|
|
17
17
|
"passed": false,
|
|
18
|
-
"reason": "
|
|
18
|
+
"reason": "The function only handles lowercase conversion and space removal, but doesn't handle punctuation, special characters, or empty strings. A complete edge case handler would address these scenarios"
|
|
19
19
|
},
|
|
20
20
|
"is_tested": {
|
|
21
21
|
"passed": false,
|
|
22
|
-
"reason": "
|
|
22
|
+
"reason": "No test cases or assertions are provided in the output to verify the function works correctly"
|
|
23
23
|
}
|
|
24
24
|
},
|
|
25
|
-
"judgeMode": "
|
|
26
|
-
"duration":
|
|
27
|
-
"output": "
|
|
25
|
+
"judgeMode": "llm",
|
|
26
|
+
"duration": 58252,
|
|
27
|
+
"output": "```python\ndef is_palindrome(s: str) -> bool:\n s = s.lower().replace(\" \", \"\")\n return s == s[::-1]\n```\n"
|
|
28
28
|
},
|
|
29
29
|
{
|
|
30
30
|
"taskId": "eval-002",
|
|
31
31
|
"taskName": "refactor_code",
|
|
32
32
|
"weight": 1.5,
|
|
33
33
|
"passed": false,
|
|
34
|
-
"passRate": 0
|
|
34
|
+
"passRate": 0,
|
|
35
35
|
"criteria": {
|
|
36
36
|
"uses_async": {
|
|
37
37
|
"passed": false,
|
|
38
|
-
"reason": "
|
|
38
|
+
"reason": "No async/await code is present in the output. The AI only requests the source code without providing any refactored implementation."
|
|
39
39
|
},
|
|
40
40
|
"preserves_behavior": {
|
|
41
41
|
"passed": false,
|
|
42
|
-
"reason": "
|
|
42
|
+
"reason": "No refactored code is provided, so behavior cannot be preserved or verified."
|
|
43
43
|
},
|
|
44
44
|
"no_nested_callbacks": {
|
|
45
|
-
"passed":
|
|
46
|
-
"reason": "
|
|
45
|
+
"passed": false,
|
|
46
|
+
"reason": "No code transformation is shown. The AI did not perform the refactoring task and therefore did not eliminate nested callbacks."
|
|
47
47
|
}
|
|
48
48
|
},
|
|
49
|
-
"judgeMode": "
|
|
50
|
-
"duration":
|
|
51
|
-
"output": "
|
|
49
|
+
"judgeMode": "llm",
|
|
50
|
+
"duration": 20841,
|
|
51
|
+
"output": "Please share the code you'd like me to refactor.\n"
|
|
52
52
|
},
|
|
53
53
|
{
|
|
54
54
|
"taskId": "eval-003",
|
|
55
55
|
"taskName": "fix_bug",
|
|
56
56
|
"weight": 1.2,
|
|
57
57
|
"passed": false,
|
|
58
|
-
"passRate": 0,
|
|
58
|
+
"passRate": 0.3333333333333333,
|
|
59
59
|
"criteria": {
|
|
60
60
|
"bug_fixed": {
|
|
61
61
|
"passed": false,
|
|
62
|
-
"reason": "
|
|
62
|
+
"reason": "No code was provided to the AI, so no bug could be fixed. The AI correctly identified this but did not fix anything."
|
|
63
63
|
},
|
|
64
64
|
"no_new_bugs": {
|
|
65
|
-
"passed":
|
|
66
|
-
"reason": "
|
|
65
|
+
"passed": true,
|
|
66
|
+
"reason": "The AI did not write any code, so no new bugs were introduced."
|
|
67
67
|
},
|
|
68
68
|
"explains_fix": {
|
|
69
69
|
"passed": false,
|
|
70
|
-
"reason": "
|
|
70
|
+
"reason": "No explanation of a fix was provided because the AI was waiting for code input. A generic explanation of off-by-one errors was not given."
|
|
71
71
|
}
|
|
72
72
|
},
|
|
73
|
-
"judgeMode": "
|
|
74
|
-
"duration":
|
|
75
|
-
"output": "
|
|
73
|
+
"judgeMode": "llm",
|
|
74
|
+
"duration": 35783,
|
|
75
|
+
"output": "I don't see any code or file provided to fix. Could you share the code with the off-by-one error?\n"
|
|
76
76
|
},
|
|
77
77
|
{
|
|
78
78
|
"taskId": "eval-004",
|
|
@@ -83,24 +83,24 @@
|
|
|
83
83
|
"criteria": {
|
|
84
84
|
"has_pagination": {
|
|
85
85
|
"passed": false,
|
|
86
|
-
"reason": "
|
|
86
|
+
"reason": "No pagination code is implemented; AI only asked a clarifying question"
|
|
87
87
|
},
|
|
88
88
|
"has_limit": {
|
|
89
89
|
"passed": false,
|
|
90
|
-
"reason": "
|
|
90
|
+
"reason": "No limit parameter is defined or implemented"
|
|
91
91
|
},
|
|
92
92
|
"has_offset": {
|
|
93
93
|
"passed": false,
|
|
94
|
-
"reason": "
|
|
94
|
+
"reason": "No offset parameter is defined or implemented"
|
|
95
95
|
},
|
|
96
96
|
"backwards_compatible": {
|
|
97
97
|
"passed": false,
|
|
98
|
-
"reason": "
|
|
98
|
+
"reason": "No implementation exists to maintain backwards compatibility"
|
|
99
99
|
}
|
|
100
100
|
},
|
|
101
|
-
"judgeMode": "
|
|
102
|
-
"duration":
|
|
103
|
-
"output": "API
|
|
101
|
+
"judgeMode": "llm",
|
|
102
|
+
"duration": 73593,
|
|
103
|
+
"output": "Which API endpoint should I add pagination to? Please share the file path or endpoint name.\n"
|
|
104
104
|
},
|
|
105
105
|
{
|
|
106
106
|
"taskId": "eval-005",
|
|
@@ -111,108 +111,108 @@
|
|
|
111
111
|
"criteria": {
|
|
112
112
|
"identifies_security_issue": {
|
|
113
113
|
"passed": false,
|
|
114
|
-
"reason": "
|
|
114
|
+
"reason": "The AI output does not identify any security issues. It only requests clarification on which PR to review."
|
|
115
115
|
},
|
|
116
116
|
"identifies_performance_issue": {
|
|
117
117
|
"passed": false,
|
|
118
|
-
"reason": "
|
|
118
|
+
"reason": "The AI output does not identify any performance issues. It only requests clarification on which PR to review."
|
|
119
119
|
},
|
|
120
120
|
"actionable_feedback": {
|
|
121
121
|
"passed": false,
|
|
122
|
-
"reason": "
|
|
122
|
+
"reason": "The AI output provides no actionable feedback about code. It only asks for a PR number or URL without performing any actual review."
|
|
123
123
|
}
|
|
124
124
|
},
|
|
125
|
-
"judgeMode": "
|
|
126
|
-
"duration":
|
|
127
|
-
"output": "
|
|
125
|
+
"judgeMode": "llm",
|
|
126
|
+
"duration": 20273,
|
|
127
|
+
"output": "I need a PR number or URL to review. Could you provide the PR you'd like me to review?\n"
|
|
128
128
|
},
|
|
129
129
|
{
|
|
130
130
|
"taskId": "sm-001",
|
|
131
131
|
"taskName": "add_mcp_tool_handler",
|
|
132
132
|
"weight": 1.5,
|
|
133
|
-
"passed":
|
|
134
|
-
"passRate":
|
|
133
|
+
"passed": true,
|
|
134
|
+
"passRate": 1,
|
|
135
135
|
"criteria": {
|
|
136
136
|
"has_switch_case": {
|
|
137
|
-
"passed":
|
|
138
|
-
"reason": "
|
|
137
|
+
"passed": true,
|
|
138
|
+
"reason": "Code includes a `case 'get_frame_summary':` statement, which is a real switch/case dispatch pattern matching the requirement"
|
|
139
139
|
},
|
|
140
140
|
"has_zod_schema": {
|
|
141
|
-
"passed":
|
|
142
|
-
"reason": "
|
|
141
|
+
"passed": true,
|
|
142
|
+
"reason": "Code includes `z.object({ frameId: z.string() }).parse(args)`, which is a real Zod schema definition and validation"
|
|
143
143
|
},
|
|
144
144
|
"validates_input": {
|
|
145
|
-
"passed":
|
|
146
|
-
"reason": "
|
|
145
|
+
"passed": true,
|
|
146
|
+
"reason": "Zod schema validation is executed with `.parse(args)`, which will throw if validation fails, properly validating the required string frameId"
|
|
147
147
|
},
|
|
148
148
|
"returns_typed_response": {
|
|
149
|
-
"passed":
|
|
150
|
-
"reason": "
|
|
149
|
+
"passed": true,
|
|
150
|
+
"reason": "Returns an object with all four required fields: frameId, name, status, and eventCount with appropriate types derived from frame object properties"
|
|
151
151
|
},
|
|
152
152
|
"handles_not_found": {
|
|
153
|
-
"passed":
|
|
154
|
-
"reason": "
|
|
153
|
+
"passed": true,
|
|
154
|
+
"reason": "Code includes null check `if (!frame)` with explicit error throw, matching the error handling pattern shown in the example"
|
|
155
155
|
}
|
|
156
156
|
},
|
|
157
|
-
"judgeMode": "
|
|
158
|
-
"duration":
|
|
159
|
-
"output": "
|
|
157
|
+
"judgeMode": "llm",
|
|
158
|
+
"duration": 28089,
|
|
159
|
+
"output": "The edit was denied. Here's the handler to add in place of the `// TODO` comment:\n\n```ts\ncase 'get_frame_summary': {\n const input = z.object({ frameId: z.string() }).parse(args);\n const frame = frames.get(input.frameId);\n if (!frame) throw new Error(`Frame not found: ${input.frameId}`);\n return {\n frameId: frame.id,\n name: frame.name,\n status: frame.status,\n eventCount: frame.events.length,\n };\n}\n```\n\n**What it does:**\n- Zod validates `frameId` as a required string (line 18 pattern)\n- Looks up the frame, throws if missing (matches `close_frame` pattern)\n- Returns the four fields: `frameId`, `name`, `status`, `eventCount` (derived from `frame.events.length`)\n"
|
|
160
160
|
},
|
|
161
161
|
{
|
|
162
162
|
"taskId": "sm-002",
|
|
163
163
|
"taskName": "fix_fts5_trigger_bug",
|
|
164
164
|
"weight": 1.8,
|
|
165
|
-
"passed":
|
|
166
|
-
"passRate":
|
|
165
|
+
"passed": true,
|
|
166
|
+
"passRate": 1,
|
|
167
167
|
"criteria": {
|
|
168
168
|
"has_delete_trigger": {
|
|
169
|
-
"passed":
|
|
170
|
-
"reason": "
|
|
169
|
+
"passed": true,
|
|
170
|
+
"reason": "A complete CREATE TRIGGER statement named 'frames_ad' with AFTER DELETE ON frames is provided with full trigger body."
|
|
171
171
|
},
|
|
172
172
|
"uses_fts_delete_syntax": {
|
|
173
|
-
"passed":
|
|
174
|
-
"reason": "
|
|
173
|
+
"passed": true,
|
|
174
|
+
"reason": "The trigger uses the correct FTS5 external content table deletion syntax: INSERT INTO frames_fts(frames_fts, rowid, ...) VALUES ('delete', old.rowid, ...)."
|
|
175
175
|
},
|
|
176
176
|
"references_old_row": {
|
|
177
|
-
"passed":
|
|
178
|
-
"reason": "
|
|
177
|
+
"passed": true,
|
|
178
|
+
"reason": "The trigger correctly references old.rowid, old.name, old.digest_text, and old.inputs, old.outputs to identify the deleted row."
|
|
179
179
|
},
|
|
180
180
|
"matches_column_list": {
|
|
181
|
-
"passed":
|
|
182
|
-
"reason": "
|
|
181
|
+
"passed": true,
|
|
182
|
+
"reason": "The column list (frames_fts, rowid, name, digest_text, inputs, outputs) matches the pattern established in the UPDATE trigger and reflects the FTS5 table schema."
|
|
183
183
|
}
|
|
184
184
|
},
|
|
185
|
-
"judgeMode": "
|
|
186
|
-
"duration":
|
|
187
|
-
"output": "
|
|
185
|
+
"judgeMode": "llm",
|
|
186
|
+
"duration": 22304,
|
|
187
|
+
"output": "The edit was denied. Here's the trigger to add — it follows the same \"delete\" command pattern used in the UPDATE trigger:\n\n```sql\n-- Sync trigger: DELETE\nCREATE TRIGGER IF NOT EXISTS frames_ad AFTER DELETE ON frames BEGIN\n INSERT INTO frames_fts(frames_fts, rowid, name, digest_text, inputs, outputs)\n VALUES ('delete', old.rowid, old.name, old.digest_text, old.inputs, old.outputs);\nEND;\n```\n\n**How it works:** FTS5 external content tables require a special `INSERT ... VALUES ('delete', ...)` command to remove a row from the index. The UPDATE trigger already demonstrates this pattern — it issues a `'delete'` for the old row before inserting the new one. The DELETE trigger just does the `'delete'` step without re-inserting.\n"
|
|
188
188
|
},
|
|
189
189
|
{
|
|
190
190
|
"taskId": "sm-003",
|
|
191
191
|
"taskName": "daemon_service_lifecycle",
|
|
192
192
|
"weight": 1.5,
|
|
193
|
-
"passed":
|
|
194
|
-
"passRate":
|
|
193
|
+
"passed": true,
|
|
194
|
+
"passRate": 1,
|
|
195
195
|
"criteria": {
|
|
196
196
|
"clears_old_interval": {
|
|
197
|
-
"passed":
|
|
198
|
-
"reason": "
|
|
197
|
+
"passed": true,
|
|
198
|
+
"reason": "The fix explicitly calls `this.stop()` before setting `isRunning = true`, which clears the existing interval before creating a new one."
|
|
199
199
|
},
|
|
200
200
|
"prevents_double_start": {
|
|
201
|
-
"passed":
|
|
202
|
-
"reason": "
|
|
201
|
+
"passed": true,
|
|
202
|
+
"reason": "The guard `if (this.isRunning) this.stop()` prevents double-start by ensuring the old interval is cleared if start() is called while already running."
|
|
203
203
|
},
|
|
204
204
|
"has_getstate_method": {
|
|
205
|
-
"passed":
|
|
206
|
-
"reason": "
|
|
205
|
+
"passed": true,
|
|
206
|
+
"reason": "A complete `getState()` method is defined with proper return type `ServiceState` and implementation."
|
|
207
207
|
},
|
|
208
208
|
"returns_correct_state_shape": {
|
|
209
|
-
"passed":
|
|
210
|
-
"reason": "
|
|
209
|
+
"passed": true,
|
|
210
|
+
"reason": "The method returns an object with all four required fields: isRunning, intervalMs, lastRunTime, and errorCount with appropriate values."
|
|
211
211
|
}
|
|
212
212
|
},
|
|
213
|
-
"judgeMode": "
|
|
214
|
-
"duration":
|
|
215
|
-
"output": "
|
|
213
|
+
"judgeMode": "llm",
|
|
214
|
+
"duration": 51339,
|
|
215
|
+
"output": "It looks like write permissions aren't enabled for this file. Here are the two changes needed:\n\n**Bug fix — `start()` guard** (`leaky-service.ts:28`):\n```ts\nstart(): void {\n if (this.isRunning) this.stop(); // clear existing interval before starting\n this.isRunning = true;\n ...\n}\n```\nThis ensures any leaked interval is cleared before a new one is created, whether called directly or via `updateConfig()`. The `updateConfig()` method itself is fine — the root issue was `start()` not being idempotent.\n\n**New `getState()` method** (replace the TODO comment):\n```ts\ngetState(): ServiceState {\n return {\n isRunning: this.isRunning,\n intervalMs: this.config.interval * 60 * 1000,\n lastRunTime: this.lastRunTime,\n errorCount: this.errorCount,\n };\n}\n```\n\nThe `ServiceState` interface is already defined at line 7-12, so `getState()` satisfies it directly. Please grant write access so I can apply these edits.\n"
|
|
216
216
|
}
|
|
217
217
|
]
|
|
218
218
|
}
|
|
@@ -7,72 +7,72 @@
|
|
|
7
7
|
"taskName": "simple_function",
|
|
8
8
|
"weight": 1,
|
|
9
9
|
"passed": false,
|
|
10
|
-
"passRate": 0,
|
|
10
|
+
"passRate": 0.3333333333333333,
|
|
11
11
|
"criteria": {
|
|
12
12
|
"has_function": {
|
|
13
|
-
"passed":
|
|
14
|
-
"reason": "
|
|
13
|
+
"passed": true,
|
|
14
|
+
"reason": "A properly defined function `is_palindrome` with type hints is provided and would execute correctly"
|
|
15
15
|
},
|
|
16
16
|
"handles_edge_cases": {
|
|
17
17
|
"passed": false,
|
|
18
|
-
"reason": "
|
|
18
|
+
"reason": "The function only handles spaces and case sensitivity, but does not handle punctuation, numbers, or special characters that should be ignored in palindrome checking"
|
|
19
19
|
},
|
|
20
20
|
"is_tested": {
|
|
21
21
|
"passed": false,
|
|
22
|
-
"reason": "
|
|
22
|
+
"reason": "Only two simple usage examples are shown, not actual test cases or assertions. No comprehensive testing framework or multiple edge case tests are provided"
|
|
23
23
|
}
|
|
24
24
|
},
|
|
25
|
-
"judgeMode": "
|
|
26
|
-
"duration":
|
|
27
|
-
"output": "
|
|
25
|
+
"judgeMode": "llm",
|
|
26
|
+
"duration": 59586,
|
|
27
|
+
"output": "```python\ndef is_palindrome(s: str) -> bool:\n s = s.lower().replace(\" \", \"\")\n return s == s[::-1]\n```\n\nUsage: `is_palindrome(\"racecar\")` → `True`, `is_palindrome(\"hello\")` → `False`\n"
|
|
28
28
|
},
|
|
29
29
|
{
|
|
30
30
|
"taskId": "eval-002",
|
|
31
31
|
"taskName": "refactor_code",
|
|
32
32
|
"weight": 1.5,
|
|
33
33
|
"passed": false,
|
|
34
|
-
"passRate": 0
|
|
34
|
+
"passRate": 0,
|
|
35
35
|
"criteria": {
|
|
36
36
|
"uses_async": {
|
|
37
37
|
"passed": false,
|
|
38
|
-
"reason": "
|
|
38
|
+
"reason": "No async/await code is present in the output. The AI only requested the input code without providing any refactored solution."
|
|
39
39
|
},
|
|
40
40
|
"preserves_behavior": {
|
|
41
41
|
"passed": false,
|
|
42
|
-
"reason": "
|
|
42
|
+
"reason": "No refactored code was provided, so behavior preservation cannot be verified. The output is a request for input rather than a solution."
|
|
43
43
|
},
|
|
44
44
|
"no_nested_callbacks": {
|
|
45
|
-
"passed":
|
|
46
|
-
"reason": "
|
|
45
|
+
"passed": false,
|
|
46
|
+
"reason": "No code was generated to evaluate callback nesting. The output contains no implementation whatsoever."
|
|
47
47
|
}
|
|
48
48
|
},
|
|
49
|
-
"judgeMode": "
|
|
50
|
-
"duration":
|
|
51
|
-
"output": "
|
|
49
|
+
"judgeMode": "llm",
|
|
50
|
+
"duration": 23652,
|
|
51
|
+
"output": "Please share the code or file path you'd like me to refactor.\n"
|
|
52
52
|
},
|
|
53
53
|
{
|
|
54
54
|
"taskId": "eval-003",
|
|
55
55
|
"taskName": "fix_bug",
|
|
56
56
|
"weight": 1.2,
|
|
57
57
|
"passed": false,
|
|
58
|
-
"passRate": 0,
|
|
58
|
+
"passRate": 0.3333333333333333,
|
|
59
59
|
"criteria": {
|
|
60
60
|
"bug_fixed": {
|
|
61
61
|
"passed": false,
|
|
62
|
-
"reason": "
|
|
62
|
+
"reason": "No code was provided by the user, so no bug could be fixed. The AI correctly identified this but did not fix anything."
|
|
63
63
|
},
|
|
64
64
|
"no_new_bugs": {
|
|
65
|
-
"passed":
|
|
66
|
-
"reason": "
|
|
65
|
+
"passed": true,
|
|
66
|
+
"reason": "The AI response contains no code, so no new bugs could have been introduced."
|
|
67
67
|
},
|
|
68
68
|
"explains_fix": {
|
|
69
69
|
"passed": false,
|
|
70
|
-
"reason": "
|
|
70
|
+
"reason": "No explanation of a fix was provided because no code was given to fix. The AI only requested the missing code."
|
|
71
71
|
}
|
|
72
72
|
},
|
|
73
|
-
"judgeMode": "
|
|
74
|
-
"duration":
|
|
75
|
-
"output": "
|
|
73
|
+
"judgeMode": "llm",
|
|
74
|
+
"duration": 40805,
|
|
75
|
+
"output": "I don't see any code in your message. Could you share the loop you'd like me to fix?\n"
|
|
76
76
|
},
|
|
77
77
|
{
|
|
78
78
|
"taskId": "eval-004",
|
|
@@ -83,24 +83,24 @@
|
|
|
83
83
|
"criteria": {
|
|
84
84
|
"has_pagination": {
|
|
85
85
|
"passed": false,
|
|
86
|
-
"reason": "
|
|
86
|
+
"reason": "No pagination code is provided; AI only asks for clarification instead of implementing pagination"
|
|
87
87
|
},
|
|
88
88
|
"has_limit": {
|
|
89
89
|
"passed": false,
|
|
90
|
-
"reason": "
|
|
90
|
+
"reason": "No limit parameter is implemented in the output"
|
|
91
91
|
},
|
|
92
92
|
"has_offset": {
|
|
93
93
|
"passed": false,
|
|
94
|
-
"reason": "
|
|
94
|
+
"reason": "No offset parameter is implemented in the output"
|
|
95
95
|
},
|
|
96
96
|
"backwards_compatible": {
|
|
97
97
|
"passed": false,
|
|
98
|
-
"reason": "
|
|
98
|
+
"reason": "No implementation provided to verify backwards compatibility"
|
|
99
99
|
}
|
|
100
100
|
},
|
|
101
|
-
"judgeMode": "
|
|
102
|
-
"duration":
|
|
103
|
-
"output": "API
|
|
101
|
+
"judgeMode": "llm",
|
|
102
|
+
"duration": 15357,
|
|
103
|
+
"output": "Which API endpoint should I add pagination to? Please share the file path or endpoint name.\n"
|
|
104
104
|
},
|
|
105
105
|
{
|
|
106
106
|
"taskId": "eval-005",
|
|
@@ -111,80 +111,80 @@
|
|
|
111
111
|
"criteria": {
|
|
112
112
|
"identifies_security_issue": {
|
|
113
113
|
"passed": false,
|
|
114
|
-
"reason": "
|
|
114
|
+
"reason": "The output does not identify any security issues. Instead, it requests input because the directory is not a git repository."
|
|
115
115
|
},
|
|
116
116
|
"identifies_performance_issue": {
|
|
117
117
|
"passed": false,
|
|
118
|
-
"reason": "
|
|
118
|
+
"reason": "The output does not identify any performance issues. No code review is performed."
|
|
119
119
|
},
|
|
120
120
|
"actionable_feedback": {
|
|
121
121
|
"passed": false,
|
|
122
|
-
"reason": "
|
|
122
|
+
"reason": "The output provides meta-level requests for information rather than actionable feedback on actual code. It explains what's needed to proceed, not how to fix issues."
|
|
123
123
|
}
|
|
124
124
|
},
|
|
125
|
-
"judgeMode": "
|
|
126
|
-
"duration":
|
|
127
|
-
"output": "
|
|
125
|
+
"judgeMode": "llm",
|
|
126
|
+
"duration": 34819,
|
|
127
|
+
"output": "This directory (`/private/tmp/gepa-eval-ERpk5g`) is not a git repository, so I can't list PRs directly.\n\nTo review a PR, please provide:\n1. A PR URL (e.g., `https://github.com/org/repo/pull/123`)\n2. Or the PR number and repo name\n"
|
|
128
128
|
},
|
|
129
129
|
{
|
|
130
130
|
"taskId": "sm-001",
|
|
131
131
|
"taskName": "add_mcp_tool_handler",
|
|
132
132
|
"weight": 1.5,
|
|
133
|
-
"passed":
|
|
134
|
-
"passRate":
|
|
133
|
+
"passed": true,
|
|
134
|
+
"passRate": 1,
|
|
135
135
|
"criteria": {
|
|
136
136
|
"has_switch_case": {
|
|
137
|
-
"passed":
|
|
138
|
-
"reason": "
|
|
137
|
+
"passed": true,
|
|
138
|
+
"reason": "Code shows a proper `case 'get_frame_summary':` block that follows the switch/case dispatch pattern"
|
|
139
139
|
},
|
|
140
140
|
"has_zod_schema": {
|
|
141
|
-
"passed":
|
|
142
|
-
"reason": "
|
|
141
|
+
"passed": true,
|
|
142
|
+
"reason": "Zod schema is defined with `z.object({ frameId: z.string() }).parse(args)` for input validation"
|
|
143
143
|
},
|
|
144
144
|
"validates_input": {
|
|
145
|
-
"passed":
|
|
146
|
-
"reason": "
|
|
145
|
+
"passed": true,
|
|
146
|
+
"reason": "Input validation occurs via Zod's `.parse()` method which throws on invalid input; frameId is required string"
|
|
147
147
|
},
|
|
148
148
|
"returns_typed_response": {
|
|
149
|
-
"passed":
|
|
150
|
-
"reason": "
|
|
149
|
+
"passed": true,
|
|
150
|
+
"reason": "Returns object with exact required fields: frameId, name, status, eventCount in correct structure"
|
|
151
151
|
},
|
|
152
152
|
"handles_not_found": {
|
|
153
|
-
"passed":
|
|
154
|
-
"reason": "
|
|
153
|
+
"passed": true,
|
|
154
|
+
"reason": "Includes explicit null check `if (!frame) throw new Error()` that handles missing frame case"
|
|
155
155
|
}
|
|
156
156
|
},
|
|
157
|
-
"judgeMode": "
|
|
158
|
-
"duration":
|
|
159
|
-
"output": "
|
|
157
|
+
"judgeMode": "llm",
|
|
158
|
+
"duration": 41911,
|
|
159
|
+
"output": "Please approve the file write permission and I'll add the handler. The change inserts this case block before the `default:` branch:\n\n```ts\ncase 'get_frame_summary': {\n const input = z.object({ frameId: z.string() }).parse(args);\n const frame = frames.get(input.frameId);\n if (!frame) throw new Error(`Frame not found: ${input.frameId}`);\n return { frameId: frame.id, name: frame.name, status: frame.status, eventCount: frame.events.length };\n}\n```\n"
|
|
160
160
|
},
|
|
161
161
|
{
|
|
162
162
|
"taskId": "sm-002",
|
|
163
163
|
"taskName": "fix_fts5_trigger_bug",
|
|
164
164
|
"weight": 1.8,
|
|
165
|
-
"passed":
|
|
166
|
-
"passRate":
|
|
165
|
+
"passed": true,
|
|
166
|
+
"passRate": 1,
|
|
167
167
|
"criteria": {
|
|
168
168
|
"has_delete_trigger": {
|
|
169
|
-
"passed":
|
|
170
|
-
"reason": "
|
|
169
|
+
"passed": true,
|
|
170
|
+
"reason": "A complete AFTER DELETE trigger named 'frames_ad' is defined with proper CREATE TRIGGER syntax"
|
|
171
171
|
},
|
|
172
172
|
"uses_fts_delete_syntax": {
|
|
173
|
-
"passed":
|
|
174
|
-
"reason": "
|
|
173
|
+
"passed": true,
|
|
174
|
+
"reason": "The trigger uses the FTS5 'delete' command by inserting 'delete' as the first value in the frames_fts table"
|
|
175
175
|
},
|
|
176
176
|
"references_old_row": {
|
|
177
|
-
"passed":
|
|
178
|
-
"reason": "
|
|
177
|
+
"passed": true,
|
|
178
|
+
"reason": "The trigger references old.rowid, old.name, old.digest_text, and old.inputs, old.outputs to access deleted row values"
|
|
179
179
|
},
|
|
180
180
|
"matches_column_list": {
|
|
181
|
-
"passed":
|
|
182
|
-
"reason": "
|
|
181
|
+
"passed": true,
|
|
182
|
+
"reason": "The column list (frames_fts, rowid, name, digest_text, inputs, outputs) matches the pattern used in the INSERT and UPDATE triggers"
|
|
183
183
|
}
|
|
184
184
|
},
|
|
185
|
-
"judgeMode": "
|
|
186
|
-
"duration":
|
|
187
|
-
"output": "
|
|
185
|
+
"judgeMode": "llm",
|
|
186
|
+
"duration": 46145,
|
|
187
|
+
"output": "It seems write permission is being blocked. Here's the trigger to add:\n\n```sql\n-- Sync trigger: DELETE\nCREATE TRIGGER IF NOT EXISTS frames_ad AFTER DELETE ON frames BEGIN\n INSERT INTO frames_fts(frames_fts, rowid, name, digest_text, inputs, outputs)\n VALUES ('delete', old.rowid, old.name, old.digest_text, old.inputs, old.outputs);\nEND;\n```\n\nThis follows the same FTS5 \"delete\" command pattern used in the UPDATE trigger — inserting a row with the special `'delete'` command removes it from the index. Replace the `-- BUG: Missing DELETE trigger!` comment block with the above.\n"
|
|
188
188
|
},
|
|
189
189
|
{
|
|
190
190
|
"taskId": "sm-003",
|
|
@@ -195,24 +195,24 @@
|
|
|
195
195
|
"criteria": {
|
|
196
196
|
"clears_old_interval": {
|
|
197
197
|
"passed": false,
|
|
198
|
-
"reason": "
|
|
198
|
+
"reason": "AI only describes the fix in pseudo-code but does not actually apply it to the file. No real code changes were made; the AI requested permission instead of providing the complete fixed code."
|
|
199
199
|
},
|
|
200
200
|
"prevents_double_start": {
|
|
201
201
|
"passed": false,
|
|
202
|
-
"reason": "
|
|
202
|
+
"reason": "While the proposed logic (if this.isRunning) this.stop() would prevent double-start, it was never actually implemented in working code. Only a description of the intended fix was provided."
|
|
203
203
|
},
|
|
204
204
|
"has_getstate_method": {
|
|
205
205
|
"passed": false,
|
|
206
|
-
"reason": "
|
|
206
|
+
"reason": "The getState() method is only shown in pseudo-code format. No actual working implementation was delivered to the file. The method was not truly defined in executable code."
|
|
207
207
|
},
|
|
208
208
|
"returns_correct_state_shape": {
|
|
209
209
|
"passed": false,
|
|
210
|
-
"reason": "
|
|
210
|
+
"reason": "While the proposed return object structure looks correct in the description, there is no actual working getState() method implemented. The shape is theoretical only, not realized in code."
|
|
211
211
|
}
|
|
212
212
|
},
|
|
213
|
-
"judgeMode": "
|
|
214
|
-
"duration":
|
|
215
|
-
"output": "
|
|
213
|
+
"judgeMode": "llm",
|
|
214
|
+
"duration": 64211,
|
|
215
|
+
"output": "It seems file writes are being blocked. Please grant write permission to `/private/tmp/gepa-eval-akYawy/leaky-service.ts` so I can apply the fixes.\n\nHere's what the changes will do:\n\n**Bug fix — `start()` guard:**\n```ts\nstart(): void {\n if (this.isRunning) this.stop(); // clear any existing interval before starting\n ...\n}\n```\nThis ensures any existing interval is always cleared before creating a new one, whether called directly or via `updateConfig()`. The `updateConfig` comment/bug note is also removed since the fix makes it a non-issue.\n\n**New `getState()` method:**\n```ts\ngetState(): ServiceState {\n return {\n isRunning: this.isRunning,\n intervalMs: this.config.interval * 60 * 1000,\n lastRunTime: this.lastRunTime,\n errorCount: this.errorCount,\n };\n}\n```\n\nOnce you grant write permissions, I'll apply both changes at once.\n"
|
|
216
216
|
}
|
|
217
217
|
]
|
|
218
218
|
}
|