@stackmemoryai/stackmemory 1.6.0 → 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cli/commands/orchestrate.js +334 -4
- package/dist/src/cli/commands/orchestrator.js +283 -24
- package/package.json +1 -2
- package/scripts/gepa/.before-optimize.md +0 -112
- package/scripts/gepa/README.md +0 -275
- package/scripts/gepa/config.json +0 -59
- package/scripts/gepa/evals/coding-tasks.jsonl +0 -5
- package/scripts/gepa/evals/fixtures/api-endpoint.ts +0 -31
- package/scripts/gepa/evals/fixtures/brittle-integration.ts +0 -38
- package/scripts/gepa/evals/fixtures/buggy-loop.js +0 -18
- package/scripts/gepa/evals/fixtures/callback-hell.js +0 -53
- package/scripts/gepa/evals/fixtures/fts5-triggers.sql +0 -23
- package/scripts/gepa/evals/fixtures/leaky-service.ts +0 -70
- package/scripts/gepa/evals/fixtures/mcp-dispatch-stub.ts +0 -39
- package/scripts/gepa/evals/fixtures/pr-diff.txt +0 -24
- package/scripts/gepa/evals/fixtures/unsafe-webhook.ts +0 -34
- package/scripts/gepa/evals/fixtures/unwrapped-db-op.ts +0 -42
- package/scripts/gepa/evals/stackmemory-tasks.jsonl +0 -8
- package/scripts/gepa/generations/gen-000/baseline.md +0 -112
- package/scripts/gepa/generations/gen-001/baseline.md +0 -112
- package/scripts/gepa/generations/gen-001/variant-a.md +0 -107
- package/scripts/gepa/generations/gen-001/variant-b.md +0 -216
- package/scripts/gepa/generations/gen-001/variant-c.md +0 -83
- package/scripts/gepa/generations/gen-001/variant-d.md +0 -90
- package/scripts/gepa/hooks/auto-optimize.js +0 -494
- package/scripts/gepa/hooks/eval-tracker.js +0 -203
- package/scripts/gepa/hooks/reflect.js +0 -350
- package/scripts/gepa/optimize.js +0 -853
- package/scripts/gepa/results/eval-1-baseline.json +0 -218
- package/scripts/gepa/results/eval-1-variant-a.json +0 -218
- package/scripts/gepa/results/eval-1-variant-b.json +0 -218
- package/scripts/gepa/results/eval-1-variant-c.json +0 -218
- package/scripts/gepa/results/eval-1-variant-d.json +0 -218
- package/scripts/gepa/state.json +0 -49
package/scripts/gepa/optimize.js
DELETED
|
@@ -1,853 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
/**
|
|
3
|
-
* GEPA Optimizer
|
|
4
|
-
*
|
|
5
|
-
* Genetic Eval-driven Prompt Algorithm for optimizing CLAUDE.md
|
|
6
|
-
*
|
|
7
|
-
* Usage:
|
|
8
|
-
* node optimize.js init # Initialize with current CLAUDE.md
|
|
9
|
-
* node optimize.js mutate # Generate new variants
|
|
10
|
-
* node optimize.js eval [variant] # Run evals on variant(s)
|
|
11
|
-
* node optimize.js score # Score all variants in generation
|
|
12
|
-
* node optimize.js select # Select best, advance generation
|
|
13
|
-
* node optimize.js run [generations] # Full optimization loop
|
|
14
|
-
* node optimize.js status # Show current status
|
|
15
|
-
* node optimize.js diff [a] [b] # Compare two variants
|
|
16
|
-
*/
|
|
17
|
-
|
|
18
|
-
import fs from 'fs';
|
|
19
|
-
import path from 'path';
|
|
20
|
-
import { fileURLToPath } from 'url';
|
|
21
|
-
import { spawn, execSync } from 'child_process';
|
|
22
|
-
|
|
23
|
-
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
24
|
-
const PROJECT_ROOT = path.resolve(__dirname, '../..');
|
|
25
|
-
|
|
26
|
-
// Load .env from project root
|
|
27
|
-
const envPath = path.join(PROJECT_ROOT, '.env');
|
|
28
|
-
if (fs.existsSync(envPath)) {
|
|
29
|
-
for (const line of fs.readFileSync(envPath, 'utf8').split('\n')) {
|
|
30
|
-
const match = line.match(/^([A-Z_][A-Z0-9_]*)=(.*)$/);
|
|
31
|
-
if (match && !process.env[match[1]]) {
|
|
32
|
-
process.env[match[1]] = match[2].replace(/^["']|["']$/g, '');
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
// Configuration
|
|
38
|
-
const CONFIG_PATH = path.join(__dirname, 'config.json');
|
|
39
|
-
const config = JSON.parse(fs.readFileSync(CONFIG_PATH, 'utf8'));
|
|
40
|
-
|
|
41
|
-
const GEPA_DIR = process.env.GEPA_DIR || __dirname;
|
|
42
|
-
const GENERATIONS_DIR = path.join(GEPA_DIR, 'generations');
|
|
43
|
-
const RESULTS_DIR = path.join(GEPA_DIR, 'results');
|
|
44
|
-
const EVALS_DIR = path.join(GEPA_DIR, 'evals');
|
|
45
|
-
|
|
46
|
-
// Ensure directories
|
|
47
|
-
[GENERATIONS_DIR, RESULTS_DIR, EVALS_DIR].forEach((dir) => {
|
|
48
|
-
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
49
|
-
});
|
|
50
|
-
|
|
51
|
-
/**
|
|
52
|
-
* State management
|
|
53
|
-
*/
|
|
54
|
-
function getState() {
|
|
55
|
-
const statePath = path.join(GEPA_DIR, 'state.json');
|
|
56
|
-
if (fs.existsSync(statePath)) {
|
|
57
|
-
return JSON.parse(fs.readFileSync(statePath, 'utf8'));
|
|
58
|
-
}
|
|
59
|
-
return {
|
|
60
|
-
currentGeneration: 0,
|
|
61
|
-
bestVariant: null,
|
|
62
|
-
bestScore: 0,
|
|
63
|
-
history: [],
|
|
64
|
-
};
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
function saveState(state) {
|
|
68
|
-
fs.writeFileSync(
|
|
69
|
-
path.join(GEPA_DIR, 'state.json'),
|
|
70
|
-
JSON.stringify(state, null, 2)
|
|
71
|
-
);
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
/**
|
|
75
|
-
* Get path for a generation/variant
|
|
76
|
-
*/
|
|
77
|
-
function getGenPath(gen, variant = null) {
|
|
78
|
-
const genDir = path.join(
|
|
79
|
-
GENERATIONS_DIR,
|
|
80
|
-
`gen-${String(gen).padStart(3, '0')}`
|
|
81
|
-
);
|
|
82
|
-
if (!fs.existsSync(genDir)) fs.mkdirSync(genDir, { recursive: true });
|
|
83
|
-
return variant ? path.join(genDir, `${variant}.md`) : genDir;
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
/**
|
|
87
|
-
* Initialize GEPA with current CLAUDE.md
|
|
88
|
-
*/
|
|
89
|
-
async function init(targetPath) {
|
|
90
|
-
const claudeMdPath = targetPath || path.join(process.cwd(), 'CLAUDE.md');
|
|
91
|
-
|
|
92
|
-
if (!fs.existsSync(claudeMdPath)) {
|
|
93
|
-
console.error(`Error: ${claudeMdPath} not found`);
|
|
94
|
-
process.exit(1);
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
const content = fs.readFileSync(claudeMdPath, 'utf8');
|
|
98
|
-
const genPath = getGenPath(0, 'baseline');
|
|
99
|
-
|
|
100
|
-
fs.writeFileSync(genPath, content);
|
|
101
|
-
|
|
102
|
-
const state = {
|
|
103
|
-
currentGeneration: 0,
|
|
104
|
-
bestVariant: 'baseline',
|
|
105
|
-
bestScore: 0,
|
|
106
|
-
targetPath: claudeMdPath,
|
|
107
|
-
history: [
|
|
108
|
-
{
|
|
109
|
-
generation: 0,
|
|
110
|
-
variant: 'baseline',
|
|
111
|
-
action: 'init',
|
|
112
|
-
timestamp: new Date().toISOString(),
|
|
113
|
-
},
|
|
114
|
-
],
|
|
115
|
-
};
|
|
116
|
-
|
|
117
|
-
saveState(state);
|
|
118
|
-
console.log(`Initialized GEPA with ${claudeMdPath}`);
|
|
119
|
-
console.log(`Baseline saved to ${genPath}`);
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
/**
|
|
123
|
-
* Generate mutations of the current best variant
|
|
124
|
-
*/
|
|
125
|
-
async function mutate() {
|
|
126
|
-
const state = getState();
|
|
127
|
-
const nextGen = state.currentGeneration + 1;
|
|
128
|
-
const currentBest = fs.readFileSync(
|
|
129
|
-
getGenPath(state.currentGeneration, state.bestVariant),
|
|
130
|
-
'utf8'
|
|
131
|
-
);
|
|
132
|
-
|
|
133
|
-
console.log(
|
|
134
|
-
`Generating ${config.evolution.populationSize} variants for generation ${nextGen}...`
|
|
135
|
-
);
|
|
136
|
-
|
|
137
|
-
const mutations = config.evolution.mutationStrategies;
|
|
138
|
-
const variants = [];
|
|
139
|
-
|
|
140
|
-
for (let i = 0; i < config.evolution.populationSize; i++) {
|
|
141
|
-
const strategy = mutations[i % mutations.length];
|
|
142
|
-
const variantName = `variant-${String.fromCharCode(97 + i)}`; // a, b, c, d...
|
|
143
|
-
|
|
144
|
-
console.log(` Creating ${variantName} using strategy: ${strategy}`);
|
|
145
|
-
|
|
146
|
-
const mutatedContent = await generateMutation(currentBest, strategy, state);
|
|
147
|
-
const variantPath = getGenPath(nextGen, variantName);
|
|
148
|
-
|
|
149
|
-
fs.writeFileSync(variantPath, mutatedContent);
|
|
150
|
-
variants.push({ name: variantName, strategy, path: variantPath });
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
// Also copy baseline for comparison
|
|
154
|
-
fs.writeFileSync(getGenPath(nextGen, 'baseline'), currentBest);
|
|
155
|
-
|
|
156
|
-
state.history.push({
|
|
157
|
-
generation: nextGen,
|
|
158
|
-
action: 'mutate',
|
|
159
|
-
variants: variants.map((v) => v.name),
|
|
160
|
-
timestamp: new Date().toISOString(),
|
|
161
|
-
});
|
|
162
|
-
saveState(state);
|
|
163
|
-
|
|
164
|
-
console.log(
|
|
165
|
-
`\nGenerated ${variants.length} variants in gen-${String(nextGen).padStart(3, '0')}/`
|
|
166
|
-
);
|
|
167
|
-
return variants;
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
/**
|
|
171
|
-
* Generate a mutation using AI
|
|
172
|
-
*/
|
|
173
|
-
async function generateMutation(content, strategy, state) {
|
|
174
|
-
const strategyPrompts = {
|
|
175
|
-
rephrase: `Rephrase instructions for clarity without changing meaning. Make them more direct and actionable.`,
|
|
176
|
-
|
|
177
|
-
add_examples: `Add concrete examples where instructions are abstract. Use <example> tags for code examples.`,
|
|
178
|
-
|
|
179
|
-
remove_redundancy: `Remove redundant or repetitive instructions. Consolidate similar rules. Keep it DRY.`,
|
|
180
|
-
|
|
181
|
-
restructure: `Reorganize sections for better flow. Group related instructions. Improve hierarchy.`,
|
|
182
|
-
|
|
183
|
-
add_constraints: `Add specific constraints and guardrails based on common failure modes. Be precise about what NOT to do.`,
|
|
184
|
-
|
|
185
|
-
simplify: `Simplify complex instructions. Break down multi-step rules. Use bullet points over paragraphs.`,
|
|
186
|
-
};
|
|
187
|
-
|
|
188
|
-
const prompt = `You are optimizing a CLAUDE.md system prompt for an AI coding agent.
|
|
189
|
-
|
|
190
|
-
CURRENT PROMPT:
|
|
191
|
-
\`\`\`markdown
|
|
192
|
-
${content}
|
|
193
|
-
\`\`\`
|
|
194
|
-
|
|
195
|
-
OPTIMIZATION STRATEGY: ${strategy}
|
|
196
|
-
${strategyPrompts[strategy]}
|
|
197
|
-
|
|
198
|
-
EVALUATION FEEDBACK FROM PREVIOUS GENERATIONS:
|
|
199
|
-
${getRecentFeedback(state)}
|
|
200
|
-
|
|
201
|
-
REFLECTION INSIGHTS (from failure pattern analysis):
|
|
202
|
-
${getReflectionInsights()}
|
|
203
|
-
|
|
204
|
-
REQUIREMENTS:
|
|
205
|
-
1. Output ONLY the improved markdown content
|
|
206
|
-
2. Preserve all critical instructions and constraints
|
|
207
|
-
3. Keep the same overall structure unless restructuring
|
|
208
|
-
4. Do not add commentary or explanations
|
|
209
|
-
5. Target <8000 tokens total length
|
|
210
|
-
|
|
211
|
-
OUTPUT THE IMPROVED CLAUDE.MD:`;
|
|
212
|
-
|
|
213
|
-
// Use Claude to generate mutation
|
|
214
|
-
const result = await callClaude(prompt);
|
|
215
|
-
return result.trim();
|
|
216
|
-
}
|
|
217
|
-
|
|
218
|
-
/**
|
|
219
|
-
* Get recent evaluation feedback for context
|
|
220
|
-
*/
|
|
221
|
-
function getRecentFeedback(state) {
|
|
222
|
-
const scoresPath = path.join(RESULTS_DIR, 'scores.jsonl');
|
|
223
|
-
if (!fs.existsSync(scoresPath)) return 'No previous evaluations.';
|
|
224
|
-
|
|
225
|
-
const lines = fs
|
|
226
|
-
.readFileSync(scoresPath, 'utf8')
|
|
227
|
-
.trim()
|
|
228
|
-
.split('\n')
|
|
229
|
-
.slice(-20);
|
|
230
|
-
const scores = lines.map((l) => JSON.parse(l));
|
|
231
|
-
|
|
232
|
-
const summary = scores.reduce((acc, s) => {
|
|
233
|
-
if (!acc[s.variant]) acc[s.variant] = { total: 0, count: 0, errors: 0 };
|
|
234
|
-
acc[s.variant].total += s.metrics?.successfulToolCalls || 0;
|
|
235
|
-
acc[s.variant].count++;
|
|
236
|
-
acc[s.variant].errors += s.metrics?.errorCount || 0;
|
|
237
|
-
return acc;
|
|
238
|
-
}, {});
|
|
239
|
-
|
|
240
|
-
return Object.entries(summary)
|
|
241
|
-
.map(
|
|
242
|
-
([v, s]) =>
|
|
243
|
-
`${v}: ${s.count} sessions, ${s.errors} errors, avg success: ${(s.total / s.count).toFixed(1)}`
|
|
244
|
-
)
|
|
245
|
-
.join('\n');
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
/**
|
|
249
|
-
* Load most recent reflection insights for mutation context
|
|
250
|
-
*/
|
|
251
|
-
function getReflectionInsights() {
|
|
252
|
-
const reflectionFiles = fs.existsSync(RESULTS_DIR)
|
|
253
|
-
? fs
|
|
254
|
-
.readdirSync(RESULTS_DIR)
|
|
255
|
-
.filter((f) => f.startsWith('reflection-') && f.endsWith('.json'))
|
|
256
|
-
: [];
|
|
257
|
-
|
|
258
|
-
if (reflectionFiles.length === 0) return 'No reflection data yet.';
|
|
259
|
-
|
|
260
|
-
// Pick the most recent reflection file
|
|
261
|
-
reflectionFiles.sort().reverse();
|
|
262
|
-
const latest = JSON.parse(
|
|
263
|
-
fs.readFileSync(path.join(RESULTS_DIR, reflectionFiles[0]), 'utf8')
|
|
264
|
-
);
|
|
265
|
-
|
|
266
|
-
const insights = latest.insights;
|
|
267
|
-
if (!insights) return 'No reflection insights available.';
|
|
268
|
-
|
|
269
|
-
const parts = [];
|
|
270
|
-
|
|
271
|
-
if (insights.failureModes?.length) {
|
|
272
|
-
parts.push(`Failure modes: ${insights.failureModes.join('; ')}`);
|
|
273
|
-
}
|
|
274
|
-
if (insights.missingInstructions?.length) {
|
|
275
|
-
parts.push(
|
|
276
|
-
`Missing instructions: ${insights.missingInstructions.join('; ')}`
|
|
277
|
-
);
|
|
278
|
-
}
|
|
279
|
-
if (insights.unclearInstructions?.length) {
|
|
280
|
-
parts.push(
|
|
281
|
-
`Unclear instructions: ${insights.unclearInstructions.join('; ')}`
|
|
282
|
-
);
|
|
283
|
-
}
|
|
284
|
-
if (insights.priorityMutations?.length) {
|
|
285
|
-
parts.push(
|
|
286
|
-
`Priority changes:\n${insights.priorityMutations
|
|
287
|
-
.map((m) => ` - [${m.type}] ${m.section}: ${m.change}`)
|
|
288
|
-
.join('\n')}`
|
|
289
|
-
);
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
return parts.join('\n') || 'No actionable insights.';
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
/**
|
|
296
|
-
* Call Claude CLI via spawn (stdin pipe, no shell interpolation)
|
|
297
|
-
*/
|
|
298
|
-
function spawnClaude(prompt, { cwd, timeoutMs } = {}) {
|
|
299
|
-
return new Promise((resolve, reject) => {
|
|
300
|
-
const args = ['--print'];
|
|
301
|
-
const child = spawn('claude', args, {
|
|
302
|
-
cwd,
|
|
303
|
-
stdio: ['pipe', 'pipe', 'pipe'],
|
|
304
|
-
env: { ...process.env },
|
|
305
|
-
});
|
|
306
|
-
|
|
307
|
-
let stdout = '';
|
|
308
|
-
let stderr = '';
|
|
309
|
-
let killed = false;
|
|
310
|
-
|
|
311
|
-
const timer = timeoutMs
|
|
312
|
-
? setTimeout(() => {
|
|
313
|
-
killed = true;
|
|
314
|
-
child.kill('SIGTERM');
|
|
315
|
-
}, timeoutMs)
|
|
316
|
-
: null;
|
|
317
|
-
|
|
318
|
-
child.stdout.on('data', (d) => (stdout += d));
|
|
319
|
-
child.stderr.on('data', (d) => (stderr += d));
|
|
320
|
-
|
|
321
|
-
child.on('close', (code) => {
|
|
322
|
-
if (timer) clearTimeout(timer);
|
|
323
|
-
if (killed)
|
|
324
|
-
return reject(new Error(`claude timed out after ${timeoutMs}ms`));
|
|
325
|
-
if (code !== 0 && !stdout)
|
|
326
|
-
return reject(new Error(stderr || `claude exited ${code}`));
|
|
327
|
-
resolve(stdout);
|
|
328
|
-
});
|
|
329
|
-
|
|
330
|
-
child.on('error', (err) => {
|
|
331
|
-
if (timer) clearTimeout(timer);
|
|
332
|
-
reject(err);
|
|
333
|
-
});
|
|
334
|
-
|
|
335
|
-
child.stdin.write(prompt);
|
|
336
|
-
child.stdin.end();
|
|
337
|
-
});
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
/**
|
|
341
|
-
* Call Claude API for mutation generation
|
|
342
|
-
*/
|
|
343
|
-
async function callClaude(prompt) {
|
|
344
|
-
// Try using claude CLI first (stdin pipe, no shell injection)
|
|
345
|
-
try {
|
|
346
|
-
return await spawnClaude(prompt);
|
|
347
|
-
} catch (e) {
|
|
348
|
-
// Fallback to API
|
|
349
|
-
const apiKey = process.env.ANTHROPIC_API_KEY;
|
|
350
|
-
if (!apiKey) {
|
|
351
|
-
console.error('Error: ANTHROPIC_API_KEY not set and claude CLI failed');
|
|
352
|
-
process.exit(1);
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
const response = await fetch('https://api.anthropic.com/v1/messages', {
|
|
356
|
-
method: 'POST',
|
|
357
|
-
headers: {
|
|
358
|
-
'Content-Type': 'application/json',
|
|
359
|
-
'x-api-key': apiKey,
|
|
360
|
-
'anthropic-version': '2023-06-01',
|
|
361
|
-
},
|
|
362
|
-
body: JSON.stringify({
|
|
363
|
-
model: 'claude-sonnet-4-20250514',
|
|
364
|
-
max_tokens: 8000,
|
|
365
|
-
messages: [{ role: 'user', content: prompt }],
|
|
366
|
-
}),
|
|
367
|
-
});
|
|
368
|
-
|
|
369
|
-
const data = await response.json();
|
|
370
|
-
return data.content[0].text;
|
|
371
|
-
}
|
|
372
|
-
}
|
|
373
|
-
|
|
374
|
-
/**
|
|
375
|
-
* Run evaluations on a variant
|
|
376
|
-
*/
|
|
377
|
-
async function runEval(variantName) {
|
|
378
|
-
const state = getState();
|
|
379
|
-
const gen = state.currentGeneration + 1; // Eval next gen variants
|
|
380
|
-
const variantPath = getGenPath(gen, variantName);
|
|
381
|
-
|
|
382
|
-
if (!fs.existsSync(variantPath)) {
|
|
383
|
-
console.error(`Variant not found: ${variantPath}`);
|
|
384
|
-
return null;
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
console.log(`Running evals on ${variantName}...`);
|
|
388
|
-
|
|
389
|
-
// Load eval tasks
|
|
390
|
-
const evalFiles = fs
|
|
391
|
-
.readdirSync(EVALS_DIR)
|
|
392
|
-
.filter((f) => f.endsWith('.jsonl'));
|
|
393
|
-
const tasks = evalFiles.flatMap((f) =>
|
|
394
|
-
fs
|
|
395
|
-
.readFileSync(path.join(EVALS_DIR, f), 'utf8')
|
|
396
|
-
.trim()
|
|
397
|
-
.split('\n')
|
|
398
|
-
.map((l) => JSON.parse(l))
|
|
399
|
-
);
|
|
400
|
-
|
|
401
|
-
console.log(` Found ${tasks.length} eval tasks`);
|
|
402
|
-
|
|
403
|
-
// Set environment for tracking
|
|
404
|
-
process.env.GEPA_VARIANT = variantName;
|
|
405
|
-
process.env.GEPA_GENERATION = String(gen);
|
|
406
|
-
|
|
407
|
-
const results = [];
|
|
408
|
-
|
|
409
|
-
for (const task of tasks.slice(0, config.evals.minSamplesPerVariant)) {
|
|
410
|
-
console.log(` Running: ${task.name}`);
|
|
411
|
-
|
|
412
|
-
const result = await runSingleEval(task, variantPath);
|
|
413
|
-
results.push({
|
|
414
|
-
taskId: task.id,
|
|
415
|
-
taskName: task.name,
|
|
416
|
-
weight: task.weight || 1.0,
|
|
417
|
-
...result,
|
|
418
|
-
});
|
|
419
|
-
}
|
|
420
|
-
|
|
421
|
-
// Save results
|
|
422
|
-
const resultsPath = path.join(RESULTS_DIR, `eval-${gen}-${variantName}.json`);
|
|
423
|
-
fs.writeFileSync(
|
|
424
|
-
resultsPath,
|
|
425
|
-
JSON.stringify({ variant: variantName, generation: gen, results }, null, 2)
|
|
426
|
-
);
|
|
427
|
-
|
|
428
|
-
// Calculate aggregate score
|
|
429
|
-
const score = calculateScore(results);
|
|
430
|
-
console.log(` Score: ${(score * 100).toFixed(1)}%`);
|
|
431
|
-
|
|
432
|
-
return { variant: variantName, score, results };
|
|
433
|
-
}
|
|
434
|
-
|
|
435
|
-
/**
|
|
436
|
-
* Run a single eval task
|
|
437
|
-
*/
|
|
438
|
-
async function runSingleEval(task, variantPath) {
|
|
439
|
-
const startTime = Date.now();
|
|
440
|
-
let tempDir;
|
|
441
|
-
|
|
442
|
-
try {
|
|
443
|
-
// Create temp project with variant as CLAUDE.md
|
|
444
|
-
tempDir = fs.mkdtempSync('/tmp/gepa-eval-');
|
|
445
|
-
fs.copyFileSync(variantPath, path.join(tempDir, 'CLAUDE.md'));
|
|
446
|
-
|
|
447
|
-
// Copy fixture if needed
|
|
448
|
-
if (task.input_file) {
|
|
449
|
-
const fixturePath = path.join(EVALS_DIR, task.input_file);
|
|
450
|
-
if (fs.existsSync(fixturePath)) {
|
|
451
|
-
fs.copyFileSync(
|
|
452
|
-
fixturePath,
|
|
453
|
-
path.join(tempDir, path.basename(task.input_file))
|
|
454
|
-
);
|
|
455
|
-
}
|
|
456
|
-
}
|
|
457
|
-
|
|
458
|
-
// Run claude via spawn with Node-native timeout (no GNU timeout needed)
|
|
459
|
-
const result = await spawnClaude(task.prompt, {
|
|
460
|
-
cwd: tempDir,
|
|
461
|
-
timeoutMs: config.evals.timeout,
|
|
462
|
-
});
|
|
463
|
-
|
|
464
|
-
// Evaluate result against expected outcomes (LLM judge with regex fallback)
|
|
465
|
-
const evaluation = await evaluateExpectations(result, task.expected, task);
|
|
466
|
-
|
|
467
|
-
return {
|
|
468
|
-
passed: evaluation.passed,
|
|
469
|
-
passRate: evaluation.passRate,
|
|
470
|
-
criteria: evaluation.criteria,
|
|
471
|
-
judgeMode: evaluation.judgeMode,
|
|
472
|
-
duration: Date.now() - startTime,
|
|
473
|
-
output: result.slice(0, 2000),
|
|
474
|
-
};
|
|
475
|
-
} catch (error) {
|
|
476
|
-
return {
|
|
477
|
-
passed: false,
|
|
478
|
-
duration: Date.now() - startTime,
|
|
479
|
-
error: error.message,
|
|
480
|
-
};
|
|
481
|
-
} finally {
|
|
482
|
-
if (tempDir) fs.rmSync(tempDir, { recursive: true, force: true });
|
|
483
|
-
}
|
|
484
|
-
}
|
|
485
|
-
|
|
486
|
-
/**
|
|
487
|
-
* Evaluate output against expectations using LLM judge (regex fallback)
|
|
488
|
-
*/
|
|
489
|
-
async function evaluateExpectations(output, expected, task) {
|
|
490
|
-
if (!expected)
|
|
491
|
-
return { passed: true, passRate: 1.0, criteria: {}, judgeMode: 'skip' };
|
|
492
|
-
|
|
493
|
-
// Try LLM judge first
|
|
494
|
-
try {
|
|
495
|
-
const result = await llmJudge(output, expected, task);
|
|
496
|
-
return { ...result, judgeMode: 'llm' };
|
|
497
|
-
} catch (e) {
|
|
498
|
-
console.log(` LLM judge failed (${e.message}), using regex fallback`);
|
|
499
|
-
const result = regexJudge(output, expected);
|
|
500
|
-
return { ...result, judgeMode: 'regex' };
|
|
501
|
-
}
|
|
502
|
-
}
|
|
503
|
-
|
|
504
|
-
/**
|
|
505
|
-
* LLM-as-judge — uses a fast model to evaluate output against criteria
|
|
506
|
-
*/
|
|
507
|
-
async function llmJudge(output, expected, task) {
|
|
508
|
-
const criteriaList = Object.entries(expected)
|
|
509
|
-
.map(
|
|
510
|
-
([key, value]) =>
|
|
511
|
-
`- ${key}: ${typeof value === 'string' ? value : 'should be ' + value}`
|
|
512
|
-
)
|
|
513
|
-
.join('\n');
|
|
514
|
-
|
|
515
|
-
const judgePrompt = `You are a strict code evaluation judge. Evaluate whether the AI output satisfies each criterion.
|
|
516
|
-
|
|
517
|
-
TASK GIVEN TO AI:
|
|
518
|
-
${task.prompt}
|
|
519
|
-
|
|
520
|
-
AI OUTPUT:
|
|
521
|
-
\`\`\`
|
|
522
|
-
${output.slice(0, 6000)}
|
|
523
|
-
\`\`\`
|
|
524
|
-
|
|
525
|
-
CRITERIA TO EVALUATE:
|
|
526
|
-
${criteriaList}
|
|
527
|
-
|
|
528
|
-
For each criterion, determine if the output genuinely satisfies it. Be strict:
|
|
529
|
-
- "has_function" means a real, working function is defined (not just mentioned)
|
|
530
|
-
- "bug_fixed" means the actual bug is corrected (not just discussed)
|
|
531
|
-
- "handles_edge_cases" means edge cases are actually handled in code
|
|
532
|
-
- "explains_fix" means there's a clear explanation of what was wrong and why
|
|
533
|
-
|
|
534
|
-
Respond with ONLY this JSON (no markdown fences):
|
|
535
|
-
{
|
|
536
|
-
"criteria": {
|
|
537
|
-
"criterion_name": {"passed": true, "reason": "brief explanation"},
|
|
538
|
-
"criterion_name": {"passed": false, "reason": "brief explanation"}
|
|
539
|
-
}
|
|
540
|
-
}`;
|
|
541
|
-
|
|
542
|
-
const judgeModel = config.judge?.model || 'claude-haiku-4-5-20251001';
|
|
543
|
-
const raw = await callJudge(judgePrompt, judgeModel);
|
|
544
|
-
|
|
545
|
-
// Extract JSON from response
|
|
546
|
-
const jsonMatch = raw.match(/\{[\s\S]*\}/);
|
|
547
|
-
if (!jsonMatch) throw new Error('No JSON in judge response');
|
|
548
|
-
|
|
549
|
-
const parsed = JSON.parse(jsonMatch[0]);
|
|
550
|
-
const criteria = parsed.criteria || {};
|
|
551
|
-
const entries = Object.values(criteria);
|
|
552
|
-
const passedCount = entries.filter((c) => c.passed).length;
|
|
553
|
-
const passRate = entries.length > 0 ? passedCount / entries.length : 0;
|
|
554
|
-
|
|
555
|
-
return {
|
|
556
|
-
passed: passRate >= 0.6,
|
|
557
|
-
passRate,
|
|
558
|
-
criteria,
|
|
559
|
-
};
|
|
560
|
-
}
|
|
561
|
-
|
|
562
|
-
/**
|
|
563
|
-
* Call judge model via Anthropic API (fast, cheap model for evaluation)
|
|
564
|
-
*/
|
|
565
|
-
async function callJudge(prompt, model) {
|
|
566
|
-
const apiKey = process.env.ANTHROPIC_API_KEY;
|
|
567
|
-
|
|
568
|
-
if (apiKey) {
|
|
569
|
-
const response = await fetch('https://api.anthropic.com/v1/messages', {
|
|
570
|
-
method: 'POST',
|
|
571
|
-
headers: {
|
|
572
|
-
'Content-Type': 'application/json',
|
|
573
|
-
'x-api-key': apiKey,
|
|
574
|
-
'anthropic-version': '2023-06-01',
|
|
575
|
-
},
|
|
576
|
-
body: JSON.stringify({
|
|
577
|
-
model,
|
|
578
|
-
max_tokens: 2000,
|
|
579
|
-
messages: [{ role: 'user', content: prompt }],
|
|
580
|
-
}),
|
|
581
|
-
});
|
|
582
|
-
|
|
583
|
-
if (!response.ok) {
|
|
584
|
-
throw new Error(`Judge API error: ${response.status}`);
|
|
585
|
-
}
|
|
586
|
-
|
|
587
|
-
const data = await response.json();
|
|
588
|
-
return data.content[0].text;
|
|
589
|
-
}
|
|
590
|
-
|
|
591
|
-
// Fallback to CLI
|
|
592
|
-
return await spawnClaude(prompt, { timeoutMs: 30000 });
|
|
593
|
-
}
|
|
594
|
-
|
|
595
|
-
/**
|
|
596
|
-
* Regex fallback judge (used when LLM judge is unavailable)
|
|
597
|
-
*/
|
|
598
|
-
function regexJudge(output, expected) {
|
|
599
|
-
const criteria = {};
|
|
600
|
-
|
|
601
|
-
for (const [key, value] of Object.entries(expected)) {
|
|
602
|
-
let passed = false;
|
|
603
|
-
switch (key) {
|
|
604
|
-
case 'has_function':
|
|
605
|
-
passed =
|
|
606
|
-
/function\s+\w+|const\s+\w+\s*=\s*(\([^)]*\)|async)?\s*(=>|\{)/.test(
|
|
607
|
-
output
|
|
608
|
-
);
|
|
609
|
-
break;
|
|
610
|
-
case 'handles_edge_cases':
|
|
611
|
-
passed = /if\s*\(|edge|empty|null|undefined|\.length/.test(output);
|
|
612
|
-
break;
|
|
613
|
-
case 'uses_async':
|
|
614
|
-
passed = /async|await|Promise/.test(output);
|
|
615
|
-
break;
|
|
616
|
-
case 'no_nested_callbacks':
|
|
617
|
-
passed = !/callback\s*\(\s*function|\.then\s*\([^)]*\.then/.test(
|
|
618
|
-
output
|
|
619
|
-
);
|
|
620
|
-
break;
|
|
621
|
-
case 'bug_fixed':
|
|
622
|
-
passed = /fix|correct|change|update/i.test(output);
|
|
623
|
-
break;
|
|
624
|
-
case 'explains_fix':
|
|
625
|
-
passed =
|
|
626
|
-
output.length > 200 &&
|
|
627
|
-
/because|since|the issue|the problem/i.test(output);
|
|
628
|
-
break;
|
|
629
|
-
default:
|
|
630
|
-
passed = output.toLowerCase().includes(key.toLowerCase());
|
|
631
|
-
}
|
|
632
|
-
criteria[key] = { passed, reason: 'regex heuristic' };
|
|
633
|
-
}
|
|
634
|
-
|
|
635
|
-
const entries = Object.values(criteria);
|
|
636
|
-
const passedCount = entries.filter((c) => c.passed).length;
|
|
637
|
-
const passRate = entries.length > 0 ? passedCount / entries.length : 0;
|
|
638
|
-
|
|
639
|
-
return { passed: passRate >= 0.6, passRate, criteria };
|
|
640
|
-
}
|
|
641
|
-
|
|
642
|
-
/**
|
|
643
|
-
* Calculate weighted score using task weights and per-criterion pass rates
|
|
644
|
-
*/
|
|
645
|
-
function calculateScore(results) {
|
|
646
|
-
if (results.length === 0) return 0;
|
|
647
|
-
|
|
648
|
-
let weightedSum = 0;
|
|
649
|
-
let totalWeight = 0;
|
|
650
|
-
|
|
651
|
-
for (const r of results) {
|
|
652
|
-
const weight = r.weight || 1.0;
|
|
653
|
-
// Use passRate for partial credit when available (LLM judge),
|
|
654
|
-
// fall back to binary passed for regex judge
|
|
655
|
-
const score = r.passRate !== undefined ? r.passRate : r.passed ? 1.0 : 0.0;
|
|
656
|
-
weightedSum += score * weight;
|
|
657
|
-
totalWeight += weight;
|
|
658
|
-
}
|
|
659
|
-
|
|
660
|
-
return totalWeight > 0 ? weightedSum / totalWeight : 0;
|
|
661
|
-
}
|
|
662
|
-
|
|
663
|
-
/**
|
|
664
|
-
* Score all variants and select best
|
|
665
|
-
*/
|
|
666
|
-
async function scoreAndSelect() {
|
|
667
|
-
const state = getState();
|
|
668
|
-
const gen = state.currentGeneration + 1;
|
|
669
|
-
const genDir = getGenPath(gen);
|
|
670
|
-
|
|
671
|
-
if (!fs.existsSync(genDir)) {
|
|
672
|
-
console.error(`Generation ${gen} not found. Run 'mutate' first.`);
|
|
673
|
-
return;
|
|
674
|
-
}
|
|
675
|
-
|
|
676
|
-
const variants = fs
|
|
677
|
-
.readdirSync(genDir)
|
|
678
|
-
.filter((f) => f.endsWith('.md'))
|
|
679
|
-
.map((f) => f.replace('.md', ''));
|
|
680
|
-
|
|
681
|
-
console.log(`Scoring ${variants.length} variants in generation ${gen}...`);
|
|
682
|
-
|
|
683
|
-
const scores = [];
|
|
684
|
-
|
|
685
|
-
for (const variant of variants) {
|
|
686
|
-
const result = await runEval(variant);
|
|
687
|
-
if (result) scores.push(result);
|
|
688
|
-
}
|
|
689
|
-
|
|
690
|
-
// Sort by score
|
|
691
|
-
scores.sort((a, b) => b.score - a.score);
|
|
692
|
-
|
|
693
|
-
console.log('\nResults:');
|
|
694
|
-
scores.forEach((s, i) => {
|
|
695
|
-
const marker = i === 0 ? ' <-- BEST' : '';
|
|
696
|
-
console.log(
|
|
697
|
-
` ${i + 1}. ${s.variant}: ${(s.score * 100).toFixed(1)}%${marker}`
|
|
698
|
-
);
|
|
699
|
-
});
|
|
700
|
-
|
|
701
|
-
// Select best
|
|
702
|
-
const best = scores[0];
|
|
703
|
-
|
|
704
|
-
if (best.score > state.bestScore) {
|
|
705
|
-
state.currentGeneration = gen;
|
|
706
|
-
state.bestVariant = best.variant;
|
|
707
|
-
state.bestScore = best.score;
|
|
708
|
-
|
|
709
|
-
// Update symlink
|
|
710
|
-
const currentLink = path.join(GENERATIONS_DIR, 'current');
|
|
711
|
-
if (fs.existsSync(currentLink)) fs.unlinkSync(currentLink);
|
|
712
|
-
fs.symlinkSync(getGenPath(gen, best.variant), currentLink);
|
|
713
|
-
|
|
714
|
-
console.log(
|
|
715
|
-
`\nNew best: ${best.variant} (${(best.score * 100).toFixed(1)}%)`
|
|
716
|
-
);
|
|
717
|
-
console.log(
|
|
718
|
-
`Symlink updated: generations/current -> gen-${String(gen).padStart(3, '0')}/${best.variant}.md`
|
|
719
|
-
);
|
|
720
|
-
} else {
|
|
721
|
-
console.log(
|
|
722
|
-
`\nNo improvement over previous best (${(state.bestScore * 100).toFixed(1)}%)`
|
|
723
|
-
);
|
|
724
|
-
}
|
|
725
|
-
|
|
726
|
-
state.history.push({
|
|
727
|
-
generation: gen,
|
|
728
|
-
action: 'select',
|
|
729
|
-
scores: scores.map((s) => ({ variant: s.variant, score: s.score })),
|
|
730
|
-
best: best.variant,
|
|
731
|
-
timestamp: new Date().toISOString(),
|
|
732
|
-
});
|
|
733
|
-
|
|
734
|
-
saveState(state);
|
|
735
|
-
return best;
|
|
736
|
-
}
|
|
737
|
-
|
|
738
|
-
/**
|
|
739
|
-
* Full optimization loop
|
|
740
|
-
*/
|
|
741
|
-
async function run(generations = config.evolution.generations) {
|
|
742
|
-
console.log(`Starting GEPA optimization for ${generations} generations...\n`);
|
|
743
|
-
|
|
744
|
-
for (let i = 0; i < generations; i++) {
|
|
745
|
-
console.log(`\n${'='.repeat(60)}`);
|
|
746
|
-
console.log(`GENERATION ${i + 1}/${generations}`);
|
|
747
|
-
console.log(`${'='.repeat(60)}\n`);
|
|
748
|
-
|
|
749
|
-
await mutate();
|
|
750
|
-
const best = await scoreAndSelect();
|
|
751
|
-
|
|
752
|
-
if (best.score >= config.scoring.threshold) {
|
|
753
|
-
console.log(
|
|
754
|
-
`\nThreshold reached (${(config.scoring.threshold * 100).toFixed(0)}%)! Stopping early.`
|
|
755
|
-
);
|
|
756
|
-
break;
|
|
757
|
-
}
|
|
758
|
-
}
|
|
759
|
-
|
|
760
|
-
console.log('\n' + '='.repeat(60));
|
|
761
|
-
console.log('OPTIMIZATION COMPLETE');
|
|
762
|
-
console.log('='.repeat(60));
|
|
763
|
-
|
|
764
|
-
const state = getState();
|
|
765
|
-
console.log(`Best variant: ${state.bestVariant}`);
|
|
766
|
-
console.log(`Best score: ${(state.bestScore * 100).toFixed(1)}%`);
|
|
767
|
-
console.log(`Generations: ${state.currentGeneration}`);
|
|
768
|
-
console.log(`\nTo apply: cp generations/current /path/to/your/CLAUDE.md`);
|
|
769
|
-
}
|
|
770
|
-
|
|
771
|
-
/**
|
|
772
|
-
* Show status
|
|
773
|
-
*/
|
|
774
|
-
function status() {
|
|
775
|
-
const state = getState();
|
|
776
|
-
|
|
777
|
-
console.log('GEPA Status');
|
|
778
|
-
console.log('===========');
|
|
779
|
-
console.log(`Current generation: ${state.currentGeneration}`);
|
|
780
|
-
console.log(`Best variant: ${state.bestVariant}`);
|
|
781
|
-
console.log(`Best score: ${(state.bestScore * 100).toFixed(1)}%`);
|
|
782
|
-
console.log(`Target: ${state.targetPath}`);
|
|
783
|
-
console.log(`\nHistory:`);
|
|
784
|
-
|
|
785
|
-
state.history.slice(-10).forEach((h) => {
|
|
786
|
-
console.log(` [${h.timestamp}] ${h.action} (gen ${h.generation})`);
|
|
787
|
-
});
|
|
788
|
-
}
|
|
789
|
-
|
|
790
|
-
/**
|
|
791
|
-
* Diff two variants
|
|
792
|
-
*/
|
|
793
|
-
function diff(a, b) {
|
|
794
|
-
const state = getState();
|
|
795
|
-
const gen = state.currentGeneration;
|
|
796
|
-
|
|
797
|
-
const pathA = getGenPath(gen, a || 'baseline');
|
|
798
|
-
const pathB = getGenPath(gen, b || state.bestVariant);
|
|
799
|
-
|
|
800
|
-
if (!fs.existsSync(pathA) || !fs.existsSync(pathB)) {
|
|
801
|
-
console.error('Variant not found');
|
|
802
|
-
return;
|
|
803
|
-
}
|
|
804
|
-
|
|
805
|
-
try {
|
|
806
|
-
execSync(`diff -u ${pathA} ${pathB}`, { stdio: 'inherit' });
|
|
807
|
-
} catch (e) {
|
|
808
|
-
// diff returns non-zero when files differ
|
|
809
|
-
}
|
|
810
|
-
}
|
|
811
|
-
|
|
812
|
-
// CLI
|
|
813
|
-
const command = process.argv[2];
|
|
814
|
-
const arg1 = process.argv[3];
|
|
815
|
-
const arg2 = process.argv[4];
|
|
816
|
-
|
|
817
|
-
switch (command) {
|
|
818
|
-
case 'init':
|
|
819
|
-
init(arg1);
|
|
820
|
-
break;
|
|
821
|
-
case 'mutate':
|
|
822
|
-
mutate();
|
|
823
|
-
break;
|
|
824
|
-
case 'eval':
|
|
825
|
-
runEval(arg1 || 'baseline');
|
|
826
|
-
break;
|
|
827
|
-
case 'select':
|
|
828
|
-
case 'score':
|
|
829
|
-
scoreAndSelect();
|
|
830
|
-
break;
|
|
831
|
-
case 'run':
|
|
832
|
-
run(parseInt(arg1) || config.evolution.generations);
|
|
833
|
-
break;
|
|
834
|
-
case 'status':
|
|
835
|
-
status();
|
|
836
|
-
break;
|
|
837
|
-
case 'diff':
|
|
838
|
-
diff(arg1, arg2);
|
|
839
|
-
break;
|
|
840
|
-
default:
|
|
841
|
-
console.log(`
|
|
842
|
-
GEPA - Genetic Eval-driven Prompt Algorithm
|
|
843
|
-
|
|
844
|
-
Usage:
|
|
845
|
-
node optimize.js init [path] Initialize with CLAUDE.md
|
|
846
|
-
node optimize.js mutate Generate new variants
|
|
847
|
-
node optimize.js eval [variant] Run evals on variant
|
|
848
|
-
node optimize.js score Score all variants, select best
|
|
849
|
-
node optimize.js run [generations] Full optimization loop
|
|
850
|
-
node optimize.js status Show current status
|
|
851
|
-
node optimize.js diff [a] [b] Compare two variants
|
|
852
|
-
`);
|
|
853
|
-
}
|