@stackmemoryai/stackmemory 1.5.9 → 1.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/dist/src/cli/commands/orchestrate.js +506 -33
  2. package/dist/src/cli/commands/orchestrator.js +208 -31
  3. package/dist/src/daemon/daemon-config.js +2 -3
  4. package/dist/src/daemon/session-daemon.js +4 -5
  5. package/dist/src/daemon/unified-daemon.js +4 -5
  6. package/dist/src/features/sweep/sweep-server-manager.js +3 -6
  7. package/dist/src/hooks/daemon.js +5 -8
  8. package/dist/src/utils/process-cleanup.js +9 -0
  9. package/package.json +1 -2
  10. package/scripts/gepa/.before-optimize.md +0 -112
  11. package/scripts/gepa/README.md +0 -275
  12. package/scripts/gepa/config.json +0 -59
  13. package/scripts/gepa/evals/coding-tasks.jsonl +0 -5
  14. package/scripts/gepa/evals/fixtures/api-endpoint.ts +0 -31
  15. package/scripts/gepa/evals/fixtures/brittle-integration.ts +0 -38
  16. package/scripts/gepa/evals/fixtures/buggy-loop.js +0 -18
  17. package/scripts/gepa/evals/fixtures/callback-hell.js +0 -53
  18. package/scripts/gepa/evals/fixtures/fts5-triggers.sql +0 -23
  19. package/scripts/gepa/evals/fixtures/leaky-service.ts +0 -70
  20. package/scripts/gepa/evals/fixtures/mcp-dispatch-stub.ts +0 -39
  21. package/scripts/gepa/evals/fixtures/pr-diff.txt +0 -24
  22. package/scripts/gepa/evals/fixtures/unsafe-webhook.ts +0 -34
  23. package/scripts/gepa/evals/fixtures/unwrapped-db-op.ts +0 -42
  24. package/scripts/gepa/evals/stackmemory-tasks.jsonl +0 -8
  25. package/scripts/gepa/generations/gen-000/baseline.md +0 -112
  26. package/scripts/gepa/generations/gen-001/baseline.md +0 -112
  27. package/scripts/gepa/generations/gen-001/variant-a.md +0 -107
  28. package/scripts/gepa/generations/gen-001/variant-b.md +0 -216
  29. package/scripts/gepa/generations/gen-001/variant-c.md +0 -83
  30. package/scripts/gepa/generations/gen-001/variant-d.md +0 -90
  31. package/scripts/gepa/hooks/auto-optimize.js +0 -494
  32. package/scripts/gepa/hooks/eval-tracker.js +0 -203
  33. package/scripts/gepa/hooks/reflect.js +0 -350
  34. package/scripts/gepa/optimize.js +0 -853
  35. package/scripts/gepa/results/eval-1-baseline.json +0 -218
  36. package/scripts/gepa/results/eval-1-variant-a.json +0 -218
  37. package/scripts/gepa/results/eval-1-variant-b.json +0 -218
  38. package/scripts/gepa/results/eval-1-variant-c.json +0 -218
  39. package/scripts/gepa/results/eval-1-variant-d.json +0 -218
  40. package/scripts/gepa/state.json +0 -49
@@ -1,853 +0,0 @@
1
- #!/usr/bin/env node
2
- /**
3
- * GEPA Optimizer
4
- *
5
- * Genetic Eval-driven Prompt Algorithm for optimizing CLAUDE.md
6
- *
7
- * Usage:
8
- * node optimize.js init # Initialize with current CLAUDE.md
9
- * node optimize.js mutate # Generate new variants
10
- * node optimize.js eval [variant] # Run evals on variant(s)
11
- * node optimize.js score # Score all variants in generation
12
- * node optimize.js select # Select best, advance generation
13
- * node optimize.js run [generations] # Full optimization loop
14
- * node optimize.js status # Show current status
15
- * node optimize.js diff [a] [b] # Compare two variants
16
- */
17
-
18
- import fs from 'fs';
19
- import path from 'path';
20
- import { fileURLToPath } from 'url';
21
- import { spawn, execSync } from 'child_process';
22
-
23
- const __dirname = path.dirname(fileURLToPath(import.meta.url));
24
- const PROJECT_ROOT = path.resolve(__dirname, '../..');
25
-
26
- // Load .env from project root
27
- const envPath = path.join(PROJECT_ROOT, '.env');
28
- if (fs.existsSync(envPath)) {
29
- for (const line of fs.readFileSync(envPath, 'utf8').split('\n')) {
30
- const match = line.match(/^([A-Z_][A-Z0-9_]*)=(.*)$/);
31
- if (match && !process.env[match[1]]) {
32
- process.env[match[1]] = match[2].replace(/^["']|["']$/g, '');
33
- }
34
- }
35
- }
36
-
37
- // Configuration
38
- const CONFIG_PATH = path.join(__dirname, 'config.json');
39
- const config = JSON.parse(fs.readFileSync(CONFIG_PATH, 'utf8'));
40
-
41
- const GEPA_DIR = process.env.GEPA_DIR || __dirname;
42
- const GENERATIONS_DIR = path.join(GEPA_DIR, 'generations');
43
- const RESULTS_DIR = path.join(GEPA_DIR, 'results');
44
- const EVALS_DIR = path.join(GEPA_DIR, 'evals');
45
-
46
- // Ensure directories
47
- [GENERATIONS_DIR, RESULTS_DIR, EVALS_DIR].forEach((dir) => {
48
- if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
49
- });
50
-
51
- /**
52
- * State management
53
- */
54
- function getState() {
55
- const statePath = path.join(GEPA_DIR, 'state.json');
56
- if (fs.existsSync(statePath)) {
57
- return JSON.parse(fs.readFileSync(statePath, 'utf8'));
58
- }
59
- return {
60
- currentGeneration: 0,
61
- bestVariant: null,
62
- bestScore: 0,
63
- history: [],
64
- };
65
- }
66
-
67
- function saveState(state) {
68
- fs.writeFileSync(
69
- path.join(GEPA_DIR, 'state.json'),
70
- JSON.stringify(state, null, 2)
71
- );
72
- }
73
-
74
- /**
75
- * Get path for a generation/variant
76
- */
77
- function getGenPath(gen, variant = null) {
78
- const genDir = path.join(
79
- GENERATIONS_DIR,
80
- `gen-${String(gen).padStart(3, '0')}`
81
- );
82
- if (!fs.existsSync(genDir)) fs.mkdirSync(genDir, { recursive: true });
83
- return variant ? path.join(genDir, `${variant}.md`) : genDir;
84
- }
85
-
86
- /**
87
- * Initialize GEPA with current CLAUDE.md
88
- */
89
- async function init(targetPath) {
90
- const claudeMdPath = targetPath || path.join(process.cwd(), 'CLAUDE.md');
91
-
92
- if (!fs.existsSync(claudeMdPath)) {
93
- console.error(`Error: ${claudeMdPath} not found`);
94
- process.exit(1);
95
- }
96
-
97
- const content = fs.readFileSync(claudeMdPath, 'utf8');
98
- const genPath = getGenPath(0, 'baseline');
99
-
100
- fs.writeFileSync(genPath, content);
101
-
102
- const state = {
103
- currentGeneration: 0,
104
- bestVariant: 'baseline',
105
- bestScore: 0,
106
- targetPath: claudeMdPath,
107
- history: [
108
- {
109
- generation: 0,
110
- variant: 'baseline',
111
- action: 'init',
112
- timestamp: new Date().toISOString(),
113
- },
114
- ],
115
- };
116
-
117
- saveState(state);
118
- console.log(`Initialized GEPA with ${claudeMdPath}`);
119
- console.log(`Baseline saved to ${genPath}`);
120
- }
121
-
122
- /**
123
- * Generate mutations of the current best variant
124
- */
125
- async function mutate() {
126
- const state = getState();
127
- const nextGen = state.currentGeneration + 1;
128
- const currentBest = fs.readFileSync(
129
- getGenPath(state.currentGeneration, state.bestVariant),
130
- 'utf8'
131
- );
132
-
133
- console.log(
134
- `Generating ${config.evolution.populationSize} variants for generation ${nextGen}...`
135
- );
136
-
137
- const mutations = config.evolution.mutationStrategies;
138
- const variants = [];
139
-
140
- for (let i = 0; i < config.evolution.populationSize; i++) {
141
- const strategy = mutations[i % mutations.length];
142
- const variantName = `variant-${String.fromCharCode(97 + i)}`; // a, b, c, d...
143
-
144
- console.log(` Creating ${variantName} using strategy: ${strategy}`);
145
-
146
- const mutatedContent = await generateMutation(currentBest, strategy, state);
147
- const variantPath = getGenPath(nextGen, variantName);
148
-
149
- fs.writeFileSync(variantPath, mutatedContent);
150
- variants.push({ name: variantName, strategy, path: variantPath });
151
- }
152
-
153
- // Also copy baseline for comparison
154
- fs.writeFileSync(getGenPath(nextGen, 'baseline'), currentBest);
155
-
156
- state.history.push({
157
- generation: nextGen,
158
- action: 'mutate',
159
- variants: variants.map((v) => v.name),
160
- timestamp: new Date().toISOString(),
161
- });
162
- saveState(state);
163
-
164
- console.log(
165
- `\nGenerated ${variants.length} variants in gen-${String(nextGen).padStart(3, '0')}/`
166
- );
167
- return variants;
168
- }
169
-
170
- /**
171
- * Generate a mutation using AI
172
- */
173
- async function generateMutation(content, strategy, state) {
174
- const strategyPrompts = {
175
- rephrase: `Rephrase instructions for clarity without changing meaning. Make them more direct and actionable.`,
176
-
177
- add_examples: `Add concrete examples where instructions are abstract. Use <example> tags for code examples.`,
178
-
179
- remove_redundancy: `Remove redundant or repetitive instructions. Consolidate similar rules. Keep it DRY.`,
180
-
181
- restructure: `Reorganize sections for better flow. Group related instructions. Improve hierarchy.`,
182
-
183
- add_constraints: `Add specific constraints and guardrails based on common failure modes. Be precise about what NOT to do.`,
184
-
185
- simplify: `Simplify complex instructions. Break down multi-step rules. Use bullet points over paragraphs.`,
186
- };
187
-
188
- const prompt = `You are optimizing a CLAUDE.md system prompt for an AI coding agent.
189
-
190
- CURRENT PROMPT:
191
- \`\`\`markdown
192
- ${content}
193
- \`\`\`
194
-
195
- OPTIMIZATION STRATEGY: ${strategy}
196
- ${strategyPrompts[strategy]}
197
-
198
- EVALUATION FEEDBACK FROM PREVIOUS GENERATIONS:
199
- ${getRecentFeedback(state)}
200
-
201
- REFLECTION INSIGHTS (from failure pattern analysis):
202
- ${getReflectionInsights()}
203
-
204
- REQUIREMENTS:
205
- 1. Output ONLY the improved markdown content
206
- 2. Preserve all critical instructions and constraints
207
- 3. Keep the same overall structure unless restructuring
208
- 4. Do not add commentary or explanations
209
- 5. Target <8000 tokens total length
210
-
211
- OUTPUT THE IMPROVED CLAUDE.MD:`;
212
-
213
- // Use Claude to generate mutation
214
- const result = await callClaude(prompt);
215
- return result.trim();
216
- }
217
-
218
- /**
219
- * Get recent evaluation feedback for context
220
- */
221
- function getRecentFeedback(state) {
222
- const scoresPath = path.join(RESULTS_DIR, 'scores.jsonl');
223
- if (!fs.existsSync(scoresPath)) return 'No previous evaluations.';
224
-
225
- const lines = fs
226
- .readFileSync(scoresPath, 'utf8')
227
- .trim()
228
- .split('\n')
229
- .slice(-20);
230
- const scores = lines.map((l) => JSON.parse(l));
231
-
232
- const summary = scores.reduce((acc, s) => {
233
- if (!acc[s.variant]) acc[s.variant] = { total: 0, count: 0, errors: 0 };
234
- acc[s.variant].total += s.metrics?.successfulToolCalls || 0;
235
- acc[s.variant].count++;
236
- acc[s.variant].errors += s.metrics?.errorCount || 0;
237
- return acc;
238
- }, {});
239
-
240
- return Object.entries(summary)
241
- .map(
242
- ([v, s]) =>
243
- `${v}: ${s.count} sessions, ${s.errors} errors, avg success: ${(s.total / s.count).toFixed(1)}`
244
- )
245
- .join('\n');
246
- }
247
-
248
- /**
249
- * Load most recent reflection insights for mutation context
250
- */
251
- function getReflectionInsights() {
252
- const reflectionFiles = fs.existsSync(RESULTS_DIR)
253
- ? fs
254
- .readdirSync(RESULTS_DIR)
255
- .filter((f) => f.startsWith('reflection-') && f.endsWith('.json'))
256
- : [];
257
-
258
- if (reflectionFiles.length === 0) return 'No reflection data yet.';
259
-
260
- // Pick the most recent reflection file
261
- reflectionFiles.sort().reverse();
262
- const latest = JSON.parse(
263
- fs.readFileSync(path.join(RESULTS_DIR, reflectionFiles[0]), 'utf8')
264
- );
265
-
266
- const insights = latest.insights;
267
- if (!insights) return 'No reflection insights available.';
268
-
269
- const parts = [];
270
-
271
- if (insights.failureModes?.length) {
272
- parts.push(`Failure modes: ${insights.failureModes.join('; ')}`);
273
- }
274
- if (insights.missingInstructions?.length) {
275
- parts.push(
276
- `Missing instructions: ${insights.missingInstructions.join('; ')}`
277
- );
278
- }
279
- if (insights.unclearInstructions?.length) {
280
- parts.push(
281
- `Unclear instructions: ${insights.unclearInstructions.join('; ')}`
282
- );
283
- }
284
- if (insights.priorityMutations?.length) {
285
- parts.push(
286
- `Priority changes:\n${insights.priorityMutations
287
- .map((m) => ` - [${m.type}] ${m.section}: ${m.change}`)
288
- .join('\n')}`
289
- );
290
- }
291
-
292
- return parts.join('\n') || 'No actionable insights.';
293
- }
294
-
295
- /**
296
- * Call Claude CLI via spawn (stdin pipe, no shell interpolation)
297
- */
298
- function spawnClaude(prompt, { cwd, timeoutMs } = {}) {
299
- return new Promise((resolve, reject) => {
300
- const args = ['--print'];
301
- const child = spawn('claude', args, {
302
- cwd,
303
- stdio: ['pipe', 'pipe', 'pipe'],
304
- env: { ...process.env },
305
- });
306
-
307
- let stdout = '';
308
- let stderr = '';
309
- let killed = false;
310
-
311
- const timer = timeoutMs
312
- ? setTimeout(() => {
313
- killed = true;
314
- child.kill('SIGTERM');
315
- }, timeoutMs)
316
- : null;
317
-
318
- child.stdout.on('data', (d) => (stdout += d));
319
- child.stderr.on('data', (d) => (stderr += d));
320
-
321
- child.on('close', (code) => {
322
- if (timer) clearTimeout(timer);
323
- if (killed)
324
- return reject(new Error(`claude timed out after ${timeoutMs}ms`));
325
- if (code !== 0 && !stdout)
326
- return reject(new Error(stderr || `claude exited ${code}`));
327
- resolve(stdout);
328
- });
329
-
330
- child.on('error', (err) => {
331
- if (timer) clearTimeout(timer);
332
- reject(err);
333
- });
334
-
335
- child.stdin.write(prompt);
336
- child.stdin.end();
337
- });
338
- }
339
-
340
- /**
341
- * Call Claude API for mutation generation
342
- */
343
- async function callClaude(prompt) {
344
- // Try using claude CLI first (stdin pipe, no shell injection)
345
- try {
346
- return await spawnClaude(prompt);
347
- } catch (e) {
348
- // Fallback to API
349
- const apiKey = process.env.ANTHROPIC_API_KEY;
350
- if (!apiKey) {
351
- console.error('Error: ANTHROPIC_API_KEY not set and claude CLI failed');
352
- process.exit(1);
353
- }
354
-
355
- const response = await fetch('https://api.anthropic.com/v1/messages', {
356
- method: 'POST',
357
- headers: {
358
- 'Content-Type': 'application/json',
359
- 'x-api-key': apiKey,
360
- 'anthropic-version': '2023-06-01',
361
- },
362
- body: JSON.stringify({
363
- model: 'claude-sonnet-4-20250514',
364
- max_tokens: 8000,
365
- messages: [{ role: 'user', content: prompt }],
366
- }),
367
- });
368
-
369
- const data = await response.json();
370
- return data.content[0].text;
371
- }
372
- }
373
-
374
- /**
375
- * Run evaluations on a variant
376
- */
377
- async function runEval(variantName) {
378
- const state = getState();
379
- const gen = state.currentGeneration + 1; // Eval next gen variants
380
- const variantPath = getGenPath(gen, variantName);
381
-
382
- if (!fs.existsSync(variantPath)) {
383
- console.error(`Variant not found: ${variantPath}`);
384
- return null;
385
- }
386
-
387
- console.log(`Running evals on ${variantName}...`);
388
-
389
- // Load eval tasks
390
- const evalFiles = fs
391
- .readdirSync(EVALS_DIR)
392
- .filter((f) => f.endsWith('.jsonl'));
393
- const tasks = evalFiles.flatMap((f) =>
394
- fs
395
- .readFileSync(path.join(EVALS_DIR, f), 'utf8')
396
- .trim()
397
- .split('\n')
398
- .map((l) => JSON.parse(l))
399
- );
400
-
401
- console.log(` Found ${tasks.length} eval tasks`);
402
-
403
- // Set environment for tracking
404
- process.env.GEPA_VARIANT = variantName;
405
- process.env.GEPA_GENERATION = String(gen);
406
-
407
- const results = [];
408
-
409
- for (const task of tasks.slice(0, config.evals.minSamplesPerVariant)) {
410
- console.log(` Running: ${task.name}`);
411
-
412
- const result = await runSingleEval(task, variantPath);
413
- results.push({
414
- taskId: task.id,
415
- taskName: task.name,
416
- weight: task.weight || 1.0,
417
- ...result,
418
- });
419
- }
420
-
421
- // Save results
422
- const resultsPath = path.join(RESULTS_DIR, `eval-${gen}-${variantName}.json`);
423
- fs.writeFileSync(
424
- resultsPath,
425
- JSON.stringify({ variant: variantName, generation: gen, results }, null, 2)
426
- );
427
-
428
- // Calculate aggregate score
429
- const score = calculateScore(results);
430
- console.log(` Score: ${(score * 100).toFixed(1)}%`);
431
-
432
- return { variant: variantName, score, results };
433
- }
434
-
435
- /**
436
- * Run a single eval task
437
- */
438
- async function runSingleEval(task, variantPath) {
439
- const startTime = Date.now();
440
- let tempDir;
441
-
442
- try {
443
- // Create temp project with variant as CLAUDE.md
444
- tempDir = fs.mkdtempSync('/tmp/gepa-eval-');
445
- fs.copyFileSync(variantPath, path.join(tempDir, 'CLAUDE.md'));
446
-
447
- // Copy fixture if needed
448
- if (task.input_file) {
449
- const fixturePath = path.join(EVALS_DIR, task.input_file);
450
- if (fs.existsSync(fixturePath)) {
451
- fs.copyFileSync(
452
- fixturePath,
453
- path.join(tempDir, path.basename(task.input_file))
454
- );
455
- }
456
- }
457
-
458
- // Run claude via spawn with Node-native timeout (no GNU timeout needed)
459
- const result = await spawnClaude(task.prompt, {
460
- cwd: tempDir,
461
- timeoutMs: config.evals.timeout,
462
- });
463
-
464
- // Evaluate result against expected outcomes (LLM judge with regex fallback)
465
- const evaluation = await evaluateExpectations(result, task.expected, task);
466
-
467
- return {
468
- passed: evaluation.passed,
469
- passRate: evaluation.passRate,
470
- criteria: evaluation.criteria,
471
- judgeMode: evaluation.judgeMode,
472
- duration: Date.now() - startTime,
473
- output: result.slice(0, 2000),
474
- };
475
- } catch (error) {
476
- return {
477
- passed: false,
478
- duration: Date.now() - startTime,
479
- error: error.message,
480
- };
481
- } finally {
482
- if (tempDir) fs.rmSync(tempDir, { recursive: true, force: true });
483
- }
484
- }
485
-
486
- /**
487
- * Evaluate output against expectations using LLM judge (regex fallback)
488
- */
489
- async function evaluateExpectations(output, expected, task) {
490
- if (!expected)
491
- return { passed: true, passRate: 1.0, criteria: {}, judgeMode: 'skip' };
492
-
493
- // Try LLM judge first
494
- try {
495
- const result = await llmJudge(output, expected, task);
496
- return { ...result, judgeMode: 'llm' };
497
- } catch (e) {
498
- console.log(` LLM judge failed (${e.message}), using regex fallback`);
499
- const result = regexJudge(output, expected);
500
- return { ...result, judgeMode: 'regex' };
501
- }
502
- }
503
-
504
- /**
505
- * LLM-as-judge — uses a fast model to evaluate output against criteria
506
- */
507
- async function llmJudge(output, expected, task) {
508
- const criteriaList = Object.entries(expected)
509
- .map(
510
- ([key, value]) =>
511
- `- ${key}: ${typeof value === 'string' ? value : 'should be ' + value}`
512
- )
513
- .join('\n');
514
-
515
- const judgePrompt = `You are a strict code evaluation judge. Evaluate whether the AI output satisfies each criterion.
516
-
517
- TASK GIVEN TO AI:
518
- ${task.prompt}
519
-
520
- AI OUTPUT:
521
- \`\`\`
522
- ${output.slice(0, 6000)}
523
- \`\`\`
524
-
525
- CRITERIA TO EVALUATE:
526
- ${criteriaList}
527
-
528
- For each criterion, determine if the output genuinely satisfies it. Be strict:
529
- - "has_function" means a real, working function is defined (not just mentioned)
530
- - "bug_fixed" means the actual bug is corrected (not just discussed)
531
- - "handles_edge_cases" means edge cases are actually handled in code
532
- - "explains_fix" means there's a clear explanation of what was wrong and why
533
-
534
- Respond with ONLY this JSON (no markdown fences):
535
- {
536
- "criteria": {
537
- "criterion_name": {"passed": true, "reason": "brief explanation"},
538
- "criterion_name": {"passed": false, "reason": "brief explanation"}
539
- }
540
- }`;
541
-
542
- const judgeModel = config.judge?.model || 'claude-haiku-4-5-20251001';
543
- const raw = await callJudge(judgePrompt, judgeModel);
544
-
545
- // Extract JSON from response
546
- const jsonMatch = raw.match(/\{[\s\S]*\}/);
547
- if (!jsonMatch) throw new Error('No JSON in judge response');
548
-
549
- const parsed = JSON.parse(jsonMatch[0]);
550
- const criteria = parsed.criteria || {};
551
- const entries = Object.values(criteria);
552
- const passedCount = entries.filter((c) => c.passed).length;
553
- const passRate = entries.length > 0 ? passedCount / entries.length : 0;
554
-
555
- return {
556
- passed: passRate >= 0.6,
557
- passRate,
558
- criteria,
559
- };
560
- }
561
-
562
- /**
563
- * Call judge model via Anthropic API (fast, cheap model for evaluation)
564
- */
565
- async function callJudge(prompt, model) {
566
- const apiKey = process.env.ANTHROPIC_API_KEY;
567
-
568
- if (apiKey) {
569
- const response = await fetch('https://api.anthropic.com/v1/messages', {
570
- method: 'POST',
571
- headers: {
572
- 'Content-Type': 'application/json',
573
- 'x-api-key': apiKey,
574
- 'anthropic-version': '2023-06-01',
575
- },
576
- body: JSON.stringify({
577
- model,
578
- max_tokens: 2000,
579
- messages: [{ role: 'user', content: prompt }],
580
- }),
581
- });
582
-
583
- if (!response.ok) {
584
- throw new Error(`Judge API error: ${response.status}`);
585
- }
586
-
587
- const data = await response.json();
588
- return data.content[0].text;
589
- }
590
-
591
- // Fallback to CLI
592
- return await spawnClaude(prompt, { timeoutMs: 30000 });
593
- }
594
-
595
- /**
596
- * Regex fallback judge (used when LLM judge is unavailable)
597
- */
598
- function regexJudge(output, expected) {
599
- const criteria = {};
600
-
601
- for (const [key, value] of Object.entries(expected)) {
602
- let passed = false;
603
- switch (key) {
604
- case 'has_function':
605
- passed =
606
- /function\s+\w+|const\s+\w+\s*=\s*(\([^)]*\)|async)?\s*(=>|\{)/.test(
607
- output
608
- );
609
- break;
610
- case 'handles_edge_cases':
611
- passed = /if\s*\(|edge|empty|null|undefined|\.length/.test(output);
612
- break;
613
- case 'uses_async':
614
- passed = /async|await|Promise/.test(output);
615
- break;
616
- case 'no_nested_callbacks':
617
- passed = !/callback\s*\(\s*function|\.then\s*\([^)]*\.then/.test(
618
- output
619
- );
620
- break;
621
- case 'bug_fixed':
622
- passed = /fix|correct|change|update/i.test(output);
623
- break;
624
- case 'explains_fix':
625
- passed =
626
- output.length > 200 &&
627
- /because|since|the issue|the problem/i.test(output);
628
- break;
629
- default:
630
- passed = output.toLowerCase().includes(key.toLowerCase());
631
- }
632
- criteria[key] = { passed, reason: 'regex heuristic' };
633
- }
634
-
635
- const entries = Object.values(criteria);
636
- const passedCount = entries.filter((c) => c.passed).length;
637
- const passRate = entries.length > 0 ? passedCount / entries.length : 0;
638
-
639
- return { passed: passRate >= 0.6, passRate, criteria };
640
- }
641
-
642
- /**
643
- * Calculate weighted score using task weights and per-criterion pass rates
644
- */
645
- function calculateScore(results) {
646
- if (results.length === 0) return 0;
647
-
648
- let weightedSum = 0;
649
- let totalWeight = 0;
650
-
651
- for (const r of results) {
652
- const weight = r.weight || 1.0;
653
- // Use passRate for partial credit when available (LLM judge),
654
- // fall back to binary passed for regex judge
655
- const score = r.passRate !== undefined ? r.passRate : r.passed ? 1.0 : 0.0;
656
- weightedSum += score * weight;
657
- totalWeight += weight;
658
- }
659
-
660
- return totalWeight > 0 ? weightedSum / totalWeight : 0;
661
- }
662
-
663
- /**
664
- * Score all variants and select best
665
- */
666
- async function scoreAndSelect() {
667
- const state = getState();
668
- const gen = state.currentGeneration + 1;
669
- const genDir = getGenPath(gen);
670
-
671
- if (!fs.existsSync(genDir)) {
672
- console.error(`Generation ${gen} not found. Run 'mutate' first.`);
673
- return;
674
- }
675
-
676
- const variants = fs
677
- .readdirSync(genDir)
678
- .filter((f) => f.endsWith('.md'))
679
- .map((f) => f.replace('.md', ''));
680
-
681
- console.log(`Scoring ${variants.length} variants in generation ${gen}...`);
682
-
683
- const scores = [];
684
-
685
- for (const variant of variants) {
686
- const result = await runEval(variant);
687
- if (result) scores.push(result);
688
- }
689
-
690
- // Sort by score
691
- scores.sort((a, b) => b.score - a.score);
692
-
693
- console.log('\nResults:');
694
- scores.forEach((s, i) => {
695
- const marker = i === 0 ? ' <-- BEST' : '';
696
- console.log(
697
- ` ${i + 1}. ${s.variant}: ${(s.score * 100).toFixed(1)}%${marker}`
698
- );
699
- });
700
-
701
- // Select best
702
- const best = scores[0];
703
-
704
- if (best.score > state.bestScore) {
705
- state.currentGeneration = gen;
706
- state.bestVariant = best.variant;
707
- state.bestScore = best.score;
708
-
709
- // Update symlink
710
- const currentLink = path.join(GENERATIONS_DIR, 'current');
711
- if (fs.existsSync(currentLink)) fs.unlinkSync(currentLink);
712
- fs.symlinkSync(getGenPath(gen, best.variant), currentLink);
713
-
714
- console.log(
715
- `\nNew best: ${best.variant} (${(best.score * 100).toFixed(1)}%)`
716
- );
717
- console.log(
718
- `Symlink updated: generations/current -> gen-${String(gen).padStart(3, '0')}/${best.variant}.md`
719
- );
720
- } else {
721
- console.log(
722
- `\nNo improvement over previous best (${(state.bestScore * 100).toFixed(1)}%)`
723
- );
724
- }
725
-
726
- state.history.push({
727
- generation: gen,
728
- action: 'select',
729
- scores: scores.map((s) => ({ variant: s.variant, score: s.score })),
730
- best: best.variant,
731
- timestamp: new Date().toISOString(),
732
- });
733
-
734
- saveState(state);
735
- return best;
736
- }
737
-
738
- /**
739
- * Full optimization loop
740
- */
741
- async function run(generations = config.evolution.generations) {
742
- console.log(`Starting GEPA optimization for ${generations} generations...\n`);
743
-
744
- for (let i = 0; i < generations; i++) {
745
- console.log(`\n${'='.repeat(60)}`);
746
- console.log(`GENERATION ${i + 1}/${generations}`);
747
- console.log(`${'='.repeat(60)}\n`);
748
-
749
- await mutate();
750
- const best = await scoreAndSelect();
751
-
752
- if (best.score >= config.scoring.threshold) {
753
- console.log(
754
- `\nThreshold reached (${(config.scoring.threshold * 100).toFixed(0)}%)! Stopping early.`
755
- );
756
- break;
757
- }
758
- }
759
-
760
- console.log('\n' + '='.repeat(60));
761
- console.log('OPTIMIZATION COMPLETE');
762
- console.log('='.repeat(60));
763
-
764
- const state = getState();
765
- console.log(`Best variant: ${state.bestVariant}`);
766
- console.log(`Best score: ${(state.bestScore * 100).toFixed(1)}%`);
767
- console.log(`Generations: ${state.currentGeneration}`);
768
- console.log(`\nTo apply: cp generations/current /path/to/your/CLAUDE.md`);
769
- }
770
-
771
- /**
772
- * Show status
773
- */
774
- function status() {
775
- const state = getState();
776
-
777
- console.log('GEPA Status');
778
- console.log('===========');
779
- console.log(`Current generation: ${state.currentGeneration}`);
780
- console.log(`Best variant: ${state.bestVariant}`);
781
- console.log(`Best score: ${(state.bestScore * 100).toFixed(1)}%`);
782
- console.log(`Target: ${state.targetPath}`);
783
- console.log(`\nHistory:`);
784
-
785
- state.history.slice(-10).forEach((h) => {
786
- console.log(` [${h.timestamp}] ${h.action} (gen ${h.generation})`);
787
- });
788
- }
789
-
790
- /**
791
- * Diff two variants
792
- */
793
- function diff(a, b) {
794
- const state = getState();
795
- const gen = state.currentGeneration;
796
-
797
- const pathA = getGenPath(gen, a || 'baseline');
798
- const pathB = getGenPath(gen, b || state.bestVariant);
799
-
800
- if (!fs.existsSync(pathA) || !fs.existsSync(pathB)) {
801
- console.error('Variant not found');
802
- return;
803
- }
804
-
805
- try {
806
- execSync(`diff -u ${pathA} ${pathB}`, { stdio: 'inherit' });
807
- } catch (e) {
808
- // diff returns non-zero when files differ
809
- }
810
- }
811
-
812
- // CLI
813
- const command = process.argv[2];
814
- const arg1 = process.argv[3];
815
- const arg2 = process.argv[4];
816
-
817
- switch (command) {
818
- case 'init':
819
- init(arg1);
820
- break;
821
- case 'mutate':
822
- mutate();
823
- break;
824
- case 'eval':
825
- runEval(arg1 || 'baseline');
826
- break;
827
- case 'select':
828
- case 'score':
829
- scoreAndSelect();
830
- break;
831
- case 'run':
832
- run(parseInt(arg1) || config.evolution.generations);
833
- break;
834
- case 'status':
835
- status();
836
- break;
837
- case 'diff':
838
- diff(arg1, arg2);
839
- break;
840
- default:
841
- console.log(`
842
- GEPA - Genetic Eval-driven Prompt Algorithm
843
-
844
- Usage:
845
- node optimize.js init [path] Initialize with CLAUDE.md
846
- node optimize.js mutate Generate new variants
847
- node optimize.js eval [variant] Run evals on variant
848
- node optimize.js score Score all variants, select best
849
- node optimize.js run [generations] Full optimization loop
850
- node optimize.js status Show current status
851
- node optimize.js diff [a] [b] Compare two variants
852
- `);
853
- }