@stackmemoryai/stackmemory 1.6.0 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cli/commands/orchestrate.js +249 -2
- package/dist/src/cli/commands/orchestrator.js +206 -23
- package/package.json +1 -2
- package/scripts/gepa/.before-optimize.md +0 -112
- package/scripts/gepa/README.md +0 -275
- package/scripts/gepa/config.json +0 -59
- package/scripts/gepa/evals/coding-tasks.jsonl +0 -5
- package/scripts/gepa/evals/fixtures/api-endpoint.ts +0 -31
- package/scripts/gepa/evals/fixtures/brittle-integration.ts +0 -38
- package/scripts/gepa/evals/fixtures/buggy-loop.js +0 -18
- package/scripts/gepa/evals/fixtures/callback-hell.js +0 -53
- package/scripts/gepa/evals/fixtures/fts5-triggers.sql +0 -23
- package/scripts/gepa/evals/fixtures/leaky-service.ts +0 -70
- package/scripts/gepa/evals/fixtures/mcp-dispatch-stub.ts +0 -39
- package/scripts/gepa/evals/fixtures/pr-diff.txt +0 -24
- package/scripts/gepa/evals/fixtures/unsafe-webhook.ts +0 -34
- package/scripts/gepa/evals/fixtures/unwrapped-db-op.ts +0 -42
- package/scripts/gepa/evals/stackmemory-tasks.jsonl +0 -8
- package/scripts/gepa/generations/gen-000/baseline.md +0 -112
- package/scripts/gepa/generations/gen-001/baseline.md +0 -112
- package/scripts/gepa/generations/gen-001/variant-a.md +0 -107
- package/scripts/gepa/generations/gen-001/variant-b.md +0 -216
- package/scripts/gepa/generations/gen-001/variant-c.md +0 -83
- package/scripts/gepa/generations/gen-001/variant-d.md +0 -90
- package/scripts/gepa/hooks/auto-optimize.js +0 -494
- package/scripts/gepa/hooks/eval-tracker.js +0 -203
- package/scripts/gepa/hooks/reflect.js +0 -350
- package/scripts/gepa/optimize.js +0 -853
- package/scripts/gepa/results/eval-1-baseline.json +0 -218
- package/scripts/gepa/results/eval-1-variant-a.json +0 -218
- package/scripts/gepa/results/eval-1-variant-b.json +0 -218
- package/scripts/gepa/results/eval-1-variant-c.json +0 -218
- package/scripts/gepa/results/eval-1-variant-d.json +0 -218
- package/scripts/gepa/state.json +0 -49
|
@@ -1,494 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
/**
|
|
3
|
-
* GEPA Auto-Optimizer
|
|
4
|
-
*
|
|
5
|
-
* Watches CLAUDE.md for changes and automatically runs optimization.
|
|
6
|
-
* Shows before/after comparison with metrics.
|
|
7
|
-
*
|
|
8
|
-
* Usage:
|
|
9
|
-
* node auto-optimize.js watch [path] # Watch and auto-optimize
|
|
10
|
-
* node auto-optimize.js compare [a] [b] # Compare two versions
|
|
11
|
-
* node auto-optimize.js report # Show optimization report
|
|
12
|
-
*/
|
|
13
|
-
|
|
14
|
-
import fs from 'fs';
|
|
15
|
-
import path from 'path';
|
|
16
|
-
import { fileURLToPath } from 'url';
|
|
17
|
-
import { execSync, spawn } from 'child_process';
|
|
18
|
-
import crypto from 'crypto';
|
|
19
|
-
|
|
20
|
-
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
21
|
-
const GEPA_DIR = path.join(__dirname, '..');
|
|
22
|
-
const GENERATIONS_DIR = path.join(GEPA_DIR, 'generations');
|
|
23
|
-
const RESULTS_DIR = path.join(GEPA_DIR, 'results');
|
|
24
|
-
|
|
25
|
-
// ANSI colors
|
|
26
|
-
const c = {
|
|
27
|
-
reset: '\x1b[0m',
|
|
28
|
-
bold: '\x1b[1m',
|
|
29
|
-
dim: '\x1b[2m',
|
|
30
|
-
red: '\x1b[31m',
|
|
31
|
-
green: '\x1b[32m',
|
|
32
|
-
yellow: '\x1b[33m',
|
|
33
|
-
blue: '\x1b[34m',
|
|
34
|
-
magenta: '\x1b[35m',
|
|
35
|
-
cyan: '\x1b[36m',
|
|
36
|
-
bgGreen: '\x1b[42m',
|
|
37
|
-
bgRed: '\x1b[41m',
|
|
38
|
-
bgYellow: '\x1b[43m',
|
|
39
|
-
};
|
|
40
|
-
|
|
41
|
-
/**
|
|
42
|
-
* Calculate content hash for change detection
|
|
43
|
-
*/
|
|
44
|
-
function hashContent(content) {
|
|
45
|
-
return crypto.createHash('md5').update(content).digest('hex').slice(0, 8);
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
/**
|
|
49
|
-
* Analyze markdown structure
|
|
50
|
-
*/
|
|
51
|
-
function analyzeMarkdown(content) {
|
|
52
|
-
const lines = content.split('\n');
|
|
53
|
-
|
|
54
|
-
return {
|
|
55
|
-
totalLines: lines.length,
|
|
56
|
-
totalChars: content.length,
|
|
57
|
-
estimatedTokens: Math.ceil(content.length / 4),
|
|
58
|
-
|
|
59
|
-
// Structure
|
|
60
|
-
h1Count: (content.match(/^# /gm) || []).length,
|
|
61
|
-
h2Count: (content.match(/^## /gm) || []).length,
|
|
62
|
-
h3Count: (content.match(/^### /gm) || []).length,
|
|
63
|
-
|
|
64
|
-
// Content types
|
|
65
|
-
codeBlocks: (content.match(/```/g) || []).length / 2,
|
|
66
|
-
bulletPoints: (content.match(/^[-*] /gm) || []).length,
|
|
67
|
-
numberedLists: (content.match(/^\d+\. /gm) || []).length,
|
|
68
|
-
|
|
69
|
-
// Keywords (instruction density)
|
|
70
|
-
mustCount: (content.match(/\bMUST\b/gi) || []).length,
|
|
71
|
-
neverCount: (content.match(/\bNEVER\b/gi) || []).length,
|
|
72
|
-
alwaysCount: (content.match(/\bALWAYS\b/gi) || []).length,
|
|
73
|
-
importantCount: (content.match(/\bIMPORTANT\b/gi) || []).length,
|
|
74
|
-
|
|
75
|
-
// Sections
|
|
76
|
-
sections: [...content.matchAll(/^##+ (.+)$/gm)].map((m) => m[1]),
|
|
77
|
-
};
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
/**
|
|
81
|
-
* Format comparison table
|
|
82
|
-
*/
|
|
83
|
-
function formatComparison(before, after, label = 'Metric') {
|
|
84
|
-
const metrics = [
|
|
85
|
-
['Lines', before.totalLines, after.totalLines],
|
|
86
|
-
['Characters', before.totalChars, after.totalChars],
|
|
87
|
-
['Est. Tokens', before.estimatedTokens, after.estimatedTokens],
|
|
88
|
-
['H2 Sections', before.h2Count, after.h2Count],
|
|
89
|
-
['Code Blocks', before.codeBlocks, after.codeBlocks],
|
|
90
|
-
['Bullet Points', before.bulletPoints, after.bulletPoints],
|
|
91
|
-
['MUST rules', before.mustCount, after.mustCount],
|
|
92
|
-
['NEVER rules', before.neverCount, after.neverCount],
|
|
93
|
-
['ALWAYS rules', before.alwaysCount, after.alwaysCount],
|
|
94
|
-
];
|
|
95
|
-
|
|
96
|
-
console.log(
|
|
97
|
-
`\n${c.bold}${c.cyan}╔════════════════════════════════════════════════════════════╗${c.reset}`
|
|
98
|
-
);
|
|
99
|
-
console.log(
|
|
100
|
-
`${c.bold}${c.cyan}║${c.reset} ${c.bold}BEFORE / AFTER COMPARISON${c.reset} ${c.cyan}║${c.reset}`
|
|
101
|
-
);
|
|
102
|
-
console.log(
|
|
103
|
-
`${c.cyan}╠════════════════════════════════════════════════════════════╣${c.reset}`
|
|
104
|
-
);
|
|
105
|
-
console.log(
|
|
106
|
-
`${c.cyan}║${c.reset} ${c.dim}Metric${c.reset} ${c.dim}Before${c.reset} ${c.dim}After${c.reset} ${c.dim}Change${c.reset} ${c.cyan}║${c.reset}`
|
|
107
|
-
);
|
|
108
|
-
console.log(
|
|
109
|
-
`${c.cyan}╠════════════════════════════════════════════════════════════╣${c.reset}`
|
|
110
|
-
);
|
|
111
|
-
|
|
112
|
-
metrics.forEach(([name, b, a]) => {
|
|
113
|
-
const diff = a - b;
|
|
114
|
-
const pct = b > 0 ? ((diff / b) * 100).toFixed(0) : '∞';
|
|
115
|
-
const sign = diff > 0 ? '+' : '';
|
|
116
|
-
const color = diff > 0 ? c.green : diff < 0 ? c.red : c.dim;
|
|
117
|
-
|
|
118
|
-
const nameStr = name.padEnd(18);
|
|
119
|
-
const beforeStr = String(b).padStart(8);
|
|
120
|
-
const afterStr = String(a).padStart(8);
|
|
121
|
-
const changeStr = `${sign}${diff} (${sign}${pct}%)`.padStart(12);
|
|
122
|
-
|
|
123
|
-
console.log(
|
|
124
|
-
`${c.cyan}║${c.reset} ${nameStr} ${beforeStr} ${c.bold}${afterStr}${c.reset} ${color}${changeStr}${c.reset} ${c.cyan}║${c.reset}`
|
|
125
|
-
);
|
|
126
|
-
});
|
|
127
|
-
|
|
128
|
-
console.log(
|
|
129
|
-
`${c.cyan}╚════════════════════════════════════════════════════════════╝${c.reset}`
|
|
130
|
-
);
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
/**
|
|
134
|
-
* Show section diff
|
|
135
|
-
*/
|
|
136
|
-
function showSectionDiff(before, after) {
|
|
137
|
-
const beforeSections = new Set(before.sections);
|
|
138
|
-
const afterSections = new Set(after.sections);
|
|
139
|
-
|
|
140
|
-
const added = after.sections.filter((s) => !beforeSections.has(s));
|
|
141
|
-
const removed = before.sections.filter((s) => !afterSections.has(s));
|
|
142
|
-
const kept = before.sections.filter((s) => afterSections.has(s));
|
|
143
|
-
|
|
144
|
-
if (added.length || removed.length) {
|
|
145
|
-
console.log(`\n${c.bold}Section Changes:${c.reset}`);
|
|
146
|
-
|
|
147
|
-
if (removed.length) {
|
|
148
|
-
console.log(`${c.red} Removed:${c.reset}`);
|
|
149
|
-
removed.forEach((s) => console.log(`${c.red} - ${s}${c.reset}`));
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
if (added.length) {
|
|
153
|
-
console.log(`${c.green} Added:${c.reset}`);
|
|
154
|
-
added.forEach((s) => console.log(`${c.green} + ${s}${c.reset}`));
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
/**
|
|
160
|
-
* Show inline diff (simplified)
|
|
161
|
-
*/
|
|
162
|
-
function showInlineDiff(beforeContent, afterContent) {
|
|
163
|
-
const beforeLines = beforeContent.split('\n');
|
|
164
|
-
const afterLines = afterContent.split('\n');
|
|
165
|
-
|
|
166
|
-
console.log(`\n${c.bold}Key Changes (first 20 diffs):${c.reset}`);
|
|
167
|
-
console.log(c.dim + '─'.repeat(60) + c.reset);
|
|
168
|
-
|
|
169
|
-
let diffCount = 0;
|
|
170
|
-
const maxDiffs = 20;
|
|
171
|
-
|
|
172
|
-
// Simple line-by-line diff
|
|
173
|
-
const maxLen = Math.max(beforeLines.length, afterLines.length);
|
|
174
|
-
|
|
175
|
-
for (let i = 0; i < maxLen && diffCount < maxDiffs; i++) {
|
|
176
|
-
const b = beforeLines[i] || '';
|
|
177
|
-
const a = afterLines[i] || '';
|
|
178
|
-
|
|
179
|
-
if (b !== a) {
|
|
180
|
-
if (b && !a) {
|
|
181
|
-
console.log(
|
|
182
|
-
`${c.red}- L${i + 1}: ${b.slice(0, 70)}${b.length > 70 ? '...' : ''}${c.reset}`
|
|
183
|
-
);
|
|
184
|
-
} else if (!b && a) {
|
|
185
|
-
console.log(
|
|
186
|
-
`${c.green}+ L${i + 1}: ${a.slice(0, 70)}${a.length > 70 ? '...' : ''}${c.reset}`
|
|
187
|
-
);
|
|
188
|
-
} else if (b.trim() !== a.trim()) {
|
|
189
|
-
console.log(`${c.yellow}~ L${i + 1}:${c.reset}`);
|
|
190
|
-
console.log(
|
|
191
|
-
`${c.red} - ${b.slice(0, 60)}${b.length > 60 ? '...' : ''}${c.reset}`
|
|
192
|
-
);
|
|
193
|
-
console.log(
|
|
194
|
-
`${c.green} + ${a.slice(0, 60)}${a.length > 60 ? '...' : ''}${c.reset}`
|
|
195
|
-
);
|
|
196
|
-
}
|
|
197
|
-
diffCount++;
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
if (diffCount >= maxDiffs) {
|
|
202
|
-
console.log(`${c.dim} ... and more changes${c.reset}`);
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
/**
|
|
207
|
-
* Compare two versions
|
|
208
|
-
*/
|
|
209
|
-
function compare(pathA, pathB) {
|
|
210
|
-
const contentA = fs.readFileSync(pathA, 'utf8');
|
|
211
|
-
const contentB = fs.readFileSync(pathB, 'utf8');
|
|
212
|
-
|
|
213
|
-
const analysisA = analyzeMarkdown(contentA);
|
|
214
|
-
const analysisB = analyzeMarkdown(contentB);
|
|
215
|
-
|
|
216
|
-
console.log(`\n${c.bold}${c.magenta}GEPA Comparison Report${c.reset}`);
|
|
217
|
-
console.log(`${c.dim}Before: ${pathA}${c.reset}`);
|
|
218
|
-
console.log(`${c.dim}After: ${pathB}${c.reset}`);
|
|
219
|
-
|
|
220
|
-
formatComparison(analysisA, analysisB);
|
|
221
|
-
showSectionDiff(analysisA, analysisB);
|
|
222
|
-
showInlineDiff(contentA, contentB);
|
|
223
|
-
|
|
224
|
-
// Score summary
|
|
225
|
-
const tokenChange = analysisB.estimatedTokens - analysisA.estimatedTokens;
|
|
226
|
-
const ruleChange =
|
|
227
|
-
analysisB.mustCount +
|
|
228
|
-
analysisB.neverCount +
|
|
229
|
-
analysisB.alwaysCount -
|
|
230
|
-
(analysisA.mustCount + analysisA.neverCount + analysisA.alwaysCount);
|
|
231
|
-
|
|
232
|
-
console.log(`\n${c.bold}Summary:${c.reset}`);
|
|
233
|
-
console.log(
|
|
234
|
-
` Token budget: ${tokenChange >= 0 ? c.yellow + '+' : c.green}${tokenChange}${c.reset} tokens`
|
|
235
|
-
);
|
|
236
|
-
console.log(
|
|
237
|
-
` Rule density: ${ruleChange >= 0 ? c.green + '+' : c.red}${ruleChange}${c.reset} explicit rules`
|
|
238
|
-
);
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
/**
|
|
242
|
-
* Watch for changes and auto-optimize
|
|
243
|
-
*/
|
|
244
|
-
async function watch(targetPath) {
|
|
245
|
-
const claudeMdPath = targetPath || path.join(process.cwd(), 'CLAUDE.md');
|
|
246
|
-
|
|
247
|
-
if (!fs.existsSync(claudeMdPath)) {
|
|
248
|
-
console.error(`${c.red}Error: ${claudeMdPath} not found${c.reset}`);
|
|
249
|
-
process.exit(1);
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
let lastHash = hashContent(fs.readFileSync(claudeMdPath, 'utf8'));
|
|
253
|
-
let isOptimizing = false;
|
|
254
|
-
let optimizeQueue = false;
|
|
255
|
-
|
|
256
|
-
console.log(`${c.bold}${c.cyan}GEPA Auto-Optimizer${c.reset}`);
|
|
257
|
-
console.log(`${c.dim}Watching: ${claudeMdPath}${c.reset}`);
|
|
258
|
-
console.log(`${c.dim}Press Ctrl+C to stop${c.reset}\n`);
|
|
259
|
-
|
|
260
|
-
// Initial analysis
|
|
261
|
-
const initial = analyzeMarkdown(fs.readFileSync(claudeMdPath, 'utf8'));
|
|
262
|
-
console.log(`${c.bold}Current state:${c.reset}`);
|
|
263
|
-
console.log(
|
|
264
|
-
` ${initial.totalLines} lines, ~${initial.estimatedTokens} tokens`
|
|
265
|
-
);
|
|
266
|
-
console.log(
|
|
267
|
-
` ${initial.h2Count} sections, ${initial.mustCount + initial.neverCount + initial.alwaysCount} explicit rules\n`
|
|
268
|
-
);
|
|
269
|
-
|
|
270
|
-
// Watch loop
|
|
271
|
-
const checkInterval = setInterval(async () => {
|
|
272
|
-
try {
|
|
273
|
-
const content = fs.readFileSync(claudeMdPath, 'utf8');
|
|
274
|
-
const currentHash = hashContent(content);
|
|
275
|
-
|
|
276
|
-
if (currentHash !== lastHash) {
|
|
277
|
-
console.log(
|
|
278
|
-
`\n${c.yellow}⚡ Change detected!${c.reset} (${new Date().toLocaleTimeString()})`
|
|
279
|
-
);
|
|
280
|
-
|
|
281
|
-
// Save before state
|
|
282
|
-
const beforePath = path.join(GEPA_DIR, '.before-optimize.md');
|
|
283
|
-
const statePath = path.join(GEPA_DIR, 'state.json');
|
|
284
|
-
|
|
285
|
-
if (fs.existsSync(statePath)) {
|
|
286
|
-
const state = JSON.parse(fs.readFileSync(statePath, 'utf8'));
|
|
287
|
-
const currentBest = path.join(
|
|
288
|
-
GENERATIONS_DIR,
|
|
289
|
-
`gen-${String(state.currentGeneration).padStart(3, '0')}`,
|
|
290
|
-
`${state.bestVariant}.md`
|
|
291
|
-
);
|
|
292
|
-
if (fs.existsSync(currentBest)) {
|
|
293
|
-
fs.copyFileSync(currentBest, beforePath);
|
|
294
|
-
}
|
|
295
|
-
}
|
|
296
|
-
|
|
297
|
-
lastHash = currentHash;
|
|
298
|
-
|
|
299
|
-
if (isOptimizing) {
|
|
300
|
-
optimizeQueue = true;
|
|
301
|
-
console.log(
|
|
302
|
-
`${c.dim} Optimization in progress, queued for next run${c.reset}`
|
|
303
|
-
);
|
|
304
|
-
return;
|
|
305
|
-
}
|
|
306
|
-
|
|
307
|
-
isOptimizing = true;
|
|
308
|
-
|
|
309
|
-
// Run quick optimization (1 generation)
|
|
310
|
-
console.log(`${c.cyan} Running GEPA optimization...${c.reset}`);
|
|
311
|
-
|
|
312
|
-
try {
|
|
313
|
-
// Re-init with new content
|
|
314
|
-
execSync(
|
|
315
|
-
`node ${path.join(GEPA_DIR, 'optimize.js')} init ${claudeMdPath}`,
|
|
316
|
-
{
|
|
317
|
-
stdio: 'pipe',
|
|
318
|
-
}
|
|
319
|
-
);
|
|
320
|
-
|
|
321
|
-
// Quick mutate + score
|
|
322
|
-
execSync(`node ${path.join(GEPA_DIR, 'optimize.js')} mutate`, {
|
|
323
|
-
stdio: 'pipe',
|
|
324
|
-
});
|
|
325
|
-
|
|
326
|
-
execSync(`node ${path.join(GEPA_DIR, 'optimize.js')} score`, {
|
|
327
|
-
stdio: 'pipe',
|
|
328
|
-
});
|
|
329
|
-
|
|
330
|
-
// Show comparison
|
|
331
|
-
const afterPath = path.join(GENERATIONS_DIR, 'current');
|
|
332
|
-
if (fs.existsSync(beforePath) && fs.existsSync(afterPath)) {
|
|
333
|
-
const resolvedAfter = fs.realpathSync(afterPath);
|
|
334
|
-
compare(beforePath, resolvedAfter);
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
// Load state for summary
|
|
338
|
-
const state = JSON.parse(fs.readFileSync(statePath, 'utf8'));
|
|
339
|
-
console.log(`\n${c.green}✓ Optimization complete${c.reset}`);
|
|
340
|
-
console.log(
|
|
341
|
-
` Best variant: ${c.bold}${state.bestVariant}${c.reset}`
|
|
342
|
-
);
|
|
343
|
-
console.log(
|
|
344
|
-
` Score: ${c.bold}${(state.bestScore * 100).toFixed(1)}%${c.reset}`
|
|
345
|
-
);
|
|
346
|
-
console.log(
|
|
347
|
-
`\n${c.dim} To apply: cp ${path.join(GENERATIONS_DIR, 'current')} ${claudeMdPath}${c.reset}`
|
|
348
|
-
);
|
|
349
|
-
} catch (e) {
|
|
350
|
-
console.error(
|
|
351
|
-
`${c.red} Optimization failed: ${e.message}${c.reset}`
|
|
352
|
-
);
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
isOptimizing = false;
|
|
356
|
-
|
|
357
|
-
// Process queue
|
|
358
|
-
if (optimizeQueue) {
|
|
359
|
-
optimizeQueue = false;
|
|
360
|
-
lastHash = ''; // Force re-check
|
|
361
|
-
}
|
|
362
|
-
}
|
|
363
|
-
} catch (e) {
|
|
364
|
-
// File might be temporarily unavailable during write
|
|
365
|
-
}
|
|
366
|
-
}, 2000); // Check every 2 seconds
|
|
367
|
-
|
|
368
|
-
// Handle shutdown
|
|
369
|
-
process.on('SIGINT', () => {
|
|
370
|
-
clearInterval(checkInterval);
|
|
371
|
-
console.log(`\n${c.dim}Watcher stopped${c.reset}`);
|
|
372
|
-
process.exit(0);
|
|
373
|
-
});
|
|
374
|
-
}
|
|
375
|
-
|
|
376
|
-
/**
|
|
377
|
-
* Generate optimization report
|
|
378
|
-
*/
|
|
379
|
-
function report() {
|
|
380
|
-
const statePath = path.join(GEPA_DIR, 'state.json');
|
|
381
|
-
|
|
382
|
-
if (!fs.existsSync(statePath)) {
|
|
383
|
-
console.error(`${c.red}No GEPA state found. Run 'init' first.${c.reset}`);
|
|
384
|
-
return;
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
const state = JSON.parse(fs.readFileSync(statePath, 'utf8'));
|
|
388
|
-
|
|
389
|
-
console.log(
|
|
390
|
-
`\n${c.bold}${c.magenta}═══════════════════════════════════════════════════════════${c.reset}`
|
|
391
|
-
);
|
|
392
|
-
console.log(
|
|
393
|
-
`${c.bold}${c.magenta} GEPA OPTIMIZATION REPORT ${c.reset}`
|
|
394
|
-
);
|
|
395
|
-
console.log(
|
|
396
|
-
`${c.bold}${c.magenta}═══════════════════════════════════════════════════════════${c.reset}\n`
|
|
397
|
-
);
|
|
398
|
-
|
|
399
|
-
console.log(`${c.bold}Current State:${c.reset}`);
|
|
400
|
-
console.log(` Generation: ${c.cyan}${state.currentGeneration}${c.reset}`);
|
|
401
|
-
console.log(` Best Variant: ${c.green}${state.bestVariant}${c.reset}`);
|
|
402
|
-
console.log(
|
|
403
|
-
` Best Score: ${c.bold}${(state.bestScore * 100).toFixed(1)}%${c.reset}`
|
|
404
|
-
);
|
|
405
|
-
console.log(` Target File: ${c.dim}${state.targetPath}${c.reset}`);
|
|
406
|
-
|
|
407
|
-
// History summary
|
|
408
|
-
if (state.history && state.history.length > 0) {
|
|
409
|
-
console.log(`\n${c.bold}Evolution History:${c.reset}`);
|
|
410
|
-
|
|
411
|
-
const scoreHistory = state.history
|
|
412
|
-
.filter((h) => h.action === 'select' && h.scores)
|
|
413
|
-
.map((h) => ({
|
|
414
|
-
gen: h.generation,
|
|
415
|
-
best: h.best,
|
|
416
|
-
score: h.scores.find((s) => s.variant === h.best)?.score || 0,
|
|
417
|
-
}));
|
|
418
|
-
|
|
419
|
-
if (scoreHistory.length > 0) {
|
|
420
|
-
// ASCII chart
|
|
421
|
-
const maxScore = Math.max(...scoreHistory.map((h) => h.score));
|
|
422
|
-
const chartWidth = 40;
|
|
423
|
-
|
|
424
|
-
scoreHistory.forEach((h) => {
|
|
425
|
-
const barLen = Math.round((h.score / maxScore) * chartWidth);
|
|
426
|
-
const bar = '█'.repeat(barLen) + '░'.repeat(chartWidth - barLen);
|
|
427
|
-
const pct = (h.score * 100).toFixed(0).padStart(3);
|
|
428
|
-
console.log(
|
|
429
|
-
` Gen ${String(h.gen).padStart(2)}: ${c.green}${bar}${c.reset} ${pct}% (${h.best})`
|
|
430
|
-
);
|
|
431
|
-
});
|
|
432
|
-
|
|
433
|
-
// Improvement
|
|
434
|
-
if (scoreHistory.length > 1) {
|
|
435
|
-
const first = scoreHistory[0].score;
|
|
436
|
-
const last = scoreHistory[scoreHistory.length - 1].score;
|
|
437
|
-
const improvement = (((last - first) / first) * 100).toFixed(1);
|
|
438
|
-
console.log(
|
|
439
|
-
`\n ${c.bold}Total improvement: ${improvement >= 0 ? c.green + '+' : c.red}${improvement}%${c.reset}`
|
|
440
|
-
);
|
|
441
|
-
}
|
|
442
|
-
}
|
|
443
|
-
}
|
|
444
|
-
|
|
445
|
-
// Show before/after if available
|
|
446
|
-
const beforePath = path.join(GEPA_DIR, '.before-optimize.md');
|
|
447
|
-
const currentPath = path.join(GENERATIONS_DIR, 'current');
|
|
448
|
-
|
|
449
|
-
if (fs.existsSync(beforePath) && fs.existsSync(currentPath)) {
|
|
450
|
-
console.log(`\n${c.bold}Latest Optimization:${c.reset}`);
|
|
451
|
-
compare(beforePath, fs.realpathSync(currentPath));
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
console.log(
|
|
455
|
-
`\n${c.dim}Run 'node auto-optimize.js watch' to auto-optimize on changes${c.reset}\n`
|
|
456
|
-
);
|
|
457
|
-
}
|
|
458
|
-
|
|
459
|
-
// CLI
|
|
460
|
-
const command = process.argv[2];
|
|
461
|
-
const arg1 = process.argv[3];
|
|
462
|
-
const arg2 = process.argv[4];
|
|
463
|
-
|
|
464
|
-
switch (command) {
|
|
465
|
-
case 'watch':
|
|
466
|
-
watch(arg1);
|
|
467
|
-
break;
|
|
468
|
-
case 'compare':
|
|
469
|
-
if (!arg1 || !arg2) {
|
|
470
|
-
console.error('Usage: compare <before.md> <after.md>');
|
|
471
|
-
process.exit(1);
|
|
472
|
-
}
|
|
473
|
-
compare(arg1, arg2);
|
|
474
|
-
break;
|
|
475
|
-
case 'report':
|
|
476
|
-
report();
|
|
477
|
-
break;
|
|
478
|
-
default:
|
|
479
|
-
console.log(`
|
|
480
|
-
${c.bold}GEPA Auto-Optimizer${c.reset}
|
|
481
|
-
|
|
482
|
-
Usage:
|
|
483
|
-
node auto-optimize.js watch [path] Watch CLAUDE.md and auto-optimize
|
|
484
|
-
node auto-optimize.js compare <a> <b> Compare two versions
|
|
485
|
-
node auto-optimize.js report Show optimization report
|
|
486
|
-
|
|
487
|
-
Examples:
|
|
488
|
-
node auto-optimize.js watch ./CLAUDE.md
|
|
489
|
-
node auto-optimize.js compare gen-000/baseline.md gen-001/variant-a.md
|
|
490
|
-
node auto-optimize.js report
|
|
491
|
-
`);
|
|
492
|
-
}
|
|
493
|
-
|
|
494
|
-
export { compare, analyzeMarkdown, watch, report };
|
|
@@ -1,203 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
/**
|
|
3
|
-
* GEPA Eval Tracker Hook
|
|
4
|
-
*
|
|
5
|
-
* Captures agent behavior during sessions for evaluation.
|
|
6
|
-
* Install in ~/.claude/settings.json under hooks.
|
|
7
|
-
*/
|
|
8
|
-
|
|
9
|
-
import fs from 'fs';
|
|
10
|
-
import path from 'path';
|
|
11
|
-
|
|
12
|
-
const GEPA_DIR =
|
|
13
|
-
process.env.GEPA_DIR || path.join(process.env.HOME, '.claude', 'gepa');
|
|
14
|
-
const RESULTS_DIR = path.join(GEPA_DIR, 'results');
|
|
15
|
-
const SESSIONS_DIR = path.join(RESULTS_DIR, 'sessions');
|
|
16
|
-
|
|
17
|
-
// Ensure directories exist
|
|
18
|
-
[RESULTS_DIR, SESSIONS_DIR].forEach((dir) => {
|
|
19
|
-
if (!fs.existsSync(dir)) {
|
|
20
|
-
fs.mkdirSync(dir, { recursive: true });
|
|
21
|
-
}
|
|
22
|
-
});
|
|
23
|
-
|
|
24
|
-
/**
|
|
25
|
-
* Session tracking state
|
|
26
|
-
*/
|
|
27
|
-
class SessionTracker {
|
|
28
|
-
constructor() {
|
|
29
|
-
this.sessionId = process.env.CLAUDE_SESSION_ID || `session-${Date.now()}`;
|
|
30
|
-
this.variant = process.env.GEPA_VARIANT || 'baseline';
|
|
31
|
-
this.generation = parseInt(process.env.GEPA_GENERATION || '0');
|
|
32
|
-
|
|
33
|
-
this.data = {
|
|
34
|
-
sessionId: this.sessionId,
|
|
35
|
-
variant: this.variant,
|
|
36
|
-
generation: this.generation,
|
|
37
|
-
startTime: new Date().toISOString(),
|
|
38
|
-
endTime: null,
|
|
39
|
-
toolCalls: [],
|
|
40
|
-
errors: [],
|
|
41
|
-
userFeedback: [],
|
|
42
|
-
tokenUsage: { input: 0, output: 0 },
|
|
43
|
-
metrics: {},
|
|
44
|
-
};
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
trackToolCall(tool, input, output, duration) {
|
|
48
|
-
this.data.toolCalls.push({
|
|
49
|
-
tool,
|
|
50
|
-
input: this.sanitize(input),
|
|
51
|
-
output: this.summarize(output),
|
|
52
|
-
duration,
|
|
53
|
-
timestamp: new Date().toISOString(),
|
|
54
|
-
success: !output?.error,
|
|
55
|
-
});
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
trackError(error, context) {
|
|
59
|
-
this.data.errors.push({
|
|
60
|
-
error: error.message || String(error),
|
|
61
|
-
context,
|
|
62
|
-
timestamp: new Date().toISOString(),
|
|
63
|
-
});
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
trackFeedback(type, value) {
|
|
67
|
-
this.data.userFeedback.push({
|
|
68
|
-
type, // 'thumbs_up', 'thumbs_down', 'correction', 'retry'
|
|
69
|
-
value,
|
|
70
|
-
timestamp: new Date().toISOString(),
|
|
71
|
-
});
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
trackTokens(input, output) {
|
|
75
|
-
this.data.tokenUsage.input += input;
|
|
76
|
-
this.data.tokenUsage.output += output;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
sanitize(obj) {
|
|
80
|
-
// Remove sensitive data
|
|
81
|
-
if (typeof obj !== 'object') return obj;
|
|
82
|
-
const sanitized = { ...obj };
|
|
83
|
-
const sensitiveKeys = ['apiKey', 'token', 'password', 'secret', 'key'];
|
|
84
|
-
for (const key of Object.keys(sanitized)) {
|
|
85
|
-
if (sensitiveKeys.some((s) => key.toLowerCase().includes(s))) {
|
|
86
|
-
sanitized[key] = '[REDACTED]';
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
return sanitized;
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
summarize(output) {
|
|
93
|
-
// Truncate long outputs
|
|
94
|
-
const str = typeof output === 'string' ? output : JSON.stringify(output);
|
|
95
|
-
return str.length > 1000 ? str.slice(0, 1000) + '...[truncated]' : str;
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
finalize() {
|
|
99
|
-
this.data.endTime = new Date().toISOString();
|
|
100
|
-
this.data.duration =
|
|
101
|
-
new Date(this.data.endTime) - new Date(this.data.startTime);
|
|
102
|
-
|
|
103
|
-
// Calculate basic metrics
|
|
104
|
-
this.data.metrics = {
|
|
105
|
-
totalToolCalls: this.data.toolCalls.length,
|
|
106
|
-
successfulToolCalls: this.data.toolCalls.filter((t) => t.success).length,
|
|
107
|
-
errorCount: this.data.errors.length,
|
|
108
|
-
avgToolDuration:
|
|
109
|
-
this.data.toolCalls.length > 0
|
|
110
|
-
? this.data.toolCalls.reduce((sum, t) => sum + (t.duration || 0), 0) /
|
|
111
|
-
this.data.toolCalls.length
|
|
112
|
-
: 0,
|
|
113
|
-
positiveFeeback: this.data.userFeedback.filter(
|
|
114
|
-
(f) => f.type === 'thumbs_up'
|
|
115
|
-
).length,
|
|
116
|
-
negativeFeeback: this.data.userFeedback.filter(
|
|
117
|
-
(f) => f.type === 'thumbs_down'
|
|
118
|
-
).length,
|
|
119
|
-
};
|
|
120
|
-
|
|
121
|
-
return this.data;
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
save() {
|
|
125
|
-
const data = this.finalize();
|
|
126
|
-
const filename = `${this.generation}-${this.variant}-${this.sessionId}.json`;
|
|
127
|
-
const filepath = path.join(SESSIONS_DIR, filename);
|
|
128
|
-
fs.writeFileSync(filepath, JSON.stringify(data, null, 2));
|
|
129
|
-
|
|
130
|
-
// Also append to scores.jsonl for quick analysis
|
|
131
|
-
const scoreLine = {
|
|
132
|
-
sessionId: this.sessionId,
|
|
133
|
-
variant: this.variant,
|
|
134
|
-
generation: this.generation,
|
|
135
|
-
metrics: data.metrics,
|
|
136
|
-
timestamp: data.endTime,
|
|
137
|
-
};
|
|
138
|
-
fs.appendFileSync(
|
|
139
|
-
path.join(RESULTS_DIR, 'scores.jsonl'),
|
|
140
|
-
JSON.stringify(scoreLine) + '\n'
|
|
141
|
-
);
|
|
142
|
-
|
|
143
|
-
return filepath;
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
// Global tracker instance
|
|
148
|
-
let tracker = null;
|
|
149
|
-
|
|
150
|
-
/**
|
|
151
|
-
* Hook handlers
|
|
152
|
-
*/
|
|
153
|
-
export function onSessionStart(context) {
|
|
154
|
-
tracker = new SessionTracker();
|
|
155
|
-
console.error(
|
|
156
|
-
`[GEPA] Tracking session: ${tracker.sessionId} (gen=${tracker.generation}, variant=${tracker.variant})`
|
|
157
|
-
);
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
export function onToolCall(tool, input) {
|
|
161
|
-
if (!tracker) return;
|
|
162
|
-
tracker._currentTool = { tool, input, startTime: Date.now() };
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
export function onToolResult(tool, result) {
|
|
166
|
-
if (!tracker || !tracker._currentTool) return;
|
|
167
|
-
const duration = Date.now() - tracker._currentTool.startTime;
|
|
168
|
-
tracker.trackToolCall(tool, tracker._currentTool.input, result, duration);
|
|
169
|
-
tracker._currentTool = null;
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
export function onError(error, context) {
|
|
173
|
-
if (!tracker) return;
|
|
174
|
-
tracker.trackError(error, context);
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
export function onUserFeedback(type, value) {
|
|
178
|
-
if (!tracker) return;
|
|
179
|
-
tracker.trackFeedback(type, value);
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
export function onSessionEnd() {
|
|
183
|
-
if (!tracker) return;
|
|
184
|
-
const filepath = tracker.save();
|
|
185
|
-
console.error(`[GEPA] Session data saved to: ${filepath}`);
|
|
186
|
-
tracker = null;
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
// CLI interface for testing
|
|
190
|
-
if (process.argv[1] === import.meta.url.replace('file://', '')) {
|
|
191
|
-
const command = process.argv[2];
|
|
192
|
-
|
|
193
|
-
if (command === 'test') {
|
|
194
|
-
onSessionStart({});
|
|
195
|
-
onToolCall('Read', { file: '/test/file.ts' });
|
|
196
|
-
onToolResult('Read', { content: 'test content...' });
|
|
197
|
-
onToolCall('Edit', { file: '/test/file.ts', changes: '...' });
|
|
198
|
-
onToolResult('Edit', { success: true });
|
|
199
|
-
onUserFeedback('thumbs_up', 'good job');
|
|
200
|
-
onSessionEnd();
|
|
201
|
-
console.log('Test session recorded');
|
|
202
|
-
}
|
|
203
|
-
}
|