@stackmemoryai/stackmemory 1.5.9 → 1.6.1

package/scripts/gepa/hooks/eval-tracker.js

@@ -1,203 +0,0 @@
-#!/usr/bin/env node
-/**
- * GEPA Eval Tracker Hook
- *
- * Captures agent behavior during sessions for evaluation.
- * Install in ~/.claude/settings.json under hooks.
- */
-
-import fs from 'fs';
-import path from 'path';
-
-const GEPA_DIR =
-  process.env.GEPA_DIR || path.join(process.env.HOME, '.claude', 'gepa');
-const RESULTS_DIR = path.join(GEPA_DIR, 'results');
-const SESSIONS_DIR = path.join(RESULTS_DIR, 'sessions');
-
-// Ensure directories exist
-[RESULTS_DIR, SESSIONS_DIR].forEach((dir) => {
-  if (!fs.existsSync(dir)) {
-    fs.mkdirSync(dir, { recursive: true });
-  }
-});
-
-/**
- * Session tracking state
- */
-class SessionTracker {
-  constructor() {
-    this.sessionId = process.env.CLAUDE_SESSION_ID || `session-${Date.now()}`;
-    this.variant = process.env.GEPA_VARIANT || 'baseline';
-    this.generation = parseInt(process.env.GEPA_GENERATION || '0');
-
-    this.data = {
-      sessionId: this.sessionId,
-      variant: this.variant,
-      generation: this.generation,
-      startTime: new Date().toISOString(),
-      endTime: null,
-      toolCalls: [],
-      errors: [],
-      userFeedback: [],
-      tokenUsage: { input: 0, output: 0 },
-      metrics: {},
-    };
-  }
-
-  trackToolCall(tool, input, output, duration) {
-    this.data.toolCalls.push({
-      tool,
-      input: this.sanitize(input),
-      output: this.summarize(output),
-      duration,
-      timestamp: new Date().toISOString(),
-      success: !output?.error,
-    });
-  }
-
-  trackError(error, context) {
-    this.data.errors.push({
-      error: error.message || String(error),
-      context,
-      timestamp: new Date().toISOString(),
-    });
-  }
-
-  trackFeedback(type, value) {
-    this.data.userFeedback.push({
-      type, // 'thumbs_up', 'thumbs_down', 'correction', 'retry'
-      value,
-      timestamp: new Date().toISOString(),
-    });
-  }
-
-  trackTokens(input, output) {
-    this.data.tokenUsage.input += input;
-    this.data.tokenUsage.output += output;
-  }
-
-  sanitize(obj) {
-    // Remove sensitive data
-    if (typeof obj !== 'object') return obj;
-    const sanitized = { ...obj };
-    const sensitiveKeys = ['apiKey', 'token', 'password', 'secret', 'key'];
-    for (const key of Object.keys(sanitized)) {
-      if (sensitiveKeys.some((s) => key.toLowerCase().includes(s))) {
-        sanitized[key] = '[REDACTED]';
-      }
-    }
-    return sanitized;
-  }
-
-  summarize(output) {
-    // Truncate long outputs
-    const str = typeof output === 'string' ? output : JSON.stringify(output);
-    return str.length > 1000 ? str.slice(0, 1000) + '...[truncated]' : str;
-  }
-
-  finalize() {
-    this.data.endTime = new Date().toISOString();
-    this.data.duration =
-      new Date(this.data.endTime) - new Date(this.data.startTime);
-
-    // Calculate basic metrics
-    this.data.metrics = {
-      totalToolCalls: this.data.toolCalls.length,
-      successfulToolCalls: this.data.toolCalls.filter((t) => t.success).length,
-      errorCount: this.data.errors.length,
-      avgToolDuration:
-        this.data.toolCalls.length > 0
-          ? this.data.toolCalls.reduce((sum, t) => sum + (t.duration || 0), 0) /
-            this.data.toolCalls.length
-          : 0,
-      positiveFeeback: this.data.userFeedback.filter(
-        (f) => f.type === 'thumbs_up'
-      ).length,
-      negativeFeeback: this.data.userFeedback.filter(
-        (f) => f.type === 'thumbs_down'
-      ).length,
-    };
-
-    return this.data;
-  }
-
-  save() {
-    const data = this.finalize();
-    const filename = `${this.generation}-${this.variant}-${this.sessionId}.json`;
-    const filepath = path.join(SESSIONS_DIR, filename);
-    fs.writeFileSync(filepath, JSON.stringify(data, null, 2));
-
-    // Also append to scores.jsonl for quick analysis
-    const scoreLine = {
-      sessionId: this.sessionId,
-      variant: this.variant,
-      generation: this.generation,
-      metrics: data.metrics,
-      timestamp: data.endTime,
-    };
-    fs.appendFileSync(
-      path.join(RESULTS_DIR, 'scores.jsonl'),
-      JSON.stringify(scoreLine) + '\n'
-    );
-
-    return filepath;
-  }
-}
-
-// Global tracker instance
-let tracker = null;
-
-/**
- * Hook handlers
- */
-export function onSessionStart(context) {
-  tracker = new SessionTracker();
-  console.error(
-    `[GEPA] Tracking session: ${tracker.sessionId} (gen=${tracker.generation}, variant=${tracker.variant})`
-  );
-}
-
-export function onToolCall(tool, input) {
-  if (!tracker) return;
-  tracker._currentTool = { tool, input, startTime: Date.now() };
-}
-
-export function onToolResult(tool, result) {
-  if (!tracker || !tracker._currentTool) return;
-  const duration = Date.now() - tracker._currentTool.startTime;
-  tracker.trackToolCall(tool, tracker._currentTool.input, result, duration);
-  tracker._currentTool = null;
-}
-
-export function onError(error, context) {
-  if (!tracker) return;
-  tracker.trackError(error, context);
-}
-
-export function onUserFeedback(type, value) {
-  if (!tracker) return;
-  tracker.trackFeedback(type, value);
-}
-
-export function onSessionEnd() {
-  if (!tracker) return;
-  const filepath = tracker.save();
-  console.error(`[GEPA] Session data saved to: ${filepath}`);
-  tracker = null;
-}
-
-// CLI interface for testing
-if (process.argv[1] === import.meta.url.replace('file://', '')) {
-  const command = process.argv[2];
-
-  if (command === 'test') {
-    onSessionStart({});
-    onToolCall('Read', { file: '/test/file.ts' });
-    onToolResult('Read', { content: 'test content...' });
-    onToolCall('Edit', { file: '/test/file.ts', changes: '...' });
-    onToolResult('Edit', { success: true });
-    onUserFeedback('thumbs_up', 'good job');
-    onSessionEnd();
-    console.log('Test session recorded');
-  }
-}

package/scripts/gepa/hooks/reflect.js

@@ -1,350 +0,0 @@
-#!/usr/bin/env node
-/**
- * GEPA Reflection Engine
- *
- * Analyzes evaluation results to generate insights for next mutation cycle.
- * This is the key differentiator from random mutations.
- */
-
-import fs from 'fs';
-import path from 'path';
-import { fileURLToPath } from 'url';
-import { spawn } from 'child_process';
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const PROJECT_ROOT = path.resolve(__dirname, '../../..');
-
-// Load .env from project root
-const envPath = path.join(PROJECT_ROOT, '.env');
-if (fs.existsSync(envPath)) {
-  for (const line of fs.readFileSync(envPath, 'utf8').split('\n')) {
-    const match = line.match(/^([A-Z_][A-Z0-9_]*)=(.*)$/);
-    if (match && !process.env[match[1]]) {
-      process.env[match[1]] = match[2].replace(/^["']|["']$/g, '');
-    }
-  }
-}
-
-/**
- * Call Claude CLI via spawn (stdin pipe, no shell interpolation)
- */
-function spawnClaude(prompt) {
-  return new Promise((resolve, reject) => {
-    const child = spawn('claude', ['--print'], {
-      stdio: ['pipe', 'pipe', 'pipe'],
-      env: { ...process.env },
-    });
-
-    let stdout = '';
-    let stderr = '';
-
-    child.stdout.on('data', (d) => (stdout += d));
-    child.stderr.on('data', (d) => (stderr += d));
-
-    child.on('close', (code) => {
-      if (code !== 0 && !stdout)
-        return reject(new Error(stderr || `claude exited ${code}`));
-      resolve(stdout);
-    });
-
-    child.on('error', reject);
-
-    child.stdin.write(prompt);
-    child.stdin.end();
-  });
-}
-
-const GEPA_DIR = path.join(__dirname, '..');
-const RESULTS_DIR = path.join(GEPA_DIR, 'results');
-const GENERATIONS_DIR = path.join(GEPA_DIR, 'generations');
-
-/**
- * Analyze session data for patterns
- */
-function analyzeSessionPatterns() {
-  const sessionsDir = path.join(RESULTS_DIR, 'sessions');
-  if (!fs.existsSync(sessionsDir)) return { patterns: [], insights: [] };
-
-  const sessions = fs
-    .readdirSync(sessionsDir)
-    .filter((f) => f.endsWith('.json'))
-    .map((f) => JSON.parse(fs.readFileSync(path.join(sessionsDir, f), 'utf8')));
-
-  const patterns = {
-    // Error patterns
-    commonErrors: extractCommonErrors(sessions),
-
-    // Tool usage patterns
-    toolUsage: extractToolPatterns(sessions),
-
-    // Feedback patterns
-    feedbackPatterns: extractFeedbackPatterns(sessions),
-
-    // Performance patterns
-    performanceByVariant: extractPerformanceByVariant(sessions),
-  };
-
-  return patterns;
-}
-
-function extractCommonErrors(sessions) {
-  const errorCounts = {};
-
-  sessions.forEach((s) => {
-    s.errors?.forEach((e) => {
-      const normalized = normalizeError(e.error);
-      errorCounts[normalized] = (errorCounts[normalized] || 0) + 1;
-    });
-  });
-
-  return Object.entries(errorCounts)
-    .sort((a, b) => b[1] - a[1])
-    .slice(0, 10)
-    .map(([error, count]) => ({ error, count }));
-}
-
-function normalizeError(error) {
-  // Normalize error messages for grouping
-  return error
-    .replace(/\d+/g, 'N')
-    .replace(/['"`][^'"`]+['"`]/g, '"..."')
-    .replace(/\/[^\s]+/g, '/path')
-    .slice(0, 100);
-}
-
-function extractToolPatterns(sessions) {
-  const toolStats = {};
-
-  sessions.forEach((s) => {
-    s.toolCalls?.forEach((t) => {
-      if (!toolStats[t.tool]) {
-        toolStats[t.tool] = { count: 0, success: 0, avgDuration: 0 };
-      }
-      toolStats[t.tool].count++;
-      if (t.success) toolStats[t.tool].success++;
-      toolStats[t.tool].avgDuration += t.duration || 0;
-    });
-  });
-
-  // Calculate averages
-  Object.values(toolStats).forEach((s) => {
-    s.avgDuration = s.count > 0 ? s.avgDuration / s.count : 0;
-    s.successRate = s.count > 0 ? s.success / s.count : 0;
-  });
-
-  return toolStats;
-}
-
-function extractFeedbackPatterns(sessions) {
-  const feedback = { positive: 0, negative: 0, corrections: 0, retries: 0 };
-
-  sessions.forEach((s) => {
-    s.userFeedback?.forEach((f) => {
-      if (f.type === 'thumbs_up') feedback.positive++;
-      if (f.type === 'thumbs_down') feedback.negative++;
-      if (f.type === 'correction') feedback.corrections++;
-      if (f.type === 'retry') feedback.retries++;
-    });
-  });
-
-  return feedback;
-}
-
-function extractPerformanceByVariant(sessions) {
-  const variants = {};
-
-  sessions.forEach((s) => {
-    if (!variants[s.variant]) {
-      variants[s.variant] = {
-        sessions: 0,
-        totalErrors: 0,
-        totalSuccess: 0,
-        avgDuration: 0,
-      };
-    }
-
-    variants[s.variant].sessions++;
-    variants[s.variant].totalErrors += s.metrics?.errorCount || 0;
-    variants[s.variant].totalSuccess += s.metrics?.successfulToolCalls || 0;
-    variants[s.variant].avgDuration += s.duration || 0;
-  });
-
-  // Normalize
-  Object.values(variants).forEach((v) => {
-    v.avgDuration = v.sessions > 0 ? v.avgDuration / v.sessions : 0;
-    v.errorRate = v.sessions > 0 ? v.totalErrors / v.sessions : 0;
-    v.successRate = v.totalSuccess / (v.totalSuccess + v.totalErrors) || 0;
-  });
-
-  return variants;
-}
-
-/**
- * Generate reflection insights
- */
-async function generateReflection() {
-  const patterns = analyzeSessionPatterns();
-  const state = JSON.parse(
-    fs.readFileSync(path.join(GEPA_DIR, 'state.json'), 'utf8')
-  );
-
-  // Load current best prompt for context
-  const currentPrompt = fs.readFileSync(
-    path.join(
-      GENERATIONS_DIR,
-      `gen-${String(state.currentGeneration).padStart(3, '0')}`,
-      `${state.bestVariant}.md`
-    ),
-    'utf8'
-  );
-
-  const reflectionPrompt = `Analyze these AI agent performance patterns and generate specific improvement recommendations.
-
-CURRENT SYSTEM PROMPT (excerpt):
-\`\`\`markdown
-${currentPrompt.slice(0, 3000)}...
-\`\`\`
-
-PERFORMANCE PATTERNS:
-
-1. COMMON ERRORS (${patterns.commonErrors.length} types):
-${patterns.commonErrors.map((e) => `   - "${e.error}" (${e.count}x)`).join('\n')}
-
-2. TOOL USAGE:
-${Object.entries(patterns.toolUsage)
-  .map(
-    ([tool, s]) =>
-      `   - ${tool}: ${s.count} calls, ${(s.successRate * 100).toFixed(0)}% success, ${s.avgDuration.toFixed(0)}ms avg`
-  )
-  .join('\n')}
-
-3. USER FEEDBACK:
-   - Positive: ${patterns.feedbackPatterns.positive}
-   - Negative: ${patterns.feedbackPatterns.negative}
-   - Corrections needed: ${patterns.feedbackPatterns.corrections}
-   - Retries: ${patterns.feedbackPatterns.retries}
-
-4. VARIANT PERFORMANCE:
-${Object.entries(patterns.performanceByVariant)
-  .map(
-    ([v, s]) =>
-      `   - ${v}: ${s.sessions} sessions, ${s.errorRate.toFixed(1)} errors/session, ${(s.successRate * 100).toFixed(0)}% success`
-  )
-  .join('\n')}
-
-Based on this data, provide:
-
-1. TOP 3 FAILURE MODES - What's causing the most errors?
-
-2. MISSING INSTRUCTIONS - What rules should be added to prevent errors?
-
-3. UNCLEAR INSTRUCTIONS - What existing rules are being misinterpreted?
-
-4. PRIORITY MUTATIONS - What specific changes would have highest impact?
-
-Format as JSON:
-{
-  "failureModes": ["...", "...", "..."],
-  "missingInstructions": ["...", "...", "..."],
-  "unclearInstructions": ["...", "...", "..."],
-  "priorityMutations": [
-    {"type": "add|modify|remove", "section": "...", "change": "...", "rationale": "..."},
-    ...
-  ]
-}`;
-
-  try {
-    const result = await spawnClaude(reflectionPrompt);
-
-    // Parse JSON from response
-    const jsonMatch = result.match(/\{[\s\S]*\}/);
-    if (jsonMatch) {
-      const insights = JSON.parse(jsonMatch[0]);
-
-      // Save reflection
-      const reflectionPath = path.join(
-        RESULTS_DIR,
-        `reflection-${Date.now()}.json`
-      );
-      fs.writeFileSync(
-        reflectionPath,
-        JSON.stringify(
-          {
-            timestamp: new Date().toISOString(),
-            generation: state.currentGeneration,
-            patterns,
-            insights,
-          },
-          null,
-          2
-        )
-      );
-
-      return insights;
-    }
-  } catch (e) {
-    console.error('Reflection failed:', e.message);
-  }
-
-  return null;
-}
-
-/**
- * Generate focused mutation prompts based on reflection
- */
-function generateMutationGuide(insights) {
-  if (!insights) return null;
-
-  return `
-REFLECTION-GUIDED MUTATIONS:
-
-Based on ${insights.failureModes.length} identified failure modes:
-${insights.failureModes.map((f, i) => `${i + 1}. ${f}`).join('\n')}
-
-PRIORITY CHANGES:
-${insights.priorityMutations
-  .map(
-    (m) =>
-      `- [${m.type.toUpperCase()}] ${m.section}: ${m.change}\n  Rationale: ${m.rationale}`
-  )
-  .join('\n\n')}
-
-MISSING INSTRUCTIONS TO ADD:
-${insights.missingInstructions.map((i) => `- ${i}`).join('\n')}
-
-UNCLEAR INSTRUCTIONS TO CLARIFY:
-${insights.unclearInstructions.map((i) => `- ${i}`).join('\n')}
-`;
-}
-
-// CLI
-const command = process.argv[2];
-
-switch (command) {
-  case 'analyze':
-    const patterns = analyzeSessionPatterns();
-    console.log(JSON.stringify(patterns, null, 2));
-    break;
-
-  case 'reflect':
-    generateReflection().then((insights) => {
-      if (insights) {
-        console.log('\nReflection Insights:');
-        console.log(JSON.stringify(insights, null, 2));
-        console.log('\nMutation Guide:');
-        console.log(generateMutationGuide(insights));
-      }
-    });
-    break;
-
-  default:
-    console.log(`
-GEPA Reflection Engine
-
-Usage:
-  node reflect.js analyze    Analyze session patterns
-  node reflect.js reflect    Generate reflection insights
-`);
-}
-
-export { analyzeSessionPatterns, generateReflection, generateMutationGuide };