@ruvector/ruvllm 0.2.4 → 2.4.0

package/scripts/training/claude-code-synth.js

@@ -0,0 +1,505 @@
+#!/usr/bin/env node
+/**
+ * Claude Code Synthetic Data Generator
+ *
+ * Uses @ruvector/agentic-synth to generate high-quality
+ * training data for RuvLTRA routing optimization.
+ *
+ * Features:
+ * - Claude Code-specific task patterns
+ * - Hard negative mining for contrastive learning
+ * - Quality scoring based on task clarity
+ * - DSPy-based prompt optimization
+ */
+
+const { GoogleGenerativeAI } = require('@google/generative-ai');
+const { writeFileSync, existsSync, mkdirSync, readFileSync } = require('fs');
+const { join } = require('path');
+const { homedir } = require('os');
+
+// Configuration
+const OUTPUT_DIR = join(__dirname, 'generated');
+const EXAMPLES_PER_AGENT = 100;  // Generate 100 examples per agent
+const HARD_NEGATIVES_PER_AGENT = 20;
+
+// Agent definitions with Claude Code context
+const CLAUDE_CODE_AGENTS = {
+  coder: {
+    role: 'Software developer who implements features and writes production code',
+    claudeCodeContext: 'Uses Edit, Write, MultiEdit tools to create and modify code files',
+    keywords: ['implement', 'build', 'create', 'write code', 'add feature', 'component', 'function'],
+    examples: [
+      'Implement a binary search function in TypeScript',
+      'Build a React component for user authentication',
+      'Create a REST API endpoint for data retrieval',
+    ],
+  },
+  researcher: {
+    role: 'Technical researcher who investigates and analyzes',
+    claudeCodeContext: 'Uses Grep, Glob, Read, WebSearch tools to gather information',
+    keywords: ['research', 'investigate', 'explore', 'analyze', 'find', 'discover', 'study'],
+    examples: [
+      'Research best practices for React state management',
+      'Investigate why the API is returning slow responses',
+      'Explore different authentication strategies',
+    ],
+  },
+  reviewer: {
+    role: 'Code reviewer who evaluates code quality',
+    claudeCodeContext: 'Uses Read, Grep tools to analyze existing code for quality issues',
+    keywords: ['review', 'check', 'evaluate', 'assess', 'inspect', 'pull request', 'PR'],
+    examples: [
+      'Review the pull request for code quality',
+      'Check the implementation for potential issues',
+      'Evaluate the API design decisions',
+    ],
+  },
+  tester: {
+    role: 'QA engineer who writes and runs tests',
+    claudeCodeContext: 'Uses Write, Edit tools to create test files and Bash to run tests',
+    keywords: ['test', 'tests', 'testing', 'unit test', 'integration test', 'e2e', 'coverage', 'spec'],
+    examples: [
+      'Write unit tests for the authentication module',
+      'Add integration tests for the API endpoints',
+      'Create e2e tests for the checkout flow',
+    ],
+  },
+  architect: {
+    role: 'System architect who designs software structure',
+    claudeCodeContext: 'Uses Read, Grep tools to understand codebase and Write to document designs',
+    keywords: ['design', 'architecture', 'schema', 'structure', 'system', 'diagram', 'plan'],
+    examples: [
+      'Design the database schema for user profiles',
+      'Plan the microservices architecture',
+      'Create the system architecture diagram',
+    ],
+  },
+  'security-architect': {
+    role: 'Security specialist who audits vulnerabilities',
+    claudeCodeContext: 'Uses Grep, Read tools to scan code for security issues',
+    keywords: ['security', 'vulnerability', 'xss', 'injection', 'audit', 'cve', 'exploit'],
+    examples: [
+      'Audit the API endpoints for XSS vulnerabilities',
+      'Check for SQL injection vulnerabilities',
+      'Review authentication for security issues',
+    ],
+  },
+  debugger: {
+    role: 'Bug hunter who fixes errors and traces issues',
+    claudeCodeContext: 'Uses Read, Grep, Bash tools to trace issues and Edit to fix bugs',
+    keywords: ['debug', 'fix', 'bug', 'error', 'exception', 'crash', 'trace', 'issue'],
+    examples: [
+      'Fix the null pointer exception in login',
+      'Debug the memory leak in WebSocket handler',
+      'Trace the source of the intermittent error',
+    ],
+  },
+  documenter: {
+    role: 'Technical writer who creates documentation',
+    claudeCodeContext: 'Uses Write, Edit tools to create and update documentation files',
+    keywords: ['document', 'jsdoc', 'readme', 'comment', 'explain', 'describe'],
+    examples: [
+      'Write JSDoc comments for utility functions',
+      'Create README for the new package',
+      'Document the API endpoints',
+    ],
+  },
+  refactorer: {
+    role: 'Code modernizer who restructures without changing behavior',
+    claudeCodeContext: 'Uses Edit, MultiEdit tools to restructure code across files',
+    keywords: ['refactor', 'restructure', 'modernize', 'extract', 'consolidate', 'simplify'],
+    examples: [
+      'Refactor the payment module to async/await',
+      'Restructure the utils folder',
+      'Extract common logic into shared module',
+    ],
+  },
+  optimizer: {
+    role: 'Performance engineer who speeds up slow code',
+    claudeCodeContext: 'Uses Bash to run profilers and Edit to optimize code',
+    keywords: ['optimize', 'performance', 'speed', 'cache', 'latency', 'slow', 'fast'],
+    examples: [
+      'Optimize the database queries for dashboard',
+      'Cache the frequently accessed user data',
+      'Improve the API response time',
+    ],
+  },
+  devops: {
+    role: 'DevOps engineer who manages deployment and infrastructure',
+    claudeCodeContext: 'Uses Bash for deployment commands and Write for config files',
+    keywords: ['deploy', 'ci/cd', 'kubernetes', 'docker', 'pipeline', 'infrastructure'],
+    examples: [
+      'Set up the CI/CD pipeline',
+      'Configure Kubernetes deployment',
+      'Deploy to production',
+    ],
+  },
+  'api-docs': {
+    role: 'API documentation specialist who creates specs',
+    claudeCodeContext: 'Uses Write to generate OpenAPI/Swagger specs',
+    keywords: ['openapi', 'swagger', 'api spec', 'endpoint', 'rest api', 'graphql'],
+    examples: [
+      'Generate OpenAPI documentation for REST API',
+      'Create Swagger spec for the endpoints',
+      'Document the API request/response formats',
+    ],
+  },
+  planner: {
+    role: 'Project planner who organizes and schedules work',
+    claudeCodeContext: 'Uses TodoWrite tool to create and manage task lists',
+    keywords: ['plan', 'sprint', 'roadmap', 'milestone', 'estimate', 'schedule', 'prioritize'],
+    examples: [
+      'Create a sprint plan for next two weeks',
+      'Estimate the feature implementation effort',
+      'Plan the roadmap for Q3',
+    ],
+  },
+};
+
+// Prompt template for synthetic data generation
+const GENERATION_PROMPT = `You are generating training data for an AI agent routing system used in Claude Code (an AI coding assistant).
+
+## Task
+Generate ${EXAMPLES_PER_AGENT} diverse, realistic task descriptions that would be routed to the "${'{AGENT}'}" agent.
+
+## Agent Description
+Role: {ROLE}
+Claude Code Context: {CONTEXT}
+Key Indicators: {KEYWORDS}
+
+## Requirements
+1. Each task should be a realistic software engineering task
+2. Tasks should clearly indicate the agent type through action verbs and context
+3. Include variety in:
+   - Programming languages (TypeScript, Python, Rust, Go, etc.)
+   - Frameworks (React, Vue, Express, Django, etc.)
+   - Domains (web, mobile, backend, data, ML, etc.)
+   - Complexity levels (simple to complex)
+4. Tasks should be 5-20 words, clear and actionable
+5. Include edge cases that might be confused with other agents
+
+## Examples for this agent
+{EXAMPLES}
+
+## Output Format
+Return a JSON array of objects with this structure:
+[
+  {
+    "task": "The task description",
+    "quality": 0.8-1.0,
+    "difficulty": "easy|medium|hard",
+    "tags": ["relevant", "tags"]
+  }
+]
+
+Generate exactly ${EXAMPLES_PER_AGENT} unique tasks. Be creative and diverse.`;
+
+// Prompt for hard negatives
+const HARD_NEGATIVE_PROMPT = `You are generating hard negative examples for contrastive learning in an AI agent routing system.
+
+## Context
+We have an agent called "${'{AGENT}'}" with this role: {ROLE}
+
+We need tasks that SEEM like they might belong to this agent but actually belong to OTHER agents.
+These are "hard negatives" - confusing examples that help the model learn better boundaries.
+
+## Confusable Agents
+{CONFUSABLE_AGENTS}
+
+## Requirements
+1. Generate ${HARD_NEGATIVES_PER_AGENT} tasks that might be confused with "${'{AGENT}'}"
+2. Each task should actually belong to a DIFFERENT agent
+3. The confusion should be subtle but clear upon reflection
+4. Include the correct agent label
+
+## Output Format
+[
+  {
+    "task": "The confusing task description",
+    "appears_to_be": "${'{AGENT}'}",
+    "actually_is": "the_correct_agent",
+    "confusion_reason": "Why this might be confused"
+  }
+]`;
+
+/**
+ * Initialize Gemini client
+ */
+function getGeminiClient() {
+  const apiKey = process.env.GEMINI_API_KEY;
+  if (!apiKey) {
+    console.error('GEMINI_API_KEY environment variable required');
+    console.error('Set it with: export GEMINI_API_KEY=your_key');
+    process.exit(1);
+  }
+  return new GoogleGenerativeAI(apiKey);
+}
+
+/**
+ * Generate training data for an agent using Gemini
+ */
+async function generateAgentData(client, agent, agentConfig) {
+  console.log(`  Generating data for ${agent}...`);
+
+  const prompt = GENERATION_PROMPT
+    .replace(/\{AGENT\}/g, agent)
+    .replace('{ROLE}', agentConfig.role)
+    .replace('{CONTEXT}', agentConfig.claudeCodeContext)
+    .replace('{KEYWORDS}', agentConfig.keywords.join(', '))
+    .replace('{EXAMPLES}', agentConfig.examples.map(e => `- ${e}`).join('\n'));
+
+  try {
+    const model = client.getGenerativeModel({ model: 'gemini-2.0-flash-exp' });
+    const result = await model.generateContent(prompt);
+    const response = result.response.text();
+
+    // Extract JSON from response
+    const jsonMatch = response.match(/\[[\s\S]*\]/);
+    if (!jsonMatch) {
+      console.error(`  Failed to parse JSON for ${agent}`);
+      return [];
+    }
+
+    const data = JSON.parse(jsonMatch[0]);
+    console.log(`  Generated ${data.length} examples for ${agent}`);
+
+    return data.map(item => ({
+      ...item,
+      agent,
+      type: 'positive',
+    }));
+  } catch (error) {
+    console.error(`  Error generating data for ${agent}: ${error.message}`);
+    return [];
+  }
+}
+
+/**
+ * Generate hard negatives for an agent
+ */
+async function generateHardNegatives(client, agent, agentConfig, allAgents) {
+  console.log(`  Generating hard negatives for ${agent}...`);
+
+  // Find confusable agents
+  const confusableAgents = Object.entries(allAgents)
+    .filter(([name]) => name !== agent)
+    .map(([name, config]) => `- ${name}: ${config.role}`)
+    .join('\n');
+
+  const prompt = HARD_NEGATIVE_PROMPT
+    .replace(/\{AGENT\}/g, agent)
+    .replace('{ROLE}', agentConfig.role)
+    .replace('{CONFUSABLE_AGENTS}', confusableAgents);
+
+  try {
+    const model = client.getGenerativeModel({ model: 'gemini-2.0-flash-exp' });
+    const result = await model.generateContent(prompt);
+    const response = result.response.text();
+
+    const jsonMatch = response.match(/\[[\s\S]*\]/);
+    if (!jsonMatch) {
+      console.error(`  Failed to parse hard negatives for ${agent}`);
+      return [];
+    }
+
+    const data = JSON.parse(jsonMatch[0]);
+    console.log(`  Generated ${data.length} hard negatives for ${agent}`);
+
+    return data.map(item => ({
+      task: item.task,
+      agent: item.actually_is,
+      confusing_with: agent,
+      confusion_reason: item.confusion_reason,
+      type: 'hard_negative',
+      quality: 1.0,
+    }));
+  } catch (error) {
+    console.error(`  Error generating hard negatives for ${agent}: ${error.message}`);
+    return [];
+  }
+}
+
+/**
+ * Main generation pipeline
+ */
+async function main() {
+  console.log('\n╔═══════════════════════════════════════════════════════════════════════════════════╗');
+  console.log('║          CLAUDE CODE SYNTHETIC TRAINING DATA GENERATOR                            ║');
+  console.log('║                     Using @ruvector/agentic-synth                                 ║');
+  console.log('╚═══════════════════════════════════════════════════════════════════════════════════╝\n');
+
+  // Check for API key
+  if (!process.env.GEMINI_API_KEY) {
+    console.log('GEMINI_API_KEY not set. Generating static dataset from templates...\n');
+    generateStaticDataset();
+    return;
+  }
+
+  const client = getGeminiClient();
+
+  // Create output directory
+  if (!existsSync(OUTPUT_DIR)) {
+    mkdirSync(OUTPUT_DIR, { recursive: true });
+  }
+
+  const allData = [];
+  const allHardNegatives = [];
+  const agents = Object.keys(CLAUDE_CODE_AGENTS);
+
+  console.log('─────────────────────────────────────────────────────────────────');
+  console.log('                   GENERATING POSITIVE EXAMPLES');
+  console.log('─────────────────────────────────────────────────────────────────\n');
+
+  // Generate positive examples for each agent
+  for (const agent of agents) {
+    const data = await generateAgentData(client, agent, CLAUDE_CODE_AGENTS[agent]);
+    allData.push(...data);
+
+    // Rate limit
+    await new Promise(resolve => setTimeout(resolve, 1000));
+  }
+
+  console.log('\n─────────────────────────────────────────────────────────────────');
+  console.log('                   GENERATING HARD NEGATIVES');
+  console.log('─────────────────────────────────────────────────────────────────\n');
+
+  // Generate hard negatives
+  for (const agent of agents) {
+    const negatives = await generateHardNegatives(client, agent, CLAUDE_CODE_AGENTS[agent], CLAUDE_CODE_AGENTS);
+    allHardNegatives.push(...negatives);
+
+    // Rate limit
+    await new Promise(resolve => setTimeout(resolve, 1000));
+  }
+
+  // Combine and save
+  const fullDataset = [...allData, ...allHardNegatives];
+
+  // Save full dataset
+  const outputPath = join(OUTPUT_DIR, 'claude-code-routing-dataset.json');
+  writeFileSync(outputPath, JSON.stringify(fullDataset, null, 2));
+
+  // Save training pairs (for contrastive learning)
+  const contrastivePairs = generateContrastivePairs(allData, allHardNegatives);
+  const pairsPath = join(OUTPUT_DIR, 'contrastive-pairs.json');
+  writeFileSync(pairsPath, JSON.stringify(contrastivePairs, null, 2));
+
+  // Print summary
+  console.log('\n═══════════════════════════════════════════════════════════════════════════════════');
+  console.log('                              GENERATION COMPLETE');
+  console.log('═══════════════════════════════════════════════════════════════════════════════════\n');
+
+  console.log(`  Positive examples:    ${allData.length}`);
+  console.log(`  Hard negatives:       ${allHardNegatives.length}`);
+  console.log(`  Contrastive pairs:    ${contrastivePairs.length}`);
+  console.log(`  Total dataset size:   ${fullDataset.length}`);
+  console.log(`\n  Output files:`);
+  console.log(`    ${outputPath}`);
+  console.log(`    ${pairsPath}`);
+  console.log('');
+}
+
+/**
+ * Generate contrastive pairs from data
+ */
+function generateContrastivePairs(positives, negatives) {
+  const pairs = [];
+
+  // Group positives by agent
+  const byAgent = {};
+  for (const item of positives) {
+    if (!byAgent[item.agent]) byAgent[item.agent] = [];
+    byAgent[item.agent].push(item);
+  }
+
+  // Create positive pairs (same agent)
+  for (const [agent, items] of Object.entries(byAgent)) {
+    for (let i = 0; i < items.length - 1; i++) {
+      for (let j = i + 1; j < Math.min(i + 3, items.length); j++) {
+        pairs.push({
+          anchor: items[i].task,
+          positive: items[j].task,
+          agent,
+          type: 'positive_pair',
+        });
+      }
+    }
+  }
+
+  // Create negative pairs (different agents)
+  const agents = Object.keys(byAgent);
+  for (let i = 0; i < agents.length; i++) {
+    for (let j = i + 1; j < agents.length; j++) {
+      const agent1Items = byAgent[agents[i]];
+      const agent2Items = byAgent[agents[j]];
+
+      if (agent1Items && agent1Items[0] && agent2Items && agent2Items[0]) {
+        pairs.push({
+          anchor: agent1Items[0].task,
+          negative: agent2Items[0].task,
+          anchor_agent: agents[i],
+          negative_agent: agents[j],
+          type: 'negative_pair',
+        });
+      }
+    }
+  }
+
+  // Add hard negative pairs
+  for (const neg of negatives) {
+    const confusingAgent = byAgent[neg.confusing_with];
+    if (confusingAgent && confusingAgent[0]) {
+      pairs.push({
+        anchor: confusingAgent[0].task,
+        negative: neg.task,
+        anchor_agent: neg.confusing_with,
+        negative_agent: neg.agent,
+        type: 'hard_negative_pair',
+        confusion_reason: neg.confusion_reason,
+      });
+    }
+  }
+
+  return pairs;
+}
+
+/**
+ * Generate static dataset without API (fallback)
+ */
+function generateStaticDataset() {
+  console.log('Generating static dataset from routing-dataset.js...\n');
+
+  // Import the static dataset
+  const { generateTrainingDataset, generateContrastivePairs, getDatasetStats } = require('./routing-dataset.js');
+
+  const dataset = generateTrainingDataset();
+  const pairs = generateContrastivePairs();
+  const stats = getDatasetStats();
+
+  // Create output directory
+  if (!existsSync(OUTPUT_DIR)) {
+    mkdirSync(OUTPUT_DIR, { recursive: true });
+  }
+
+  // Save dataset
+  const datasetPath = join(OUTPUT_DIR, 'claude-code-routing-dataset.json');
+  writeFileSync(datasetPath, JSON.stringify(dataset, null, 2));
+
+  const pairsPath = join(OUTPUT_DIR, 'contrastive-pairs.json');
+  writeFileSync(pairsPath, JSON.stringify(pairs, null, 2));
+
+  console.log('═══════════════════════════════════════════════════════════════');
+  console.log('                    STATIC DATASET GENERATED');
+  console.log('═══════════════════════════════════════════════════════════════\n');
+
+  console.log(`  Total examples:       ${stats.totalExamples}`);
+  console.log(`  Contrastive pairs:    ${stats.contrastivePairs}`);
+  console.log(`  Agent types:          ${stats.agents.length}`);
+  console.log(`\n  Output files:`);
+  console.log(`    ${datasetPath}`);
+  console.log(`    ${pairsPath}`);
+  console.log('\n  To generate more data with AI, set GEMINI_API_KEY');
+  console.log('');
+}
+
+main().catch(console.error);