@cogitator-ai/self-modifying 0.1.0

package/src/architecture-evolution/parameter-optimizer.ts

@@ -0,0 +1,489 @@
+import type {
+  LLMBackend,
+  TaskProfile,
+  ArchitectureConfig,
+  EvolutionCandidate,
+  ArchitectureEvolutionConfig,
+  EvolutionMetrics,
+} from '@cogitator-ai/types';
+import { CapabilityAnalyzer } from './capability-analyzer';
+import { EvolutionStrategy, type SelectionResult } from './evolution-strategy';
+import {
+  buildCandidateGenerationPrompt,
+  buildPerformanceAnalysisPrompt,
+  parseCandidateGenerationResponse,
+  parsePerformanceAnalysisResponse,
+} from './prompts';
+
+export interface ParameterOptimizerOptions {
+  llm: LLMBackend;
+  config: ArchitectureEvolutionConfig;
+  baseConfig: ArchitectureConfig;
+}
+
+export interface OptimizationResult {
+  recommendedConfig: ArchitectureConfig;
+  candidate: EvolutionCandidate | null;
+  metrics: EvolutionMetrics;
+  shouldAdopt: boolean;
+  confidence: number;
+  reasoning: string;
+}
+
+interface HistoricalRecord {
+  taskProfile: TaskProfile;
+  config: Partial<ArchitectureConfig>;
+  score: number;
+  metrics: Record<string, number>;
+  timestamp: number;
+}
+
+export class ParameterOptimizer {
+  private readonly llm: LLMBackend;
+  private readonly config: ArchitectureEvolutionConfig;
+  private readonly baseConfig: ArchitectureConfig;
+  private readonly capabilityAnalyzer: CapabilityAnalyzer;
+  private readonly evolutionStrategy: EvolutionStrategy;
+
+  private candidates: EvolutionCandidate[] = [];
+  private history: HistoricalRecord[] = [];
+  private currentGeneration = 0;
+  private readonly maxHistorySize = 100;
+
+  constructor(options: ParameterOptimizerOptions) {
+    this.llm = options.llm;
+    this.config = options.config;
+    this.baseConfig = options.baseConfig;
+
+    this.capabilityAnalyzer = new CapabilityAnalyzer({
+      llm: options.llm,
+      enableLLMAnalysis: true,
+    });
+
+    this.evolutionStrategy = new EvolutionStrategy({
+      strategy: options.config.strategy,
+      explorationBonus: 2.0,
+    });
+  }
+
+  async optimize(taskDescription: string): Promise<OptimizationResult> {
+    const profile = await this.capabilityAnalyzer.analyzeTask(taskDescription);
+
+    if (this.candidates.length === 0) {
+      await this.generateInitialCandidates(profile);
+    }
+
+    const selection = this.evolutionStrategy.select(this.candidates);
+
+    const mergedConfig = this.mergeConfigs(this.baseConfig, selection.candidate.config);
+
+    return {
+      recommendedConfig: mergedConfig,
+      candidate: selection.candidate,
+      metrics: this.calculateMetrics(),
+      shouldAdopt: true,
+      confidence: selection.score,
+      reasoning: selection.reasoning,
+    };
+  }
+
+  async recordOutcome(
+    candidateId: string,
+    taskProfile: TaskProfile,
+    metrics: {
+      successRate: number;
+      latency: number;
+      tokenUsage: number;
+      qualityScore: number;
+    }
+  ): Promise<void> {
+    const candidate = this.candidates.find((c) => c.id === candidateId);
+    if (!candidate) return;
+
+    const reward = this.calculateReward(metrics);
+    this.evolutionStrategy.updateCandidate(candidate, reward);
+
+    this.history.push({
+      taskProfile,
+      config: candidate.config,
+      score: reward,
+      metrics: {
+        successRate: metrics.successRate,
+        latency: metrics.latency,
+        tokenUsage: metrics.tokenUsage,
+        qualityScore: metrics.qualityScore,
+      },
+      timestamp: Date.now(),
+    });
+
+    if (this.history.length > this.maxHistorySize) {
+      this.history = this.history.slice(-this.maxHistorySize);
+    }
+
+    if (this.shouldEvolve()) {
+      await this.evolve(taskProfile);
+    }
+  }
+
+  private async generateInitialCandidates(profile: TaskProfile): Promise<void> {
+    const prompt = buildCandidateGenerationPrompt(
+      profile,
+      this.baseConfig,
+      this.getRelevantHistory(profile)
+    );
+
+    try {
+      const response = await this.callLLM([
+        {
+          role: 'system',
+          content: 'You are an AI architecture optimizer. Generate candidate configurations.',
+        },
+        { role: 'user', content: prompt },
+      ], 0.5);
+
+      const generated = parseCandidateGenerationResponse(response.content);
+
+      this.candidates = [
+        this.createBaselineCandidate(),
+        ...generated.map((c) => ({
+          ...c,
+          generation: this.currentGeneration,
+        })),
+      ];
+    } catch {
+      this.candidates = [this.createBaselineCandidate()];
+    }
+  }
+
+  private createBaselineCandidate(): EvolutionCandidate {
+    return {
+      id: 'baseline',
+      config: {},
+      reasoning: 'Baseline configuration with no modifications',
+      expectedImprovement: 0,
+      risk: 'low',
+      generation: 0,
+      score: 0.5,
+      evaluationCount: 0,
+    };
+  }
+
+  private async evolve(currentProfile: TaskProfile): Promise<void> {
+    this.currentGeneration++;
+
+    const topCandidates = [...this.candidates]
+      .filter((c) => c.evaluationCount > 0)
+      .sort((a, b) => b.score - a.score)
+      .slice(0, 3);
+
+    if (topCandidates.length === 0) return;
+
+    const newCandidates = await this.generateOffspring(topCandidates, currentProfile);
+
+    const keepCount = Math.max(3, Math.floor(this.candidates.length * 0.5));
+    const survivingCandidates = [...this.candidates]
+      .sort((a, b) => {
+        const scoreA = a.score + (a.evaluationCount > 0 ? 0 : 0.1);
+        const scoreB = b.score + (b.evaluationCount > 0 ? 0 : 0.1);
+        return scoreB - scoreA;
+      })
+      .slice(0, keepCount);
+
+    this.candidates = [...survivingCandidates, ...newCandidates];
+
+    const maxCandidates = this.config.maxCandidates ?? 10;
+    if (this.candidates.length > maxCandidates) {
+      this.candidates = this.candidates.slice(0, maxCandidates);
+    }
+  }
+
+  private async generateOffspring(
+    parents: EvolutionCandidate[],
+    profile: TaskProfile
+  ): Promise<EvolutionCandidate[]> {
+    const offspring: EvolutionCandidate[] = [];
+
+    for (const parent of parents) {
+      const mutated = this.mutateConfig(parent.config);
+      offspring.push({
+        id: `gen${this.currentGeneration}_${offspring.length}`,
+        config: mutated,
+        reasoning: `Mutation of ${parent.id}`,
+        expectedImprovement: parent.score * 0.9,
+        risk: 'medium',
+        generation: this.currentGeneration,
+        score: 0,
+        evaluationCount: 0,
+      });
+    }
+
+    if (parents.length >= 2) {
+      const crossed = this.crossover(parents[0].config, parents[1].config);
+      offspring.push({
+        id: `gen${this.currentGeneration}_crossover`,
+        config: crossed,
+        reasoning: `Crossover of ${parents[0].id} and ${parents[1].id}`,
+        expectedImprovement: (parents[0].score + parents[1].score) / 2,
+        risk: 'medium',
+        generation: this.currentGeneration,
+        score: 0,
+        evaluationCount: 0,
+      });
+    }
+
+    return offspring;
+  }
+
+  private mutateConfig(config: Partial<ArchitectureConfig>): Partial<ArchitectureConfig> {
+    const mutated = { ...config };
+    const mutationRate = 0.3;
+
+    if (Math.random() < mutationRate && mutated.temperature !== undefined) {
+      mutated.temperature = Math.max(0, Math.min(2, mutated.temperature + (Math.random() - 0.5) * 0.3));
+    }
+
+    if (Math.random() < mutationRate && mutated.maxTokens !== undefined) {
+      const delta = Math.floor((Math.random() - 0.5) * 1000);
+      mutated.maxTokens = Math.max(100, Math.min(32000, mutated.maxTokens + delta));
+    }
+
+    if (Math.random() < mutationRate && mutated.reflectionDepth !== undefined) {
+      mutated.reflectionDepth = Math.max(0, Math.min(5, mutated.reflectionDepth + Math.round(Math.random() * 2 - 1)));
+    }
+
+    return mutated;
+  }
+
+  private crossover(
+    configA: Partial<ArchitectureConfig>,
+    configB: Partial<ArchitectureConfig>
+  ): Partial<ArchitectureConfig> {
+    const result: Partial<ArchitectureConfig> = {};
+
+    const allKeys = new Set([...Object.keys(configA), ...Object.keys(configB)]);
+
+    for (const key of allKeys) {
+      const typedKey = key as keyof ArchitectureConfig;
+      if (Math.random() < 0.5 && configA[typedKey] !== undefined) {
+        (result as Record<string, unknown>)[key] = configA[typedKey];
+      } else if (configB[typedKey] !== undefined) {
+        (result as Record<string, unknown>)[key] = configB[typedKey];
+      }
+    }
+
+    return result;
+  }
+
+  private calculateReward(metrics: {
+    successRate: number;
+    latency: number;
+    tokenUsage: number;
+    qualityScore: number;
+  }): number {
+    const weights = {
+      success: 0.4,
+      quality: 0.3,
+      latency: 0.15,
+      tokens: 0.15,
+    };
+
+    const successScore = metrics.successRate;
+    const qualityScore = metrics.qualityScore;
+    const latencyScore = Math.max(0, 1 - metrics.latency / 30000);
+    const tokenScore = Math.max(0, 1 - metrics.tokenUsage / 10000);
+
+    return (
+      weights.success * successScore +
+      weights.quality * qualityScore +
+      weights.latency * latencyScore +
+      weights.tokens * tokenScore
+    );
+  }
+
+  private shouldEvolve(): boolean {
+    const minEvaluations = this.candidates.reduce(
+      (min, c) => Math.min(min, c.evaluationCount),
+      Infinity
+    );
+
+    if (minEvaluations < 3) return false;
+
+    const evaluationsSinceEvolution = this.history.filter(
+      (h) => h.timestamp > this.getLastEvolutionTime()
+    ).length;
+
+    return evaluationsSinceEvolution >= 5;
+  }
+
+  private getLastEvolutionTime(): number {
+    const latestGenCandidate = this.candidates.find(
+      (c) => c.generation === this.currentGeneration && c.evaluationCount === 0
+    );
+    return latestGenCandidate ? Date.now() - 300000 : 0;
+  }
+
+  private getRelevantHistory(profile: TaskProfile): HistoricalRecord[] {
+    return this.history
+      .filter((h) => h.taskProfile.domain === profile.domain || h.taskProfile.complexity === profile.complexity)
+      .slice(-10);
+  }
+
+  private mergeConfigs(
+    base: ArchitectureConfig,
+    overrides: Partial<ArchitectureConfig>
+  ): ArchitectureConfig {
+    return {
+      ...base,
+      ...Object.fromEntries(
+        Object.entries(overrides).filter(([, v]) => v !== undefined && v !== null)
+      ),
+    } as ArchitectureConfig;
+  }
+
+  private calculateMetrics(): EvolutionMetrics {
+    const evaluatedCandidates = this.candidates.filter((c) => c.evaluationCount > 0);
+
+    if (evaluatedCandidates.length === 0) {
+      return {
+        generation: this.currentGeneration,
+        bestScore: 0,
+        averageScore: 0,
+        diversity: 1,
+        convergenceRate: 0,
+      };
+    }
+
+    const scores = evaluatedCandidates.map((c) => c.score);
+    const bestScore = Math.max(...scores);
+    const averageScore = scores.reduce((a, b) => a + b, 0) / scores.length;
+    const diversity = this.calculateDiversity();
+    const convergenceRate = this.calculateConvergenceRate();
+
+    return {
+      generation: this.currentGeneration,
+      bestScore,
+      averageScore,
+      diversity,
+      convergenceRate,
+    };
+  }
+
+  private calculateDiversity(): number {
+    if (this.candidates.length < 2) return 1;
+
+    let totalDiff = 0;
+    let comparisons = 0;
+
+    for (let i = 0; i < this.candidates.length; i++) {
+      for (let j = i + 1; j < this.candidates.length; j++) {
+        totalDiff += this.configDistance(this.candidates[i].config, this.candidates[j].config);
+        comparisons++;
+      }
+    }
+
+    return comparisons > 0 ? totalDiff / comparisons : 0;
+  }
+
+  private configDistance(a: Partial<ArchitectureConfig>, b: Partial<ArchitectureConfig>): number {
+    let distance = 0;
+    let fields = 0;
+
+    if (a.temperature !== undefined && b.temperature !== undefined) {
+      distance += Math.abs(a.temperature - b.temperature) / 2;
+      fields++;
+    }
+
+    if (a.maxTokens !== undefined && b.maxTokens !== undefined) {
+      distance += Math.abs(a.maxTokens - b.maxTokens) / 32000;
+      fields++;
+    }
+
+    if (a.model !== b.model) {
+      distance += 1;
+      fields++;
+    }
+
+    return fields > 0 ? distance / fields : 0;
+  }
+
+  private calculateConvergenceRate(): number {
+    if (this.history.length < 10) return 0;
+
+    const recentScores = this.history.slice(-10).map((h) => h.score);
+    const variance = this.calculateVariance(recentScores);
+
+    return Math.max(0, 1 - variance * 4);
+  }
+
+  private calculateVariance(values: number[]): number {
+    if (values.length === 0) return 0;
+    const mean = values.reduce((a, b) => a + b, 0) / values.length;
+    const squaredDiffs = values.map((v) => Math.pow(v - mean, 2));
+    return squaredDiffs.reduce((a, b) => a + b, 0) / values.length;
+  }
+
+  async analyzePerformance(): Promise<{
+    recommendation: string;
+    confidence: number;
+    analysis: string;
+  } | null> {
+    if (this.candidates.length === 0 || this.history.length < 5) {
+      return null;
+    }
+
+    const evaluatedCandidates = this.candidates.filter((c) => c.evaluationCount > 0);
+    if (evaluatedCandidates.length === 0) return null;
+
+    const results = evaluatedCandidates.map((c) => {
+      const records = this.history.filter((h) => JSON.stringify(h.config) === JSON.stringify(c.config));
+      const avgMetrics = records.length > 0
+        ? {
+            successRate: records.reduce((s, r) => s + r.metrics.successRate, 0) / records.length,
+            avgLatency: records.reduce((s, r) => s + r.metrics.latency, 0) / records.length,
+            avgTokens: records.reduce((s, r) => s + r.metrics.tokenUsage, 0) / records.length,
+            qualityScore: records.reduce((s, r) => s + r.metrics.qualityScore, 0) / records.length,
+          }
+        : { successRate: 0, avgLatency: 0, avgTokens: 0, qualityScore: 0 };
+
+      return { candidateId: c.id, metrics: avgMetrics };
+    });
+
+    const prompt = buildPerformanceAnalysisPrompt(evaluatedCandidates, results);
+
+    try {
+      const response = await this.callLLM([
+        { role: 'system', content: 'You are an AI performance analyst.' },
+        { role: 'user', content: prompt },
+      ], 0.2);
+
+      return parsePerformanceAnalysisResponse(response.content);
+    } catch {
+      return null;
+    }
+  }
+
+  private async callLLM(
+    messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>,
+    temperature: number
+  ) {
+    if (this.llm.complete) {
+      return this.llm.complete({ messages, temperature });
+    }
+    return this.llm.chat({ model: 'default', messages, temperature });
+  }
+
+  getCandidates(): EvolutionCandidate[] {
+    return [...this.candidates];
+  }
+
+  getHistory(): HistoricalRecord[] {
+    return [...this.history];
+  }
+
+  reset(): void {
+    this.candidates = [];
+    this.history = [];
+    this.currentGeneration = 0;
+    this.evolutionStrategy.reset();
+  }
+}

package/src/architecture-evolution/prompts.ts

@@ -0,0 +1,216 @@
+import type { TaskProfile, ArchitectureConfig, EvolutionCandidate } from '@cogitator-ai/types';
+
+export const ARCHITECTURE_ANALYSIS_SYSTEM_PROMPT = `You are an expert in AI agent architecture optimization.
+Your task is to analyze tasks and recommend optimal configurations.
+
+Consider these factors:
+1. Task complexity - simple tasks need lighter models, complex need stronger
+2. Domain requirements - some domains need specific capabilities
+3. Resource constraints - balance performance vs cost
+4. Historical performance - learn from past executions
+
+Always provide structured JSON responses.`;
+
+export function buildTaskProfilePrompt(
+  taskDescription: string,
+  context?: {
+    previousTasks?: string[];
+    availableModels?: string[];
+    constraints?: { maxCost?: number; maxLatency?: number };
+  }
+): string {
+  const contextSection = context
+    ? `
+CONTEXT:
+${context.previousTasks?.length ? `Previous tasks: ${context.previousTasks.slice(-3).join(', ')}` : ''}
+${context.availableModels?.length ? `Available models: ${context.availableModels.join(', ')}` : ''}
+${context.constraints?.maxCost ? `Max cost: $${context.constraints.maxCost}` : ''}
+${context.constraints?.maxLatency ? `Max latency: ${context.constraints.maxLatency}ms` : ''}`
+    : '';
+
+  return `Analyze the following task and create a profile for architecture optimization.
+
+TASK:
+${taskDescription}
+${contextSection}
+
+Respond with a JSON object:
+{
+  "complexity": "trivial" | "simple" | "moderate" | "complex" | "extreme",
+  "domain": "general" | "coding" | "reasoning" | "creative" | "factual" | "conversational",
+  "estimatedTokens": number,
+  "requiresTools": boolean,
+  "toolIntensity": "none" | "light" | "moderate" | "heavy",
+  "reasoningDepth": "shallow" | "moderate" | "deep" | "exhaustive",
+  "creativityLevel": "low" | "moderate" | "high",
+  "accuracyRequirement": "approximate" | "moderate" | "high" | "critical",
+  "timeConstraint": "none" | "relaxed" | "moderate" | "strict",
+  "suggestedApproach": "Brief description of recommended approach",
+  "riskFactors": ["List of potential challenges"]
+}`;
+}
+
+export function buildCandidateGenerationPrompt(
+  profile: TaskProfile,
+  currentConfig: ArchitectureConfig,
+  historicalPerformance?: Array<{
+    config: Partial<ArchitectureConfig>;
+    score: number;
+    metrics: Record<string, number>;
+  }>
+): string {
+  const historySection = historicalPerformance?.length
+    ? `
+HISTORICAL PERFORMANCE:
+${historicalPerformance
+    .slice(-5)
+    .map(
+      (h, i) =>
+        `${i + 1}. Config: ${JSON.stringify(h.config)} → Score: ${h.score.toFixed(2)}, Metrics: ${JSON.stringify(h.metrics)}`
+    )
+    .join('\n')}`
+    : '';
+
+  return `Generate candidate configurations for architecture evolution.
+
+TASK PROFILE:
+${JSON.stringify(profile, null, 2)}
+
+CURRENT CONFIG:
+${JSON.stringify(currentConfig, null, 2)}
+${historySection}
+
+Generate 3-5 candidate configurations that might improve performance.
+Each candidate should modify 1-3 parameters from current config.
+
+Respond with a JSON array:
+[
+  {
+    "id": "candidate_1",
+    "config": {
+      "model": "model name or null to keep current",
+      "temperature": number or null,
+      "maxTokens": number or null,
+      "systemPromptAdditions": "additional instructions or null",
+      "toolStrategy": "sequential" | "parallel" | "adaptive" | null,
+      "reflectionDepth": number or null
+    },
+    "reasoning": "Why this configuration might help",
+    "expectedImprovement": 0.0-1.0,
+    "risk": "low" | "medium" | "high"
+  }
+]`;
+}
+
+export function buildPerformanceAnalysisPrompt(
+  candidates: EvolutionCandidate[],
+  results: Array<{
+    candidateId: string;
+    metrics: {
+      successRate: number;
+      avgLatency: number;
+      avgTokens: number;
+      qualityScore: number;
+    };
+  }>
+): string {
+  return `Analyze the performance of architecture candidates and recommend the best configuration.
+
+CANDIDATES AND RESULTS:
+${candidates
+    .map((c) => {
+      const result = results.find((r) => r.candidateId === c.id);
+      return `
+${c.id}:
+  Config: ${JSON.stringify(c.config)}
+  Reasoning: ${c.reasoning}
+  ${result ? `Results: ${JSON.stringify(result.metrics)}` : 'Not yet tested'}`;
+    })
+    .join('\n')}
+
+Respond with:
+{
+  "recommendation": "candidate_id of best candidate",
+  "confidence": 0.0-1.0,
+  "analysis": "Detailed analysis of results",
+  "suggestedNextExperiments": ["List of additional experiments to try"],
+  "shouldAdopt": boolean,
+  "adoptionReason": "Why to adopt or not adopt the recommendation"
+}`;
+}
+
+export function parseTaskProfileResponse(response: string): TaskProfile | null {
+  const jsonMatch = response.match(/\{[\s\S]*\}/);
+  if (!jsonMatch) return null;
+
+  try {
+    const parsed = JSON.parse(jsonMatch[0]);
+
+    return {
+      complexity: parsed.complexity || 'moderate',
+      domain: parsed.domain || 'general',
+      estimatedTokens: parsed.estimatedTokens || 1000,
+      requiresTools: Boolean(parsed.requiresTools),
+      toolIntensity: parsed.toolIntensity || 'none',
+      reasoningDepth: parsed.reasoningDepth || 'moderate',
+      creativityLevel: parsed.creativityLevel || 'moderate',
+      accuracyRequirement: parsed.accuracyRequirement || 'moderate',
+      timeConstraint: parsed.timeConstraint || 'none',
+    };
+  } catch {
+    return null;
+  }
+}
+
+export function parseCandidateGenerationResponse(
+  response: string
+): EvolutionCandidate[] {
+  const jsonMatch = response.match(/\[[\s\S]*\]/);
+  if (!jsonMatch) return [];
+
+  try {
+    const parsed = JSON.parse(jsonMatch[0]);
+    if (!Array.isArray(parsed)) return [];
+
+    return parsed
+      .filter((c: Record<string, unknown>) => c && typeof c === 'object' && c.config)
+      .map((c: Record<string, unknown>, idx: number) => ({
+        id: String(c.id || `candidate_${idx}`),
+        config: c.config as Partial<ArchitectureConfig>,
+        reasoning: String(c.reasoning || ''),
+        expectedImprovement: typeof c.expectedImprovement === 'number' ? c.expectedImprovement : 0.5,
+        risk: (['low', 'medium', 'high'].includes(String(c.risk)) ? c.risk : 'medium') as
+          | 'low'
+          | 'medium'
+          | 'high',
+        generation: 0,
+        score: 0,
+        evaluationCount: 0,
+      }));
+  } catch {
+    return [];
+  }
+}
+
+export function parsePerformanceAnalysisResponse(response: string): {
+  recommendation: string;
+  confidence: number;
+  shouldAdopt: boolean;
+  analysis: string;
+} | null {
+  const jsonMatch = response.match(/\{[\s\S]*\}/);
+  if (!jsonMatch) return null;
+
+  try {
+    const parsed = JSON.parse(jsonMatch[0]);
+
+    return {
+      recommendation: String(parsed.recommendation || ''),
+      confidence: typeof parsed.confidence === 'number' ? parsed.confidence : 0.5,
+      shouldAdopt: Boolean(parsed.shouldAdopt),
+      analysis: String(parsed.analysis || ''),
+    };
+  } catch {
+    return null;
+  }
+}