agentdb 3.0.0-alpha.2 → 3.0.0-alpha.4

package/src/controllers/ContextSynthesizer.ts

@@ -0,0 +1,285 @@
+/**
+ * Context Synthesis - Generate coherent narratives from multiple memories
+ *
+ * Takes a collection of retrieved episodes/patterns and synthesizes
+ * a coherent context summary with extracted patterns, success rates,
+ * and actionable insights.
+ */
+
+export interface MemoryPattern {
+  task: string;
+  reward: number;
+  success: boolean;
+  critique?: string;
+  input?: string;
+  output?: string;
+  similarity?: number;
+  [key: string]: any;
+}
+
+export interface SynthesizedContext {
+  summary: string;
+  patterns: string[];
+  successRate: number;
+  averageReward: number;
+  recommendations: string[];
+  keyInsights: string[];
+  totalMemories: number;
+}
+
+export class ContextSynthesizer {
+  /**
+   * Synthesize context from multiple memories
+   *
+   * @param memories - Retrieved episodes/patterns
+   * @param options - Synthesis options
+   * @returns Synthesized context with insights
+   */
+  static synthesize(
+    memories: MemoryPattern[],
+    options: {
+      minPatternFrequency?: number;
+      includeRecommendations?: boolean;
+      maxSummaryLength?: number;
+    } = {}
+  ): SynthesizedContext {
+    const minPatternFrequency = options.minPatternFrequency ?? 2;
+    const includeRecommendations = options.includeRecommendations ?? true;
+
+    if (memories.length === 0) {
+      return {
+        summary: 'No relevant memories found.',
+        patterns: [],
+        successRate: 0,
+        averageReward: 0,
+        recommendations: [],
+        keyInsights: [],
+        totalMemories: 0,
+      };
+    }
+
+    // Calculate statistics
+    const successCount = memories.filter(m => m.success).length;
+    const successRate = successCount / memories.length;
+    const averageReward = memories.reduce((sum, m) => sum + (m.reward || 0), 0) / memories.length;
+
+    // Extract common patterns from critiques
+    const patterns = this.extractPatterns(memories, minPatternFrequency);
+
+    // Generate key insights
+    const keyInsights = this.generateKeyInsights(memories, successRate, averageReward);
+
+    // Generate recommendations
+    const recommendations = includeRecommendations
+      ? this.generateRecommendations(memories, patterns, successRate)
+      : [];
+
+    // Generate summary narrative
+    const summary = this.generateSummary(memories, patterns, successRate, averageReward);
+
+    return {
+      summary,
+      patterns,
+      successRate,
+      averageReward,
+      recommendations,
+      keyInsights,
+      totalMemories: memories.length,
+    };
+  }
+
+  /**
+   * Extract common patterns from memory critiques
+   */
+  private static extractPatterns(memories: MemoryPattern[], minFrequency: number): string[] {
+    const phraseCount = new Map<string, number>();
+
+    // Extract meaningful phrases from critiques
+    for (const memory of memories) {
+      if (memory.critique) {
+        const phrases = this.extractPhrases(memory.critique);
+        for (const phrase of phrases) {
+          phraseCount.set(phrase, (phraseCount.get(phrase) || 0) + 1);
+        }
+      }
+    }
+
+    // Filter by minimum frequency
+    const patterns: string[] = [];
+    for (const [phrase, count] of phraseCount.entries()) {
+      if (count >= minFrequency) {
+        patterns.push(`${phrase} (${count}/${memories.length} times)`);
+      }
+    }
+
+    return patterns.slice(0, 10);  // Top 10 patterns
+  }
+
+  /**
+   * Extract meaningful phrases from text
+   */
+  private static extractPhrases(text: string): string[] {
+    // Simple phrase extraction (can be enhanced with NLP)
+    const sentences = text.split(/[.!?]/).filter(s => s.trim().length > 0);
+    const phrases: string[] = [];
+
+    for (const sentence of sentences) {
+      const trimmed = sentence.trim();
+      // Extract phrases with key action words
+      if (
+        trimmed.match(/\b(use|implement|fix|improve|add|create|design|test)\b/i) &&
+        trimmed.length > 10 &&
+        trimmed.length < 100
+      ) {
+        phrases.push(trimmed.toLowerCase());
+      }
+    }
+
+    return phrases;
+  }
+
+  /**
+   * Generate key insights from memories
+   */
+  private static generateKeyInsights(
+    memories: MemoryPattern[],
+    successRate: number,
+    averageReward: number
+  ): string[] {
+    const insights: string[] = [];
+
+    // Success rate insight
+    if (successRate >= 0.8) {
+      insights.push(`High success rate (${(successRate * 100).toFixed(0)}%) indicates strong pattern match`);
+    } else if (successRate < 0.5) {
+      insights.push(`Low success rate (${(successRate * 100).toFixed(0)}%) suggests exploring alternative approaches`);
+    }
+
+    // Reward insight
+    if (averageReward >= 0.8) {
+      insights.push(`High average reward (${averageReward.toFixed(2)}) shows effective past solutions`);
+    } else if (averageReward < 0.6) {
+      insights.push(`Moderate reward (${averageReward.toFixed(2)}) indicates room for improvement`);
+    }
+
+    // Identify high-performing memories
+    const topMemories = memories
+      .filter(m => m.success && (m.reward || 0) >= 0.9)
+      .slice(0, 3);
+
+    if (topMemories.length > 0) {
+      insights.push(`${topMemories.length} exemplary solution(s) found with reward ≥0.9`);
+    }
+
+    // Task diversity
+    const uniqueTasks = new Set(memories.map(m => m.task)).size;
+    if (uniqueTasks > 1) {
+      insights.push(`${uniqueTasks} different task types provide diverse perspectives`);
+    }
+
+    return insights;
+  }
+
+  /**
+   * Generate actionable recommendations
+   */
+  private static generateRecommendations(
+    memories: MemoryPattern[],
+    patterns: string[],
+    successRate: number
+  ): string[] {
+    const recommendations: string[] = [];
+
+    // Extract successful strategies
+    const successfulMemories = memories.filter(m => m.success && (m.reward || 0) >= 0.8);
+
+    if (successfulMemories.length > 0) {
+      recommendations.push('Apply strategies from high-reward solutions');
+    }
+
+    // Pattern-based recommendations
+    if (patterns.length > 0) {
+      recommendations.push(`Follow common patterns: ${patterns[0].split(' (')[0]}`);
+    }
+
+    // Success rate recommendations
+    if (successRate >= 0.7) {
+      recommendations.push('Previous approaches were effective - follow similar methodology');
+    } else {
+      recommendations.push('Consider alternative approaches given mixed past results');
+    }
+
+    // Add general recommendations
+    if (memories.length >= 5) {
+      recommendations.push('Sufficient historical data available for confident decision-making');
+    } else {
+      recommendations.push('Limited data - proceed with caution and validate assumptions');
+    }
+
+    return recommendations.slice(0, 5);
+  }
+
+  /**
+   * Generate narrative summary
+   */
+  private static generateSummary(
+    memories: MemoryPattern[],
+    patterns: string[],
+    successRate: number,
+    averageReward: number
+  ): string {
+    const parts: string[] = [];
+
+    // Opening
+    parts.push(`Based on ${memories.length} similar past ${memories.length === 1 ? 'experience' : 'experiences'}`);
+
+    // Success rate
+    const successPercent = (successRate * 100).toFixed(0);
+    if (successRate >= 0.7) {
+      parts.push(`with a high success rate of ${successPercent}%`);
+    } else {
+      parts.push(`with a ${successPercent}% success rate`);
+    }
+
+    // Average reward
+    parts.push(`and average reward of ${averageReward.toFixed(2)}`);
+
+    // Key patterns
+    if (patterns.length > 0) {
+      parts.push('. Common effective approaches include:');
+      const topPatterns = patterns.slice(0, 3).map(p => p.split(' (')[0]);
+      parts.push(topPatterns.map((p, i) => `${i + 1}) ${p}`).join(', '));
+    }
+
+    // High performers
+    const topMemories = memories.filter(m => m.success && (m.reward || 0) >= 0.9);
+    if (topMemories.length > 0) {
+      parts.push(`. ${topMemories.length} exemplary solution(s) achieved reward ≥0.9`);
+    }
+
+    parts.push('.');
+
+    return parts.join(' ');
+  }
+
+  /**
+   * Extract actionable steps from successful memories
+   */
+  static extractActionableSteps(memories: MemoryPattern[]): string[] {
+    const steps: string[] = [];
+    const successfulMemories = memories.filter(m => m.success && (m.reward || 0) >= 0.8);
+
+    for (const memory of successfulMemories) {
+      if (memory.critique) {
+        // Extract step-like phrases
+        const matches = memory.critique.match(/\b\d+\.\s+([^.!?]+)/g);
+        if (matches) {
+          steps.push(...matches.map(m => m.trim()));
+        }
+      }
+    }
+
+    // Remove duplicates and return top 5
+    return Array.from(new Set(steps)).slice(0, 5);
+  }
+}

package/src/controllers/EmbeddingService.ts

@@ -0,0 +1,161 @@
+/**
+ * EmbeddingService - Text Embedding Generation
+ *
+ * Handles text-to-vector embedding generation using various models.
+ * Supports both local (transformers.js) and remote (OpenAI, etc.) embeddings.
+ */
+
+export interface EmbeddingConfig {
+  model: string;
+  dimension: number;
+  provider: 'transformers' | 'openai' | 'local';
+  apiKey?: string;
+}
+
+export class EmbeddingService {
+  private config: EmbeddingConfig;
+  private pipeline: any; // transformers.js pipeline
+  private cache: Map<string, Float32Array>;
+
+  constructor(config: EmbeddingConfig) {
+    this.config = config;
+    this.cache = new Map();
+  }
+
+  /**
+   * Initialize the embedding service
+   */
+  async initialize(): Promise<void> {
+    if (this.config.provider === 'transformers') {
+      // Use transformers.js for local embeddings
+      try {
+        const transformers = await import('@xenova/transformers');
+
+        // Set Hugging Face token if available from environment
+        const hfToken = process.env.HUGGINGFACE_API_KEY || process.env.HF_TOKEN;
+        if (hfToken) {
+          // Set the token for Transformers.js to use
+          if (transformers.env && typeof transformers.env === 'object') {
+            (transformers.env as any).HF_TOKEN = hfToken;
+            console.log('🔑 Using Hugging Face API key from environment');
+          }
+        }
+
+        this.pipeline = await transformers.pipeline('feature-extraction', this.config.model);
+        console.log(`✅ Transformers.js loaded: ${this.config.model}`);
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        console.warn(`⚠️  Transformers.js initialization failed: ${errorMessage}`);
+        console.warn('   Falling back to mock embeddings for testing');
+        console.warn('   Set HUGGINGFACE_API_KEY environment variable for real embeddings');
+        this.pipeline = null;
+      }
+    }
+  }
+
+  /**
+   * Generate embedding for text
+   */
+  async embed(text: string): Promise<Float32Array> {
+    // Check cache
+    const cacheKey = `${this.config.model}:${text}`;
+    if (this.cache.has(cacheKey)) {
+      return this.cache.get(cacheKey)!;
+    }
+
+    let embedding: Float32Array;
+
+    if (this.config.provider === 'transformers' && this.pipeline) {
+      // Use transformers.js
+      const output = await this.pipeline(text, { pooling: 'mean', normalize: true });
+      embedding = new Float32Array(output.data);
+    } else if (this.config.provider === 'openai' && this.config.apiKey) {
+      // Use OpenAI API
+      embedding = await this.embedOpenAI(text);
+    } else {
+      // Mock embedding for testing
+      embedding = this.mockEmbedding(text);
+    }
+
+    // Cache result
+    if (this.cache.size > 10000) {
+      // Simple LRU: clear half the cache
+      const keysToDelete = Array.from(this.cache.keys()).slice(0, 5000);
+      keysToDelete.forEach(k => this.cache.delete(k));
+    }
+    this.cache.set(cacheKey, embedding);
+
+    return embedding;
+  }
+
+  /**
+   * Batch embed multiple texts
+   */
+  async embedBatch(texts: string[]): Promise<Float32Array[]> {
+    return Promise.all(texts.map(text => this.embed(text)));
+  }
+
+  /**
+   * Clear embedding cache
+   */
+  clearCache(): void {
+    this.cache.clear();
+  }
+
+  // ========================================================================
+  // Private Methods
+  // ========================================================================
+
+  private async embedOpenAI(text: string): Promise<Float32Array> {
+    const response = await fetch('https://api.openai.com/v1/embeddings', {
+      method: 'POST',
+      headers: {
+        'Authorization': `Bearer ${this.config.apiKey}`,
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify({
+        model: this.config.model,
+        input: text
+      })
+    });
+
+    const data: any = await response.json();
+    return new Float32Array(data.data[0].embedding);
+  }
+
+  private mockEmbedding(text: string): Float32Array {
+    // Simple deterministic mock embedding for testing
+    const embedding = new Float32Array(this.config.dimension);
+
+    // Handle null/undefined/empty text
+    if (!text || text.length === 0) {
+      return new Array(this.config.dimension).fill(0) as any as Float32Array;
+    }
+
+    // Use simple hash-based generation
+    let hash = 0;
+    for (let i = 0; i < text.length; i++) {
+      hash = ((hash << 5) - hash) + text.charCodeAt(i);
+      hash = hash & hash; // Convert to 32bit integer
+    }
+
+    // Fill embedding with pseudo-random values based on hash
+    for (let i = 0; i < this.config.dimension; i++) {
+      const seed = hash + i * 31;
+      embedding[i] = Math.sin(seed) * Math.cos(seed * 0.5);
+    }
+
+    // Normalize
+    let norm = 0;
+    for (let i = 0; i < embedding.length; i++) {
+      norm += embedding[i] * embedding[i];
+    }
+    norm = Math.sqrt(norm);
+
+    for (let i = 0; i < embedding.length; i++) {
+      embedding[i] /= norm;
+    }
+
+    return embedding;
+  }
+}

package/src/controllers/EnhancedEmbeddingService.ts

@@ -0,0 +1,143 @@
+/**
+ * Enhanced EmbeddingService with WASM Acceleration
+ *
+ * Extends the base EmbeddingService with WASM-accelerated batch operations
+ * and improved performance for large-scale embedding generation.
+ */
+
+import { EmbeddingService, EmbeddingConfig } from './EmbeddingService.js';
+import { WASMVectorSearch } from './WASMVectorSearch.js';
+import { cosineSimilarity } from '../utils/vector-math.js';
+
+export interface EnhancedEmbeddingConfig extends EmbeddingConfig {
+  enableWASM?: boolean;
+  enableBatchProcessing?: boolean;
+  batchSize?: number;
+}
+
+export class EnhancedEmbeddingService extends EmbeddingService {
+  private wasmSearch: WASMVectorSearch | null = null;
+  private enhancedConfig: EnhancedEmbeddingConfig;
+
+  constructor(config: EnhancedEmbeddingConfig) {
+    super(config);
+    this.enhancedConfig = {
+      enableWASM: true,
+      enableBatchProcessing: true,
+      batchSize: 100,
+      ...config,
+    };
+
+    if (this.enhancedConfig.enableWASM) {
+      this.initializeWASM();
+    }
+  }
+
+  /**
+   * Initialize WASM acceleration
+   */
+  private initializeWASM(): void {
+    const mockDb = {
+      prepare: () => ({ all: () => [], get: () => null, run: () => ({}) }),
+      exec: () => {},
+    };
+
+    this.wasmSearch = new WASMVectorSearch(mockDb, {
+      enableWASM: true,
+      batchSize: this.enhancedConfig.batchSize || 100,
+    });
+  }
+
+  /**
+   * Enhanced batch embedding with parallel processing
+   */
+  async embedBatch(texts: string[]): Promise<Float32Array[]> {
+    if (!this.enhancedConfig.enableBatchProcessing || texts.length < 10) {
+      return super.embedBatch(texts);
+    }
+
+    const batchSize = this.enhancedConfig.batchSize || 100;
+    const batches: string[][] = [];
+
+    // Split into batches
+    for (let i = 0; i < texts.length; i += batchSize) {
+      batches.push(texts.slice(i, i + batchSize));
+    }
+
+    // Process batches in parallel
+    const results = await Promise.all(
+      batches.map(batch => super.embedBatch(batch))
+    );
+
+    // Flatten results
+    return results.flat();
+  }
+
+  /**
+   * Calculate similarity between two texts using WASM acceleration
+   */
+  async similarity(textA: string, textB: string): Promise<number> {
+    const [embeddingA, embeddingB] = await Promise.all([
+      this.embed(textA),
+      this.embed(textB),
+    ]);
+
+    if (this.wasmSearch) {
+      return this.wasmSearch.cosineSimilarity(embeddingA, embeddingB);
+    }
+
+    // Fallback to manual calculation
+    return cosineSimilarity(embeddingA, embeddingB);
+  }
+
+  /**
+   * Find most similar texts from a corpus
+   */
+  async findMostSimilar(
+    query: string,
+    corpus: string[],
+    k: number = 5
+  ): Promise<Array<{ text: string; similarity: number; index: number }>> {
+    const queryEmbedding = await this.embed(query);
+    const corpusEmbeddings = await this.embedBatch(corpus);
+
+    let similarities: number[];
+
+    if (this.wasmSearch) {
+      similarities = this.wasmSearch.batchSimilarity(queryEmbedding, corpusEmbeddings);
+    } else {
+      similarities = corpusEmbeddings.map(emb =>
+        cosineSimilarity(queryEmbedding, emb)
+      );
+    }
+
+    // Create results with indices
+    const results = corpus.map((text, index) => ({
+      text,
+      similarity: similarities[index],
+      index,
+    }));
+
+    // Sort by similarity and take top k
+    results.sort((a, b) => b.similarity - a.similarity);
+    return results.slice(0, k);
+  }
+
+  /**
+   * Get service statistics
+   */
+  getStats(): {
+    cacheSize: number;
+    wasmEnabled: boolean;
+    simdEnabled: boolean;
+  } {
+    const wasmStats = this.wasmSearch?.getStats();
+
+    return {
+      cacheSize: (this as any).cache.size,
+      wasmEnabled: wasmStats?.wasmAvailable ?? false,
+      simdEnabled: wasmStats?.simdAvailable ?? false,
+    };
+  }
+
+}