agentdb 1.0.12 → 1.1.0

package/src/optimizations/BatchOperations.ts

@@ -0,0 +1,292 @@
+/**
+ * BatchOperations - Optimized Batch Processing for AgentDB
+ *
+ * Implements efficient batch operations:
+ * - Bulk inserts with transactions
+ * - Batch embedding generation
+ * - Parallel processing
+ * - Progress tracking
+ */
+
+import { Database } from 'better-sqlite3';
+import { EmbeddingService } from '../controllers/EmbeddingService';
+import { Episode } from '../controllers/ReflexionMemory';
+
+export interface BatchConfig {
+  batchSize: number;
+  parallelism: number;
+  progressCallback?: (progress: number, total: number) => void;
+}
+
+export class BatchOperations {
+  private db: Database;
+  private embedder: EmbeddingService;
+  private config: BatchConfig;
+
+  constructor(db: Database, embedder: EmbeddingService, config?: Partial<BatchConfig>) {
+    this.db = db;
+    this.embedder = embedder;
+    this.config = {
+      batchSize: 100,
+      parallelism: 4,
+      ...config
+    };
+  }
+
+  /**
+   * Bulk insert episodes with embeddings
+   */
+  async insertEpisodes(episodes: Episode[]): Promise<number> {
+    const totalBatches = Math.ceil(episodes.length / this.config.batchSize);
+    let completed = 0;
+
+    for (let i = 0; i < episodes.length; i += this.config.batchSize) {
+      const batch = episodes.slice(i, i + this.config.batchSize);
+
+      // Generate embeddings in parallel
+      const texts = batch.map(ep => this.buildEpisodeText(ep));
+      const embeddings = await this.embedder.embedBatch(texts);
+
+      // Insert with transaction
+      const transaction = this.db.transaction(() => {
+        const episodeStmt = this.db.prepare(`
+          INSERT INTO episodes (
+            session_id, task, input, output, critique, reward, success,
+            latency_ms, tokens_used, tags, metadata
+          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        `);
+
+        const embeddingStmt = this.db.prepare(`
+          INSERT INTO episode_embeddings (episode_id, embedding)
+          VALUES (?, ?)
+        `);
+
+        batch.forEach((episode, idx) => {
+          const result = episodeStmt.run(
+            episode.sessionId,
+            episode.task,
+            episode.input || null,
+            episode.output || null,
+            episode.critique || null,
+            episode.reward,
+            episode.success ? 1 : 0,
+            episode.latencyMs || null,
+            episode.tokensUsed || null,
+            episode.tags ? JSON.stringify(episode.tags) : null,
+            episode.metadata ? JSON.stringify(episode.metadata) : null
+          );
+
+          const episodeId = result.lastInsertRowid as number;
+          embeddingStmt.run(episodeId, Buffer.from(embeddings[idx].buffer));
+        });
+      });
+
+      transaction();
+
+      completed += batch.length;
+
+      if (this.config.progressCallback) {
+        this.config.progressCallback(completed, episodes.length);
+      }
+    }
+
+    return completed;
+  }
+
+  /**
+   * Bulk update embeddings for existing episodes
+   */
+  async regenerateEmbeddings(episodeIds?: number[]): Promise<number> {
+    let episodes: any[];
+
+    if (episodeIds) {
+      const placeholders = episodeIds.map(() => '?').join(',');
+      episodes = this.db.prepare(
+        `SELECT id, task, critique, output FROM episodes WHERE id IN (${placeholders})`
+      ).all(...episodeIds);
+    } else {
+      episodes = this.db.prepare(
+        'SELECT id, task, critique, output FROM episodes'
+      ).all();
+    }
+
+    let completed = 0;
+    const totalBatches = Math.ceil(episodes.length / this.config.batchSize);
+
+    for (let i = 0; i < episodes.length; i += this.config.batchSize) {
+      const batch = episodes.slice(i, i + this.config.batchSize);
+
+      // Generate embeddings
+      const texts = batch.map((ep: any) =>
+        [ep.task, ep.critique, ep.output].filter(Boolean).join('\n')
+      );
+      const embeddings = await this.embedder.embedBatch(texts);
+
+      // Update with transaction
+      const transaction = this.db.transaction(() => {
+        const stmt = this.db.prepare(`
+          INSERT OR REPLACE INTO episode_embeddings (episode_id, embedding)
+          VALUES (?, ?)
+        `);
+
+        batch.forEach((episode: any, idx: number) => {
+          stmt.run(episode.id, Buffer.from(embeddings[idx].buffer));
+        });
+      });
+
+      transaction();
+
+      completed += batch.length;
+
+      if (this.config.progressCallback) {
+        this.config.progressCallback(completed, episodes.length);
+      }
+    }
+
+    return completed;
+  }
+
+  /**
+   * Parallel batch processing with worker pool
+   */
+  async processInParallel<T, R>(
+    items: T[],
+    processor: (item: T) => Promise<R>
+  ): Promise<R[]> {
+    const results: R[] = [];
+    const chunks = this.chunkArray(items, this.config.parallelism);
+
+    for (const chunk of chunks) {
+      const chunkResults = await Promise.all(
+        chunk.map(item => processor(item))
+      );
+      results.push(...chunkResults);
+
+      if (this.config.progressCallback) {
+        this.config.progressCallback(results.length, items.length);
+      }
+    }
+
+    return results;
+  }
+
+  /**
+   * Bulk delete with conditions
+   */
+  bulkDelete(table: string, conditions: Record<string, any>): number {
+    const whereClause = Object.keys(conditions)
+      .map(key => `${key} = ?`)
+      .join(' AND ');
+
+    const values = Object.values(conditions);
+
+    const stmt = this.db.prepare(`DELETE FROM ${table} WHERE ${whereClause}`);
+    const result = stmt.run(...values);
+
+    return result.changes;
+  }
+
+  /**
+   * Bulk update with conditions
+   */
+  bulkUpdate(
+    table: string,
+    updates: Record<string, any>,
+    conditions: Record<string, any>
+  ): number {
+    const setClause = Object.keys(updates)
+      .map(key => `${key} = ?`)
+      .join(', ');
+
+    const whereClause = Object.keys(conditions)
+      .map(key => `${key} = ?`)
+      .join(' AND ');
+
+    const values = [...Object.values(updates), ...Object.values(conditions)];
+
+    const stmt = this.db.prepare(
+      `UPDATE ${table} SET ${setClause} WHERE ${whereClause}`
+    );
+    const result = stmt.run(...values);
+
+    return result.changes;
+  }
+
+  /**
+   * Vacuum and optimize database
+   */
+  optimize(): void {
+    console.log('🔧 Optimizing database...');
+
+    // Analyze tables for query planner
+    this.db.exec('ANALYZE');
+
+    // Rebuild indexes
+    const tables = this.db.prepare(`
+      SELECT name FROM sqlite_master
+      WHERE type='table' AND name NOT LIKE 'sqlite_%'
+    `).all() as any[];
+
+    for (const { name } of tables) {
+      this.db.exec(`REINDEX ${name}`);
+    }
+
+    // Vacuum to reclaim space
+    this.db.exec('VACUUM');
+
+    console.log('✅ Database optimized');
+  }
+
+  /**
+   * Get database statistics
+   */
+  getStats(): {
+    totalSize: number;
+    tableStats: Array<{
+      name: string;
+      rows: number;
+      size: number;
+    }>;
+  } {
+    const pageSize = this.db.pragma('page_size', { simple: true }) as number;
+    const pageCount = this.db.pragma('page_count', { simple: true }) as number;
+    const totalSize = pageSize * pageCount;
+
+    const tables = this.db.prepare(`
+      SELECT name FROM sqlite_master
+      WHERE type='table' AND name NOT LIKE 'sqlite_%'
+    `).all() as any[];
+
+    const tableStats = tables.map(({ name }) => {
+      const count = this.db.prepare(`SELECT COUNT(*) as count FROM ${name}`).get() as any;
+      const pages = this.db.prepare(`SELECT COUNT(*) as count FROM dbstat WHERE name = ?`).get(name) as any;
+
+      return {
+        name,
+        rows: count.count,
+        size: (pages?.count || 0) * pageSize
+      };
+    });
+
+    return { totalSize, tableStats };
+  }
+
+  // ========================================================================
+  // Private Methods
+  // ========================================================================
+
+  private buildEpisodeText(episode: Episode): string {
+    const parts = [episode.task];
+    if (episode.critique) parts.push(episode.critique);
+    if (episode.output) parts.push(episode.output);
+    return parts.join('\n');
+  }
+
+  private chunkArray<T>(array: T[], size: number): T[][] {
+    const chunks: T[][] = [];
+    for (let i = 0; i < array.length; i += size) {
+      chunks.push(array.slice(i, i + size));
+    }
+    return chunks;
+  }
+}

package/src/optimizations/QueryOptimizer.ts

@@ -0,0 +1,294 @@
+/**
+ * QueryOptimizer - Advanced Query Optimization for AgentDB
+ *
+ * Implements:
+ * - Query result caching with TTL
+ * - Prepared statement pooling
+ * - Batch operation optimization
+ * - Index usage analysis
+ * - Query plan analysis
+ */
+
+import { Database } from 'better-sqlite3';
+
+export interface CacheConfig {
+  maxSize: number;
+  ttl: number; // milliseconds
+  enabled: boolean;
+}
+
+export interface QueryStats {
+  query: string;
+  executionCount: number;
+  totalTime: number;
+  avgTime: number;
+  cacheHits: number;
+  cacheMisses: number;
+}
+
+export class QueryOptimizer {
+  private db: Database;
+  private cache: Map<string, { result: any; timestamp: number }>;
+  private stats: Map<string, QueryStats>;
+  private config: CacheConfig;
+
+  constructor(db: Database, config?: Partial<CacheConfig>) {
+    this.db = db;
+    this.cache = new Map();
+    this.stats = new Map();
+    this.config = {
+      maxSize: 1000,
+      ttl: 60000, // 1 minute default
+      enabled: true,
+      ...config
+    };
+  }
+
+  /**
+   * Execute query with caching
+   */
+  query<T = any>(sql: string, params: any[] = [], cacheKey?: string): T {
+    const key = cacheKey || this.generateCacheKey(sql, params);
+    const startTime = Date.now();
+
+    // Check cache
+    if (this.config.enabled && this.cache.has(key)) {
+      const cached = this.cache.get(key)!;
+      if (Date.now() - cached.timestamp < this.config.ttl) {
+        this.recordStats(sql, Date.now() - startTime, true);
+        return cached.result;
+      } else {
+        this.cache.delete(key);
+      }
+    }
+
+    // Execute query
+    const stmt = this.db.prepare(sql);
+    const result = params.length > 0 ? stmt.all(...params) : stmt.all();
+
+    const executionTime = Date.now() - startTime;
+    this.recordStats(sql, executionTime, false);
+
+    // Cache result
+    if (this.config.enabled) {
+      this.cacheResult(key, result);
+    }
+
+    return result as T;
+  }
+
+  /**
+   * Execute query that returns single row
+   */
+  queryOne<T = any>(sql: string, params: any[] = [], cacheKey?: string): T | undefined {
+    const results = this.query<T[]>(sql, params, cacheKey);
+    return results[0];
+  }
+
+  /**
+   * Execute write operation (no caching)
+   */
+  execute(sql: string, params: any[] = []): any {
+    const startTime = Date.now();
+    const stmt = this.db.prepare(sql);
+    const result = params.length > 0 ? stmt.run(...params) : stmt.run();
+
+    this.recordStats(sql, Date.now() - startTime, false);
+
+    // Invalidate relevant cache entries
+    this.invalidateCache(sql);
+
+    return result;
+  }
+
+  /**
+   * Batch insert optimization
+   */
+  batchInsert(table: string, columns: string[], rows: any[][]): void {
+    const placeholders = columns.map(() => '?').join(', ');
+    const sql = `INSERT INTO ${table} (${columns.join(', ')}) VALUES (${placeholders})`;
+
+    const transaction = this.db.transaction((rows: any[][]) => {
+      const stmt = this.db.prepare(sql);
+      for (const row of rows) {
+        stmt.run(...row);
+      }
+    });
+
+    const startTime = Date.now();
+    transaction(rows);
+    this.recordStats(`BATCH INSERT ${table}`, Date.now() - startTime, false);
+  }
+
+  /**
+   * Analyze query plan
+   */
+  analyzeQuery(sql: string): {
+    plan: string;
+    usesIndex: boolean;
+    estimatedCost: number;
+  } {
+    const plan = this.db.prepare(`EXPLAIN QUERY PLAN ${sql}`).all();
+    const planText = plan.map((row: any) => row.detail).join(' ');
+
+    const usesIndex = planText.toLowerCase().includes('index');
+    const hasFullScan = planText.toLowerCase().includes('scan');
+
+    // Simple cost estimation
+    let estimatedCost = 1;
+    if (hasFullScan) estimatedCost *= 10;
+    if (!usesIndex) estimatedCost *= 5;
+
+    return {
+      plan: planText,
+      usesIndex,
+      estimatedCost
+    };
+  }
+
+  /**
+   * Get optimization suggestions
+   */
+  getSuggestions(): string[] {
+    const suggestions: string[] = [];
+
+    // Analyze frequently run queries
+    const frequentQueries = Array.from(this.stats.values())
+      .filter(s => s.executionCount > 100)
+      .sort((a, b) => b.totalTime - a.totalTime)
+      .slice(0, 10);
+
+    for (const stat of frequentQueries) {
+      if (stat.avgTime > 50) {
+        const analysis = this.analyzeQuery(stat.query);
+
+        if (!analysis.usesIndex) {
+          suggestions.push(
+            `Slow query (${stat.avgTime.toFixed(1)}ms avg): Consider adding index for:\n${stat.query}`
+          );
+        }
+
+        if (stat.cacheHits === 0 && stat.executionCount > 50) {
+          suggestions.push(
+            `Frequently run query without cache hits: ${stat.query.substring(0, 50)}...`
+          );
+        }
+      }
+    }
+
+    // Check cache efficiency
+    const totalHits = Array.from(this.stats.values()).reduce((sum, s) => sum + s.cacheHits, 0);
+    const totalMisses = Array.from(this.stats.values()).reduce((sum, s) => sum + s.cacheMisses, 0);
+    const hitRate = totalHits / (totalHits + totalMisses) || 0;
+
+    if (hitRate < 0.3 && totalHits + totalMisses > 1000) {
+      suggestions.push(`Low cache hit rate (${(hitRate * 100).toFixed(1)}%). Consider increasing cache size or TTL.`);
+    }
+
+    return suggestions;
+  }
+
+  /**
+   * Get query statistics
+   */
+  getStats(): QueryStats[] {
+    return Array.from(this.stats.values())
+      .sort((a, b) => b.totalTime - a.totalTime);
+  }
+
+  /**
+   * Clear cache
+   */
+  clearCache(): void {
+    this.cache.clear();
+  }
+
+  /**
+   * Get cache statistics
+   */
+  getCacheStats(): {
+    size: number;
+    hitRate: number;
+    totalHits: number;
+    totalMisses: number;
+  } {
+    const totalHits = Array.from(this.stats.values()).reduce((sum, s) => sum + s.cacheHits, 0);
+    const totalMisses = Array.from(this.stats.values()).reduce((sum, s) => sum + s.cacheMisses, 0);
+
+    return {
+      size: this.cache.size,
+      hitRate: totalHits / (totalHits + totalMisses) || 0,
+      totalHits,
+      totalMisses
+    };
+  }
+
+  // ========================================================================
+  // Private Methods
+  // ========================================================================
+
+  private generateCacheKey(sql: string, params: any[]): string {
+    return `${sql}:${JSON.stringify(params)}`;
+  }
+
+  private cacheResult(key: string, result: any): void {
+    if (this.cache.size >= this.config.maxSize) {
+      // Simple LRU: remove oldest entry
+      const oldestKey = this.cache.keys().next().value;
+      this.cache.delete(oldestKey);
+    }
+
+    this.cache.set(key, {
+      result,
+      timestamp: Date.now()
+    });
+  }
+
+  private invalidateCache(sql: string): void {
+    // Invalidate cache entries related to modified tables
+    const tables = this.extractTables(sql);
+
+    for (const [key] of this.cache) {
+      for (const table of tables) {
+        if (key.toLowerCase().includes(table.toLowerCase())) {
+          this.cache.delete(key);
+        }
+      }
+    }
+  }
+
+  private extractTables(sql: string): string[] {
+    const matches = sql.match(/(?:FROM|INTO|UPDATE|JOIN)\s+(\w+)/gi);
+    if (!matches) return [];
+
+    return matches
+      .map(m => m.split(/\s+/)[1])
+      .filter((v, i, a) => a.indexOf(v) === i); // unique
+  }
+
+  private recordStats(sql: string, time: number, cacheHit: boolean): void {
+    const key = sql.substring(0, 100); // Use first 100 chars as key
+
+    if (!this.stats.has(key)) {
+      this.stats.set(key, {
+        query: sql,
+        executionCount: 0,
+        totalTime: 0,
+        avgTime: 0,
+        cacheHits: 0,
+        cacheMisses: 0
+      });
+    }
+
+    const stat = this.stats.get(key)!;
+    stat.executionCount++;
+    stat.totalTime += time;
+    stat.avgTime = stat.totalTime / stat.executionCount;
+
+    if (cacheHit) {
+      stat.cacheHits++;
+    } else {
+      stat.cacheMisses++;
+    }
+  }
+}

package/src/optimizations/index.ts

@@ -0,0 +1,11 @@
+/**
+ * AgentDB Optimizations
+ * 
+ * Performance optimization utilities
+ */
+
+export { QueryOptimizer } from './QueryOptimizer';
+export { BatchOperations } from './BatchOperations';
+
+export type { CacheConfig, QueryStats } from './QueryOptimizer';
+export type { BatchConfig } from './BatchOperations';