agentdb 3.0.0-alpha.2 → 3.0.0-alpha.4

package/src/db-unified.ts

@@ -0,0 +1,327 @@
+/**
+ * Unified Database Layer for AgentDB v2
+ *
+ * Architecture:
+ * - PRIMARY: RuVector GraphDatabase (@ruvector/graph-node) for new databases
+ * - FALLBACK: SQLite (sql.js) for legacy databases
+ *
+ * Detection Logic:
+ * 1. Check if database file exists
+ * 2. If exists, check file signature to determine type
+ * 3. If new database or .graph extension → use GraphDatabase
+ * 4. If .db extension and SQLite signature → use SQLite (legacy mode)
+ *
+ * Migration:
+ * - Provides migration tool to convert SQLite → GraphDatabase
+ * - Maintains backward compatibility with existing databases
+ */
+
+import { GraphDatabaseAdapter, type GraphDatabaseConfig } from './backends/graph/GraphDatabaseAdapter.js';
+import { getDatabaseImplementation } from './db-fallback.js';
+import * as fs from 'fs';
+import * as path from 'path';
+
+export type DatabaseMode = 'graph' | 'sqlite-legacy';
+
+export interface UnifiedDatabaseConfig {
+  path: string;
+  dimensions?: number;       // Default: 384 (sentence-transformers standard)
+  forceMode?: DatabaseMode;  // Force specific mode
+  autoMigrate?: boolean;     // Auto-migrate SQLite → Graph
+}
+
+/**
+ * Unified Database - Smart detection and mode selection
+ */
+export class UnifiedDatabase {
+  private mode: DatabaseMode;
+  private graphDb?: GraphDatabaseAdapter;
+  private sqliteDb?: any;
+  private config: UnifiedDatabaseConfig;
+
+  constructor(config: UnifiedDatabaseConfig) {
+    this.config = config;
+    this.mode = 'graph'; // Default to graph mode
+  }
+
+  /**
+   * Initialize database with automatic mode detection
+   */
+  async initialize(embedder: any): Promise<void> {
+    const dbPath = this.config.path;
+
+    // Check if user forced a specific mode
+    if (this.config.forceMode) {
+      this.mode = this.config.forceMode;
+      await this.initializeMode(embedder);
+      return;
+    }
+
+    // Auto-detect based on file extension and content
+    if (fs.existsSync(dbPath)) {
+      const ext = path.extname(dbPath);
+
+      // .graph extension = always use graph mode
+      if (ext === '.graph') {
+        this.mode = 'graph';
+        console.log('🔍 Detected .graph extension → Using RuVector GraphDatabase');
+      }
+      // .db extension = check if it's SQLite
+      else if (ext === '.db') {
+        const isLegacySQLite = await this.isSQLiteDatabase(dbPath);
+
+        if (isLegacySQLite) {
+          this.mode = 'sqlite-legacy';
+          console.log('🔍 Detected legacy SQLite database');
+
+          // Offer migration if autoMigrate is enabled
+          if (this.config.autoMigrate) {
+            console.log('🔄 Auto-migration enabled, will migrate to GraphDatabase...');
+            await this.migrateSQLiteToGraph(dbPath, embedder);
+            this.mode = 'graph';
+            // Migration already initialized graphDb, skip initializeMode
+            return;
+          } else {
+            console.log('ℹ️  Running in legacy SQLite mode');
+            console.log('💡 To migrate to RuVector Graph: set autoMigrate: true');
+          }
+        } else {
+          // Not SQLite, use graph mode
+          this.mode = 'graph';
+          console.log('🔍 Using RuVector GraphDatabase');
+        }
+      } else {
+        // Unknown extension, default to graph
+        this.mode = 'graph';
+      }
+    } else {
+      // New database - use graph mode (recommended)
+      this.mode = 'graph';
+      console.log('✨ Creating new RuVector GraphDatabase');
+
+      // Suggest .graph extension if not using it
+      if (!dbPath.endsWith('.graph') && !dbPath.endsWith('.db')) {
+        console.log('💡 Tip: Use .graph extension for clarity (e.g., agentdb.graph)');
+      }
+    }
+
+    await this.initializeMode(embedder);
+  }
+
+  /**
+   * Initialize the selected database mode
+   */
+  private async initializeMode(embedder: any): Promise<void> {
+    if (this.mode === 'graph') {
+      // Use RuVector GraphDatabase
+      const config: GraphDatabaseConfig = {
+        storagePath: this.config.path,
+        dimensions: this.config.dimensions || 384,
+        distanceMetric: 'Cosine'
+      };
+
+      this.graphDb = new GraphDatabaseAdapter(config, embedder);
+      await this.graphDb.initialize();
+
+      console.log('✅ RuVector GraphDatabase ready (Primary Mode)');
+      console.log('   - Cypher queries enabled');
+      console.log('   - Hypergraph support active');
+      console.log('   - ACID transactions available');
+      console.log('   - 131K+ ops/sec batch inserts');
+    } else {
+      // Use legacy SQLite
+      const { createDatabase } = await import('./db-fallback.js');
+      this.sqliteDb = await createDatabase(this.config.path);
+
+      console.log('⚠️  Using legacy SQLite mode');
+      console.log('   - Limited to SQL queries');
+      console.log('   - No hypergraph support');
+      console.log('   - Consider migration to GraphDatabase');
+    }
+  }
+
+  /**
+   * Check if file is a SQLite database
+   */
+  private async isSQLiteDatabase(filePath: string): Promise<boolean> {
+    try {
+      const buffer = fs.readFileSync(filePath);
+
+      // SQLite databases start with "SQLite format 3\0"
+      const signature = buffer.slice(0, 16).toString();
+      return signature.startsWith('SQLite format 3');
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Migrate SQLite database to RuVector GraphDatabase
+   */
+  private async migrateSQLiteToGraph(sqlitePath: string, embedder: any): Promise<void> {
+    console.log('🔄 Starting migration from SQLite to RuVector Graph...');
+
+    const startTime = Date.now();
+
+    // Load SQLite database using ESM import
+    const { createDatabase } = await import('./db-fallback.js');
+    const sqliteDb = await createDatabase(sqlitePath);
+
+    // Create new GraphDatabase
+    const graphPath = sqlitePath.replace(/\.db$/, '.graph');
+    const graphConfig: GraphDatabaseConfig = {
+      storagePath: graphPath,
+      dimensions: this.config.dimensions || 384,
+      distanceMetric: 'Cosine'
+    };
+
+    const graphDb = new GraphDatabaseAdapter(graphConfig, embedder);
+    await graphDb.initialize();
+
+    // Migrate episodes
+    console.log('   📦 Migrating episodes...');
+    const episodes = sqliteDb.prepare('SELECT * FROM episodes').all();
+
+    for (const ep of episodes) {
+      // Generate embedding for episode
+      const text = `${ep.task} ${ep.input || ''} ${ep.output || ''}`;
+      const embedding = await embedder.embed(text);
+
+      await graphDb.storeEpisode({
+        id: `ep-${ep.id}`,
+        sessionId: ep.session_id,
+        task: ep.task,
+        reward: ep.reward,
+        success: ep.success === 1,
+        input: ep.input,
+        output: ep.output,
+        critique: ep.critique,
+        createdAt: ep.created_at,
+        tokensUsed: ep.tokens_used,
+        latencyMs: ep.latency_ms
+      }, embedding);
+    }
+    console.log(`   ✅ Migrated ${episodes.length} episodes`);
+
+    // Migrate skills
+    console.log('   📦 Migrating skills...');
+    const skills = sqliteDb.prepare('SELECT * FROM skills').all();
+
+    for (const skill of skills) {
+      const text = `${skill.name} ${skill.description} ${skill.code}`;
+      const embedding = await embedder.embed(text);
+
+      await graphDb.storeSkill({
+        id: `skill-${skill.id}`,
+        name: skill.name,
+        description: skill.description,
+        code: skill.code,
+        usageCount: skill.usage_count,
+        avgReward: skill.avg_reward,
+        createdAt: skill.created_at,
+        updatedAt: skill.updated_at,
+        tags: skill.tags
+      }, embedding);
+    }
+    console.log(`   ✅ Migrated ${skills.length} skills`);
+
+    // Migrate causal edges
+    console.log('   📦 Migrating causal relationships...');
+    const edges = sqliteDb.prepare('SELECT * FROM causal_edges').all();
+
+    for (const edge of edges) {
+      const text = edge.mechanism;
+      const embedding = await embedder.embed(text);
+
+      await graphDb.createCausalEdge({
+        from: `ep-${edge.from_memory_id}`,
+        to: `ep-${edge.to_memory_id}`,
+        mechanism: edge.mechanism,
+        uplift: edge.uplift,
+        confidence: edge.confidence,
+        sampleSize: edge.sample_size
+      }, embedding);
+    }
+    console.log(`   ✅ Migrated ${edges.length} causal edges`);
+
+    const duration = ((Date.now() - startTime) / 1000).toFixed(2);
+    console.log(`\n🎉 Migration complete in ${duration}s!`);
+    console.log(`   Old SQLite: ${sqlitePath}`);
+    console.log(`   New Graph: ${graphPath}`);
+    console.log(`\n💡 Backup your SQLite file and update path to use .graph`);
+
+    // Close SQLite
+    sqliteDb.close();
+
+    // Update config to use new graph database
+    this.config.path = graphPath;
+    this.graphDb = graphDb;
+  }
+
+  /**
+   * Get the active database mode
+   */
+  getMode(): DatabaseMode {
+    return this.mode;
+  }
+
+  /**
+   * Get the graph database (if in graph mode)
+   */
+  getGraphDatabase(): GraphDatabaseAdapter | undefined {
+    return this.graphDb;
+  }
+
+  /**
+   * Get the SQLite database (if in legacy mode)
+   */
+  getSQLiteDatabase(): any | undefined {
+    return this.sqliteDb;
+  }
+
+  /**
+   * Execute a query (auto-routes to correct database)
+   */
+  async query(queryOrCypher: string): Promise<any> {
+    if (this.mode === 'graph') {
+      // Execute Cypher query
+      return await this.graphDb!.query(queryOrCypher);
+    } else {
+      // Execute SQL query
+      return this.sqliteDb!.prepare(queryOrCypher).all();
+    }
+  }
+
+  /**
+   * Close database
+   */
+  close(): void {
+    if (this.graphDb) {
+      this.graphDb.close();
+    }
+    if (this.sqliteDb) {
+      this.sqliteDb.close();
+    }
+  }
+}
+
+/**
+ * Create unified database (smart mode detection)
+ */
+export async function createUnifiedDatabase(
+  path: string,
+  embedder: any,
+  options?: Partial<UnifiedDatabaseConfig>
+): Promise<UnifiedDatabase> {
+  const config: UnifiedDatabaseConfig = {
+    path,
+    dimensions: options?.dimensions || 384,
+    forceMode: options?.forceMode,
+    autoMigrate: options?.autoMigrate ?? false  // Default: manual migration
+  };
+
+  const db = new UnifiedDatabase(config);
+  await db.initialize(embedder);
+
+  return db;
+}

package/src/examples/quic-sync-example.ts

@@ -0,0 +1,198 @@
+/**
+ * QUIC Synchronization Example
+ *
+ * Demonstrates how to use QUICServer, QUICClient, and SyncCoordinator
+ * for bidirectional synchronization between AgentDB instances.
+ */
+
+import { createDatabase } from '../db-fallback.js';
+import {
+  QUICServer,
+  QUICClient,
+  SyncCoordinator,
+  ReflexionMemory,
+  SkillLibrary,
+  EmbeddingService,
+} from '../controllers/index.js';
+
+async function exampleQUICSync() {
+  console.log('🚀 QUIC Synchronization Example\n');
+
+  // Initialize two database instances (simulating local and remote)
+  const localDB = createDatabase('./local-agent.db');
+  const remoteDB = createDatabase('./remote-agent.db');
+
+  // Initialize embedding services
+  const localEmbedder = new EmbeddingService({
+    model: 'Xenova/all-MiniLM-L6-v2',
+    dimension: 384,
+    provider: 'transformers',
+  });
+  const remoteEmbedder = new EmbeddingService({
+    model: 'Xenova/all-MiniLM-L6-v2',
+    dimension: 384,
+    provider: 'transformers',
+  });
+
+  // Initialize memory controllers
+  const localReflexion = new ReflexionMemory(localDB, localEmbedder);
+  const remoteReflexion = new ReflexionMemory(remoteDB, remoteEmbedder);
+  const remoteSkillLib = new SkillLibrary(remoteDB, remoteEmbedder);
+
+  try {
+    // ===== Setup Remote Server =====
+    console.log('📡 Setting up remote QUIC server...');
+    const server = new QUICServer(remoteDB, {
+      host: '0.0.0.0',
+      port: 4433,
+      authToken: 'secret-token-123',
+      maxConnections: 10,
+      rateLimit: {
+        maxRequestsPerMinute: 60,
+        maxBytesPerMinute: 10 * 1024 * 1024,
+      },
+    });
+
+    await server.start();
+    console.log('✓ Server started\n');
+
+    // ===== Setup Local Client =====
+    console.log('📱 Setting up local QUIC client...');
+    const client = new QUICClient({
+      serverHost: 'localhost',
+      serverPort: 4433,
+      authToken: 'secret-token-123',
+      maxRetries: 3,
+      retryDelayMs: 1000,
+      timeoutMs: 30000,
+      poolSize: 5,
+    });
+
+    await client.connect();
+    console.log('✓ Client connected\n');
+
+    // Test ping
+    console.log('🏓 Testing connection...');
+    const pingResult = await client.ping();
+    console.log(`✓ Ping: ${pingResult.latencyMs}ms\n`);
+
+    // ===== Add some data to remote =====
+    console.log('📝 Adding test data to remote database...');
+    await remoteReflexion.storeEpisode({
+      sessionId: 'test-session',
+      task: 'Calculate fibonacci',
+      input: 'n=10',
+      output: '55',
+      critique: 'Efficient implementation',
+      reward: 1.0,
+      success: true,
+      latencyMs: 150,
+      tokensUsed: 200,
+      tags: ['math', 'algorithm'],
+    });
+
+    await remoteSkillLib.createSkill({
+      name: 'fibonacci',
+      description: 'Calculate fibonacci numbers',
+      signature: {
+        inputs: { n: 'number' },
+        outputs: { result: 'number' },
+      },
+      code: 'function fib(n) { ... }',
+      successRate: 1.0,
+      uses: 5,
+      avgReward: 0.95,
+      avgLatencyMs: 100,
+    });
+    console.log('✓ Test data added\n');
+
+    // ===== Setup Sync Coordinator =====
+    console.log('🔄 Setting up sync coordinator...');
+    const coordinator = new SyncCoordinator({
+      db: localDB,
+      client: client,
+      server: server,
+      conflictStrategy: 'latest-wins',
+      batchSize: 100,
+      autoSync: false,
+    });
+
+    // ===== Perform Manual Sync =====
+    console.log('🔄 Starting synchronization...\n');
+    const syncReport = await coordinator.sync((progress) => {
+      console.log(`  [${progress.phase}] ${progress.message || ''}`);
+      if (progress.itemType) {
+        console.log(`    Syncing ${progress.itemType}: ${progress.current}/${progress.total}`);
+      }
+    });
+
+    console.log('\n📊 Sync Report:');
+    console.log(`  Success: ${syncReport.success}`);
+    console.log(`  Duration: ${syncReport.durationMs}ms`);
+    console.log(`  Items pulled: ${syncReport.itemsPulled}`);
+    console.log(`  Items pushed: ${syncReport.itemsPushed}`);
+    console.log(`  Conflicts resolved: ${syncReport.conflictsResolved}`);
+    console.log(`  Bytes transferred: ${syncReport.bytesTransferred}`);
+    if (syncReport.errors.length > 0) {
+      console.log(`  Errors: ${syncReport.errors.join(', ')}`);
+    }
+
+    // ===== Check Sync State =====
+    console.log('\n📊 Sync State:');
+    const state = coordinator.getSyncState();
+    console.log(`  Last sync: ${new Date(state.lastSyncAt).toISOString()}`);
+    console.log(`  Total items synced: ${state.totalItemsSynced}`);
+    console.log(`  Total bytes synced: ${state.totalBytesSynced}`);
+    console.log(`  Sync count: ${state.syncCount}`);
+
+    // ===== Server Status =====
+    console.log('\n📡 Server Status:');
+    const serverStatus = server.getStatus();
+    console.log(`  Running: ${serverStatus.isRunning}`);
+    console.log(`  Active connections: ${serverStatus.activeConnections}`);
+    console.log(`  Total requests: ${serverStatus.totalRequests}`);
+
+    // ===== Client Status =====
+    console.log('\n📱 Client Status:');
+    const clientStatus = client.getStatus();
+    console.log(`  Connected: ${clientStatus.isConnected}`);
+    console.log(`  Pool size: ${clientStatus.poolSize}`);
+    console.log(`  Active connections: ${clientStatus.activeConnections}`);
+    console.log(`  Total requests: ${clientStatus.totalRequests}`);
+
+    // ===== Enable Auto-Sync =====
+    console.log('\n🔄 Testing auto-sync...');
+    coordinator['config'].autoSync = true;
+    coordinator['config'].syncIntervalMs = 5000; // 5 seconds
+    coordinator['startAutoSync']();
+    console.log('✓ Auto-sync enabled (5s interval)');
+
+    // Wait for one auto-sync cycle
+    await new Promise((resolve) => setTimeout(resolve, 6000));
+    coordinator.stopAutoSync();
+    console.log('✓ Auto-sync disabled');
+
+    // ===== Cleanup =====
+    console.log('\n🧹 Cleaning up...');
+    await client.disconnect();
+    await server.stop();
+    console.log('✓ Cleanup complete');
+
+    console.log('\n✅ QUIC Sync example completed successfully!');
+  } catch (error) {
+    console.error('\n❌ Error:', error);
+    throw error;
+  }
+}
+
+// Run example if executed directly
+if (import.meta.url === `file://${process.argv[1]}`) {
+  exampleQUICSync()
+    .then(() => process.exit(0))
+    .catch((error) => {
+      console.error('Fatal error:', error);
+      process.exit(1);
+    });
+}
+
+export { exampleQUICSync };