agentdb 3.0.0-alpha.2 → 3.0.0-alpha.4

package/src/coordination/index.ts

@@ -0,0 +1,24 @@
+/**
+ * AgentDB Coordination Module
+ *
+ * Multi-database coordination, synchronization, and distributed operations
+ * for AgentDB vector databases.
+ *
+ * @module coordination
+ */
+
+export {
+  MultiDatabaseCoordinator,
+  // Types
+  type DatabaseInstance,
+  type InstanceStatus,
+  type ConflictResolutionStrategy,
+  type SyncOptions,
+  type SyncProgress,
+  type SyncResult,
+  type ConflictInfo,
+  type VectorData,
+  type MultiDatabaseCoordinatorConfig,
+  type StatusChangeCallback,
+  type DistributedOperationResult,
+} from './MultiDatabaseCoordinator.js';

package/src/core/AgentDB.ts

@@ -0,0 +1,211 @@
+/**
+ * AgentDB v3 - Main database wrapper class
+ *
+ * Provides a unified interface to all AgentDB controllers with
+ * proof-gated mutations via MutationGuard and @ruvector/graph-transformer.
+ */
+import Database from 'better-sqlite3';
+import { ReflexionMemory } from '../controllers/ReflexionMemory.js';
+import { SkillLibrary } from '../controllers/SkillLibrary.js';
+import { ReasoningBank } from '../controllers/ReasoningBank.js';
+import { CausalMemoryGraph } from '../controllers/CausalMemoryGraph.js';
+import { CausalRecall } from '../controllers/CausalRecall.js';
+import { LearningSystem } from '../controllers/LearningSystem.js';
+import { ExplainableRecall } from '../controllers/ExplainableRecall.js';
+import { NightlyLearner } from '../controllers/NightlyLearner.js';
+import { EmbeddingService } from '../controllers/EmbeddingService.js';
+import { createGuardedBackend } from '../backends/factory.js';
+import type { VectorBackend } from '../backends/VectorBackend.js';
+import type { GuardedVectorBackend } from '../backends/ruvector/GuardedVectorBackend.js';
+import type { MutationGuard } from '../security/MutationGuard.js';
+import type { AttestationLog } from '../security/AttestationLog.js';
+import { GraphTransformerService } from '../services/GraphTransformerService.js';
+import * as fs from 'fs';
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+export interface AgentDBConfig {
+  dbPath?: string;
+  namespace?: string;
+  dimension?: number;
+  maxElements?: number;
+  enableAttention?: boolean;
+  attentionConfig?: Record<string, any>;
+}
+
+export class AgentDB {
+  private db: Database.Database;
+  private reflexion!: ReflexionMemory;
+  private skills!: SkillLibrary;
+  private reasoning!: ReasoningBank;
+  private causalGraph!: CausalMemoryGraph;
+  private causalRecall!: CausalRecall;
+  private learningSystem!: LearningSystem;
+  private explainableRecall!: ExplainableRecall;
+  private nightlyLearner!: NightlyLearner;
+  private embedder!: EmbeddingService;
+  private vectorBackend!: VectorBackend;
+  private guardedBackend: GuardedVectorBackend | null = null;
+  private mutationGuard: MutationGuard | null = null;
+  private attestationLog: AttestationLog | null = null;
+  private graphTransformer!: GraphTransformerService;
+  private graphAdapter: any = null;
+  private initialized = false;
+  private config: AgentDBConfig;
+
+  constructor(config: AgentDBConfig = {}) {
+    this.config = config;
+    const dbPath = config.dbPath || ':memory:';
+    this.db = new Database(dbPath);
+    this.db.pragma('journal_mode = WAL');
+  }
+
+  async initialize(): Promise<void> {
+    if (this.initialized) return;
+
+    const dim = this.config.dimension ?? 384;
+
+    // Load schemas
+    const schemaPath = path.join(__dirname, '../../schemas/schema.sql');
+    if (fs.existsSync(schemaPath)) {
+      const schema = fs.readFileSync(schemaPath, 'utf-8');
+      this.db.exec(schema);
+    }
+
+    const frontierSchemaPath = path.join(__dirname, '../../schemas/frontier-schema.sql');
+    if (fs.existsSync(frontierSchemaPath)) {
+      const frontierSchema = fs.readFileSync(frontierSchemaPath, 'utf-8');
+      this.db.exec(frontierSchema);
+    }
+
+    // Initialize embedder
+    this.embedder = new EmbeddingService({
+      model: 'Xenova/all-MiniLM-L6-v2',
+      dimension: dim,
+      provider: 'transformers'
+    });
+    await this.embedder.initialize();
+
+    // Initialize GraphTransformerService (8 verified modules)
+    this.graphTransformer = new GraphTransformerService();
+    await this.graphTransformer.initialize();
+
+    // Initialize proof-gated vector backend (ADR-060)
+    let controllerVB: VectorBackend | null = null;
+    try {
+      const { backend, guard, log } = await createGuardedBackend('auto', {
+        dimensions: dim,
+        metric: 'cosine',
+        maxElements: this.config.maxElements ?? 10000,
+        database: this.db,
+      });
+      this.guardedBackend = backend;
+      this.mutationGuard = guard;
+      this.attestationLog = log;
+      this.vectorBackend = backend;
+      controllerVB = backend;
+    } catch {
+      // Guarded backend unavailable — controllers work without vectorBackend
+      controllerVB = null;
+    }
+
+    // Initialize controllers — wire vectorBackend where supported
+    this.reflexion = new ReflexionMemory(this.db, this.embedder, controllerVB ?? undefined);
+    this.skills = new SkillLibrary(this.db, this.embedder, controllerVB ?? undefined);
+    this.reasoning = new ReasoningBank(this.db, this.embedder, controllerVB ?? undefined);
+    this.causalGraph = new CausalMemoryGraph(this.db);
+    this.causalRecall = new CausalRecall(this.db, this.embedder);
+    this.learningSystem = new LearningSystem(this.db, this.embedder);
+    this.explainableRecall = new ExplainableRecall(this.db, this.embedder);
+    this.nightlyLearner = new NightlyLearner(this.db, this.embedder);
+
+    // Initialize optional graph database adapter
+    try {
+      const { GraphDatabaseAdapter } = await import('../backends/graph/GraphDatabaseAdapter.js');
+      const storagePath = this.config.dbPath && this.config.dbPath !== ':memory:'
+        ? this.config.dbPath.replace(/\.db$/, '') + '.graph'
+        : null;
+
+      if (storagePath) {
+        this.graphAdapter = new GraphDatabaseAdapter(
+          { storagePath, dimensions: dim },
+          this.embedder
+        );
+        await this.graphAdapter.initialize();
+      }
+    } catch {
+      this.graphAdapter = null;
+    }
+
+    this.initialized = true;
+  }
+
+  getController(name: string): any {
+    if (!this.initialized) {
+      throw new Error('AgentDB not initialized. Call initialize() first.');
+    }
+
+    switch (name) {
+      case 'memory':
+      case 'reflexion':
+        return this.reflexion;
+      case 'skills':
+        return this.skills;
+      case 'reasoning':
+      case 'reasoningBank':
+        return this.reasoning;
+      case 'causal':
+      case 'causalGraph':
+        return this.causalGraph;
+      case 'causalRecall':
+        return this.causalRecall;
+      case 'learning':
+      case 'learningSystem':
+        return this.learningSystem;
+      case 'explainableRecall':
+        return this.explainableRecall;
+      case 'nightlyLearner':
+        return this.nightlyLearner;
+      case 'graph':
+      case 'graphAdapter':
+        return this.graphAdapter;
+      case 'graphTransformer':
+        return this.graphTransformer;
+      case 'mutationGuard':
+        return this.mutationGuard;
+      case 'attestationLog':
+        return this.attestationLog;
+      case 'vectorBackend':
+        return this.vectorBackend;
+      default:
+        throw new Error(`Unknown controller: ${name}`);
+    }
+  }
+
+  getGraphAdapter(): any {
+    return this.graphAdapter;
+  }
+
+  getGraphTransformer(): GraphTransformerService {
+    return this.graphTransformer;
+  }
+
+  getMutationGuard(): MutationGuard | null {
+    return this.mutationGuard;
+  }
+
+  async close(): Promise<void> {
+    if (this.vectorBackend) {
+      try { this.vectorBackend.close(); } catch { /* ignore */ }
+    }
+    if (this.db) {
+      this.db.close();
+    }
+  }
+
+  get database(): Database.Database {
+    return this.db;
+  }
+}

package/src/db-fallback.ts

@@ -0,0 +1,296 @@
+/**
+ * Database System using sql.js (WASM SQLite)
+ * Pure JavaScript implementation with NO build dependencies
+ *
+ * SECURITY: Fixed SQL injection vulnerabilities:
+ * - PRAGMA commands validated against whitelist
+ * - Removed eval() usage (replaced with async import)
+ */
+
+import { validatePragmaCommand, ValidationError } from './security/input-validation.js';
+import * as fs from 'fs';
+import * as path from 'path';
+
+// Type-only for compatibility
+type Database = any;
+
+let sqlJsWrapper: any = null;
+
+/**
+ * Get sql.js database implementation (ONLY sql.js, no better-sqlite3)
+ */
+export async function getDatabaseImplementation(): Promise<any> {
+  // Return cached wrapper
+  if (sqlJsWrapper) {
+    return sqlJsWrapper;
+  }
+
+  try {
+    console.log('✅ Using sql.js (WASM SQLite, no build tools required)');
+
+    // sql.js requires async initialization
+    const mod = await import('sql.js');
+    const SQL = await mod.default();
+
+    // Create database wrapper
+    sqlJsWrapper = createSqlJsWrapper(SQL);
+
+    return sqlJsWrapper;
+  } catch (error) {
+    console.error('❌ Failed to initialize sql.js:', (error as Error).message);
+    throw new Error(
+      'Failed to initialize SQLite. Please ensure sql.js is installed:\n' +
+      'npm install sql.js'
+    );
+  }
+}
+
+/**
+ * Create a better-sqlite3 compatible wrapper around sql.js
+ * This allows AgentDB to work (with reduced performance) without native compilation
+ */
+function createSqlJsWrapper(SQL: any) {
+  return class SqlJsDatabase {
+    private db: any;
+    private filename: string;
+    private activeStatements: Map<number, any> = new Map();
+    private statementCounter: number = 0;
+    private intervalId: NodeJS.Timeout | null = null;
+
+    constructor(filename: string, options?: any) {
+      this.filename = filename;
+
+      // In-memory database
+      if (filename === ':memory:') {
+        this.db = new SQL.Database();
+      } else {
+        // File-based database - use safe fs module (no eval)
+        try {
+          if (fs.existsSync(filename)) {
+            const buffer = fs.readFileSync(filename);
+            this.db = new SQL.Database(buffer);
+          } else {
+            this.db = new SQL.Database();
+          }
+        } catch (error) {
+          console.warn('⚠️  Could not read database file:', (error as Error).message);
+          this.db = new SQL.Database();
+        }
+      }
+
+      // Warn if too many active statements (memory leak detection)
+      this.intervalId = setInterval(() => {
+        if (this.activeStatements.size > 50) {
+          console.warn(`⚠️  Detected ${this.activeStatements.size} active SQL statements - possible memory leak`);
+        }
+      }, 10000);
+    }
+
+    prepare(sql: string) {
+      const stmt = this.db.prepare(sql);
+      let isFinalized = false;
+      const stmtId = ++this.statementCounter;
+
+      // Track active statement
+      this.activeStatements.set(stmtId, stmt);
+
+      return {
+        run: (...params: any[]) => {
+          if (isFinalized) throw new Error('Statement already finalized');
+          try {
+            stmt.bind(params);
+            stmt.step();
+            stmt.reset();
+
+            return {
+              changes: this.db.getRowsModified(),
+              lastInsertRowid: this.db.exec('SELECT last_insert_rowid()')[0]?.values[0]?.[0] || 0
+            };
+          } catch (error) {
+            // Auto-free on error to prevent memory leak
+            if (!isFinalized) {
+              stmt.free();
+              isFinalized = true;
+              this.activeStatements.delete(stmtId);
+            }
+            throw error;
+          }
+        },
+
+        get: (...params: any[]) => {
+          if (isFinalized) throw new Error('Statement already finalized');
+          try {
+            stmt.bind(params);
+            const hasRow = stmt.step();
+
+            if (!hasRow) {
+              stmt.reset();
+              return undefined;
+            }
+
+            const columns = stmt.getColumnNames();
+            const values = stmt.get();
+            stmt.reset();
+
+            const result: any = {};
+            columns.forEach((col: string, idx: number) => {
+              result[col] = values[idx];
+            });
+
+            return result;
+          } catch (error) {
+            // Auto-free on error to prevent memory leak
+            if (!isFinalized) {
+              stmt.free();
+              isFinalized = true;
+              this.activeStatements.delete(stmtId);
+            }
+            throw error;
+          }
+        },
+
+        all: (...params: any[]) => {
+          if (isFinalized) throw new Error('Statement already finalized');
+          try {
+            stmt.bind(params);
+            const results: any[] = [];
+
+            while (stmt.step()) {
+              const columns = stmt.getColumnNames();
+              const values = stmt.get();
+
+              const result: any = {};
+              columns.forEach((col: string, idx: number) => {
+                result[col] = values[idx];
+              });
+
+              results.push(result);
+            }
+
+            stmt.reset();
+            return results;
+          } catch (error) {
+            // Auto-free on error to prevent memory leak
+            if (!isFinalized) {
+              stmt.free();
+              isFinalized = true;
+              this.activeStatements.delete(stmtId);
+            }
+            throw error;
+          }
+        },
+
+        finalize: () => {
+          if (!isFinalized) {
+            stmt.free();
+            isFinalized = true;
+            this.activeStatements.delete(stmtId);
+          }
+        }
+      };
+    }
+
+    exec(sql: string) {
+      return this.db.exec(sql);
+    }
+
+    save() {
+      // Save to file if needed
+      if (this.filename !== ':memory:') {
+        try {
+          // Create parent directories if they don't exist
+          const dir = path.dirname(this.filename);
+          if (!fs.existsSync(dir)) {
+            fs.mkdirSync(dir, { recursive: true });
+          }
+
+          const data = this.db.export();
+          fs.writeFileSync(this.filename, Buffer.from(data));
+        } catch (error) {
+          console.error('❌ Could not save database to file:', (error as Error).message);
+          throw error;
+        }
+      }
+    }
+
+    close() {
+      // Clear interval timer
+      if (this.intervalId) {
+        clearInterval(this.intervalId);
+        this.intervalId = null;
+      }
+
+      // Free all active statements to prevent memory leaks
+      for (const [stmtId, stmt] of this.activeStatements.entries()) {
+        try {
+          stmt.free();
+        } catch (e) {
+          // Statement may already be freed
+        }
+      }
+      this.activeStatements.clear();
+
+      // Save to file before closing
+      this.save();
+      this.db.close();
+    }
+
+    pragma(pragma: string, options?: any) {
+      try {
+        // SECURITY: Validate PRAGMA command against whitelist to prevent SQL injection
+        const validatedPragma = validatePragmaCommand(pragma);
+
+        // Execute validated PRAGMA
+        const result = this.db.exec(`PRAGMA ${validatedPragma}`);
+        return result[0]?.values[0]?.[0];
+      } catch (error) {
+        if (error instanceof ValidationError) {
+          console.error(`❌ Invalid PRAGMA command: ${error.message}`);
+          throw error;
+        }
+        throw error;
+      }
+    }
+
+    transaction(fn: () => any) {
+      // Return a function that executes the transaction when called
+      // This matches better-sqlite3 API where transaction() returns a callable function
+      return () => {
+        try {
+          this.db.exec('BEGIN TRANSACTION');
+          const result = fn();
+          this.db.exec('COMMIT');
+          return result;
+        } catch (error) {
+          this.db.exec('ROLLBACK');
+          throw error;
+        }
+      };
+    }
+  };
+}
+
+/**
+ * Create a database instance using sql.js
+ */
+export async function createDatabase(filename: string, options?: any): Promise<any> {
+  const DatabaseImpl = await getDatabaseImplementation();
+  return new DatabaseImpl(filename, options);
+}
+
+/**
+ * Get information about current database implementation
+ */
+export function getDatabaseInfo(): {
+  implementation: string;
+  isNative: boolean;
+  performance: 'high' | 'medium' | 'low';
+  requiresBuildTools: boolean;
+} {
+  return {
+    implementation: 'sql.js (WASM)',
+    isNative: false,
+    performance: 'medium',
+    requiresBuildTools: false
+  };
+}

package/src/db-test.ts

@@ -0,0 +1,59 @@
+/**
+ * Test Database Factory - Uses better-sqlite3 when available for testing
+ *
+ * Integration tests need to create large datasets and run many queries,
+ * which can exceed sql.js WASM memory limit (64MB). This factory attempts
+ * to use better-sqlite3 for tests, falling back to sql.js if unavailable.
+ */
+
+import type { Database } from 'better-sqlite3';
+
+let betterSqlite3: any = null;
+
+/**
+ * Try to load better-sqlite3 (optional dependency)
+ */
+async function loadBetterSqlite3(): Promise<any> {
+  if (betterSqlite3) return betterSqlite3;
+
+  try {
+    betterSqlite3 = (await import('better-sqlite3')).default;
+    console.log('✅ Using better-sqlite3 for tests (no memory limit)');
+    return betterSqlite3;
+  } catch (error) {
+    console.warn('⚠️  better-sqlite3 not available, falling back to sql.js (64MB memory limit)');
+    console.warn('   Install with: npm install --save-dev better-sqlite3');
+    return null;
+  }
+}
+
+/**
+ * Wrap better-sqlite3 to add sql.js-compatible save() method
+ */
+function wrapBetterSqlite3(db: any): any {
+  // Add sql.js-compatible save() method (no-op for better-sqlite3)
+  if (!db.save) {
+    db.save = function() {
+      // better-sqlite3 auto-saves, so this is a no-op
+      return;
+    };
+  }
+  return db;
+}
+
+/**
+ * Create database for testing - prefers better-sqlite3, falls back to sql.js
+ */
+export async function createTestDatabase(filename: string, options?: any): Promise<Database> {
+  const BetterSqlite3 = await loadBetterSqlite3();
+
+  if (BetterSqlite3) {
+    // Use better-sqlite3 (native, no memory limit)
+    const db = new BetterSqlite3(filename, options);
+    return wrapBetterSqlite3(db);
+  } else {
+    // Fall back to sql.js (WASM, 64MB limit)
+    const { createDatabase } = await import('./db-fallback.js');
+    return createDatabase(filename, options);
+  }
+}