moflo 4.8.21 → 4.8.23

package/src/@claude-flow/memory/dist/hybrid-backend.js

@@ -0,0 +1,569 @@
+/**
+ * HybridBackend - Combines SQLite (structured queries) + AgentDB (vector search)
+ *
+ * Per ADR-009: "HybridBackend (SQLite + AgentDB) as default"
+ * - SQLite for: Structured queries, ACID transactions, exact matches
+ * - AgentDB for: Semantic search, vector similarity, RAG
+ *
+ * @module v3/memory/hybrid-backend
+ */
+import { EventEmitter } from 'node:events';
+import { SQLiteBackend } from './sqlite-backend.js';
+import { AgentDBBackend } from './agentdb-backend.js';
+/**
+ * Default configuration
+ */
+const DEFAULT_CONFIG = {
+    sqlite: {},
+    agentdb: {},
+    defaultNamespace: 'default',
+    embeddingGenerator: undefined,
+    routingStrategy: 'auto',
+    dualWrite: true,
+    semanticThreshold: 0.7,
+    hybridMaxResults: 100,
+};
+/**
+ * HybridBackend Implementation
+ *
+ * Intelligently routes queries between SQLite and AgentDB:
+ * - Exact matches, prefix queries → SQLite
+ * - Semantic search, similarity → AgentDB
+ * - Complex hybrid queries → Both backends with intelligent merging
+ */
+export class HybridBackend extends EventEmitter {
+    sqlite;
+    agentdb;
+    config;
+    initialized = false;
+    // Performance tracking
+    stats = {
+        sqliteQueries: 0,
+        agentdbQueries: 0,
+        hybridQueries: 0,
+        totalQueryTime: 0,
+    };
+    constructor(config = {}) {
+        super();
+        this.config = { ...DEFAULT_CONFIG, ...config };
+        // Initialize SQLite backend
+        this.sqlite = new SQLiteBackend({
+            ...this.config.sqlite,
+            defaultNamespace: this.config.defaultNamespace,
+            embeddingGenerator: this.config.embeddingGenerator,
+        });
+        // Initialize AgentDB backend
+        this.agentdb = new AgentDBBackend({
+            ...this.config.agentdb,
+            namespace: this.config.defaultNamespace,
+            embeddingGenerator: this.config.embeddingGenerator,
+        });
+        // Forward events from both backends
+        this.sqlite.on('entry:stored', (data) => this.emit('sqlite:stored', data));
+        this.sqlite.on('entry:updated', (data) => this.emit('sqlite:updated', data));
+        this.sqlite.on('entry:deleted', (data) => this.emit('sqlite:deleted', data));
+        this.agentdb.on('entry:stored', (data) => this.emit('agentdb:stored', data));
+        this.agentdb.on('entry:updated', (data) => this.emit('agentdb:updated', data));
+        this.agentdb.on('entry:deleted', (data) => this.emit('agentdb:deleted', data));
+        this.agentdb.on('cache:hit', (data) => this.emit('cache:hit', data));
+        this.agentdb.on('cache:miss', (data) => this.emit('cache:miss', data));
+    }
+    /**
+     * Initialize both backends
+     */
+    async initialize() {
+        if (this.initialized)
+            return;
+        await Promise.all([this.sqlite.initialize(), this.agentdb.initialize()]);
+        this.initialized = true;
+        this.emit('initialized');
+    }
+    /**
+     * Shutdown both backends
+     */
+    async shutdown() {
+        if (!this.initialized)
+            return;
+        await Promise.all([this.sqlite.shutdown(), this.agentdb.shutdown()]);
+        this.initialized = false;
+        this.emit('shutdown');
+    }
+    /**
+     * Store in both backends (dual-write for consistency)
+     */
+    async store(entry) {
+        if (this.config.dualWrite) {
+            // Write to both backends in parallel
+            await Promise.all([this.sqlite.store(entry), this.agentdb.store(entry)]);
+        }
+        else {
+            // Write to primary backend only (AgentDB has vector search)
+            await this.agentdb.store(entry);
+        }
+        this.emit('entry:stored', { id: entry.id });
+    }
+    /**
+     * Get from AgentDB (has caching enabled)
+     */
+    async get(id) {
+        return this.agentdb.get(id);
+    }
+    /**
+     * Get by key (SQLite optimized for exact matches)
+     */
+    async getByKey(namespace, key) {
+        return this.sqlite.getByKey(namespace, key);
+    }
+    /**
+     * Update in both backends
+     */
+    async update(id, update) {
+        if (this.config.dualWrite) {
+            // Update both backends
+            const [sqliteResult, agentdbResult] = await Promise.all([
+                this.sqlite.update(id, update),
+                this.agentdb.update(id, update),
+            ]);
+            return agentdbResult || sqliteResult;
+        }
+        else {
+            return this.agentdb.update(id, update);
+        }
+    }
+    /**
+     * Delete from both backends
+     */
+    async delete(id) {
+        if (this.config.dualWrite) {
+            const [sqliteResult, agentdbResult] = await Promise.all([
+                this.sqlite.delete(id),
+                this.agentdb.delete(id),
+            ]);
+            return sqliteResult || agentdbResult;
+        }
+        else {
+            return this.agentdb.delete(id);
+        }
+    }
+    /**
+     * Query routing - semantic goes to AgentDB, structured to SQLite
+     */
+    async query(query) {
+        const startTime = performance.now();
+        let results;
+        // Route based on query type
+        switch (query.type) {
+            case 'exact':
+                // SQLite optimized for exact matches
+                this.stats.sqliteQueries++;
+                results = await this.sqlite.query(query);
+                break;
+            case 'prefix':
+                // SQLite optimized for prefix queries
+                this.stats.sqliteQueries++;
+                results = await this.sqlite.query(query);
+                break;
+            case 'tag':
+                // Both can handle tags, use SQLite for structured filtering
+                this.stats.sqliteQueries++;
+                results = await this.sqlite.query(query);
+                break;
+            case 'semantic':
+                // AgentDB optimized for semantic search
+                this.stats.agentdbQueries++;
+                results = await this.agentdb.query(query);
+                break;
+            case 'hybrid':
+                // Use hybrid query combining both backends
+                this.stats.hybridQueries++;
+                results = await this.queryHybridInternal(query);
+                break;
+            default:
+                // Auto-routing based on query properties
+                results = await this.autoRoute(query);
+        }
+        const duration = performance.now() - startTime;
+        this.stats.totalQueryTime += duration;
+        this.emit('query:completed', { type: query.type, duration, count: results.length });
+        return results;
+    }
+    /**
+     * Structured queries (SQL)
+     * Routes to SQLite for optimal performance
+     */
+    async queryStructured(query) {
+        this.stats.sqliteQueries++;
+        const memoryQuery = {
+            type: query.key ? 'exact' : query.keyPrefix ? 'prefix' : 'hybrid',
+            key: query.key,
+            keyPrefix: query.keyPrefix,
+            namespace: query.namespace,
+            ownerId: query.ownerId,
+            memoryType: query.type,
+            createdAfter: query.createdAfter,
+            createdBefore: query.createdBefore,
+            updatedAfter: query.updatedAfter,
+            updatedBefore: query.updatedBefore,
+            limit: query.limit || 100,
+            offset: query.offset || 0,
+        };
+        return this.sqlite.query(memoryQuery);
+    }
+    /**
+     * Semantic queries (vector)
+     * Routes to AgentDB for HNSW-based vector search
+     */
+    async querySemantic(query) {
+        this.stats.agentdbQueries++;
+        let embedding = query.embedding;
+        // Generate embedding if content provided
+        if (!embedding && query.content && this.config.embeddingGenerator) {
+            embedding = await this.config.embeddingGenerator(query.content);
+        }
+        if (!embedding) {
+            throw new Error('SemanticQuery requires either content or embedding');
+        }
+        const searchResults = await this.agentdb.search(embedding, {
+            k: (query.k || 10) * 2, // Over-fetch to account for post-filtering
+            threshold: query.threshold || this.config.semanticThreshold,
+            filters: query.filters,
+        });
+        let entries = searchResults.map((r) => r.entry);
+        // Apply tag/namespace/type filters that AgentDB may not enforce
+        if (query.filters) {
+            const f = query.filters;
+            if (f.tags && Array.isArray(f.tags)) {
+                const requiredTags = f.tags;
+                entries = entries.filter((e) => requiredTags.every((t) => e.tags.includes(t)));
+            }
+            if (f.namespace && typeof f.namespace === 'string') {
+                entries = entries.filter((e) => e.namespace === f.namespace);
+            }
+            if (f.type && typeof f.type === 'string' && f.type !== 'semantic') {
+                entries = entries.filter((e) => e.type === f.type);
+            }
+        }
+        return entries.slice(0, query.k || 10);
+    }
+    /**
+     * Hybrid queries (combine both)
+     * Intelligently merges results from both backends
+     */
+    async queryHybrid(query) {
+        this.stats.hybridQueries++;
+        const strategy = query.combineStrategy || 'semantic-first';
+        const weights = query.weights || { semantic: 0.7, structured: 0.3 };
+        // Execute both queries in parallel
+        const [semanticResults, structuredResults] = await Promise.all([
+            this.querySemantic(query.semantic),
+            query.structured ? this.queryStructured(query.structured) : Promise.resolve([]),
+        ]);
+        // Combine results based on strategy
+        switch (strategy) {
+            case 'union':
+                return this.combineUnion(semanticResults, structuredResults);
+            case 'intersection':
+                return this.combineIntersection(semanticResults, structuredResults);
+            case 'semantic-first':
+                return this.combineSemanticFirst(semanticResults, structuredResults);
+            case 'structured-first':
+                return this.combineStructuredFirst(semanticResults, structuredResults);
+            default:
+                return this.combineUnion(semanticResults, structuredResults);
+        }
+    }
+    /**
+     * Semantic vector search (routes to AgentDB)
+     */
+    async search(embedding, options) {
+        this.stats.agentdbQueries++;
+        return this.agentdb.search(embedding, options);
+    }
+    /**
+     * Bulk insert to both backends
+     */
+    async bulkInsert(entries) {
+        if (this.config.dualWrite) {
+            await Promise.all([this.sqlite.bulkInsert(entries), this.agentdb.bulkInsert(entries)]);
+        }
+        else {
+            await this.agentdb.bulkInsert(entries);
+        }
+    }
+    /**
+     * Bulk delete from both backends
+     */
+    async bulkDelete(ids) {
+        if (this.config.dualWrite) {
+            const [sqliteCount, agentdbCount] = await Promise.all([
+                this.sqlite.bulkDelete(ids),
+                this.agentdb.bulkDelete(ids),
+            ]);
+            return Math.max(sqliteCount, agentdbCount);
+        }
+        else {
+            return this.agentdb.bulkDelete(ids);
+        }
+    }
+    /**
+     * Count entries (use SQLite for efficiency)
+     */
+    async count(namespace) {
+        return this.sqlite.count(namespace);
+    }
+    /**
+     * List namespaces (use SQLite)
+     */
+    async listNamespaces() {
+        return this.sqlite.listNamespaces();
+    }
+    /**
+     * Clear namespace in both backends
+     */
+    async clearNamespace(namespace) {
+        if (this.config.dualWrite) {
+            const [sqliteCount, agentdbCount] = await Promise.all([
+                this.sqlite.clearNamespace(namespace),
+                this.agentdb.clearNamespace(namespace),
+            ]);
+            return Math.max(sqliteCount, agentdbCount);
+        }
+        else {
+            return this.agentdb.clearNamespace(namespace);
+        }
+    }
+    /**
+     * Get combined statistics from both backends
+     */
+    async getStats() {
+        const [sqliteStats, agentdbStats] = await Promise.all([
+            this.sqlite.getStats(),
+            this.agentdb.getStats(),
+        ]);
+        return {
+            totalEntries: Math.max(sqliteStats.totalEntries, agentdbStats.totalEntries),
+            entriesByNamespace: agentdbStats.entriesByNamespace,
+            entriesByType: agentdbStats.entriesByType,
+            memoryUsage: sqliteStats.memoryUsage + agentdbStats.memoryUsage,
+            hnswStats: agentdbStats.hnswStats ?? {
+                vectorCount: agentdbStats.totalEntries,
+                memoryUsage: 0,
+                avgSearchTime: agentdbStats.avgSearchTime,
+                buildTime: 0,
+                compressionRatio: 1.0,
+            },
+            cacheStats: agentdbStats.cacheStats ?? {
+                hitRate: 0,
+                size: 0,
+                maxSize: 1000,
+            },
+            avgQueryTime: this.stats.hybridQueries + this.stats.sqliteQueries + this.stats.agentdbQueries > 0
+                ? this.stats.totalQueryTime /
+                    (this.stats.hybridQueries + this.stats.sqliteQueries + this.stats.agentdbQueries)
+                : 0,
+            avgSearchTime: agentdbStats.avgSearchTime,
+        };
+    }
+    /**
+     * Health check for both backends
+     */
+    async healthCheck() {
+        const [sqliteHealth, agentdbHealth] = await Promise.all([
+            this.sqlite.healthCheck(),
+            this.agentdb.healthCheck(),
+        ]);
+        const allIssues = [...sqliteHealth.issues, ...agentdbHealth.issues];
+        const allRecommendations = [
+            ...sqliteHealth.recommendations,
+            ...agentdbHealth.recommendations,
+        ];
+        // Determine overall status
+        let status = 'healthy';
+        if (sqliteHealth.status === 'unhealthy' ||
+            agentdbHealth.status === 'unhealthy') {
+            status = 'unhealthy';
+        }
+        else if (sqliteHealth.status === 'degraded' ||
+            agentdbHealth.status === 'degraded') {
+            status = 'degraded';
+        }
+        return {
+            status,
+            components: {
+                storage: sqliteHealth.components.storage,
+                index: agentdbHealth.components.index,
+                cache: agentdbHealth.components.cache,
+            },
+            timestamp: Date.now(),
+            issues: allIssues,
+            recommendations: allRecommendations,
+        };
+    }
+    // ===== Private Methods =====
+    /**
+     * Auto-route queries based on properties
+     */
+    async autoRoute(query) {
+        // If has embedding or content, use semantic search (AgentDB)
+        const hasEmbeddingGenerator = typeof this.config.embeddingGenerator === 'function';
+        if (query.embedding || (query.content && hasEmbeddingGenerator)) {
+            this.stats.agentdbQueries++;
+            return this.agentdb.query(query);
+        }
+        // If has exact key or prefix, use structured search (SQLite)
+        if (query.key || query.keyPrefix) {
+            this.stats.sqliteQueries++;
+            return this.sqlite.query(query);
+        }
+        // For other filters, use routing strategy
+        switch (this.config.routingStrategy) {
+            case 'sqlite-first':
+                this.stats.sqliteQueries++;
+                return this.sqlite.query(query);
+            case 'agentdb-first':
+                this.stats.agentdbQueries++;
+                return this.agentdb.query(query);
+            case 'auto':
+            default:
+                // Default to AgentDB (has caching)
+                this.stats.agentdbQueries++;
+                return this.agentdb.query(query);
+        }
+    }
+    /**
+     * Internal hybrid query implementation
+     */
+    async queryHybridInternal(query) {
+        // If semantic component exists, use hybrid
+        if (query.embedding || query.content) {
+            const semanticQuery = {
+                content: query.content,
+                embedding: query.embedding,
+                k: query.limit || 10,
+                threshold: query.threshold,
+                filters: query,
+            };
+            const structuredQuery = {
+                namespace: query.namespace,
+                key: query.key,
+                keyPrefix: query.keyPrefix,
+                ownerId: query.ownerId,
+                type: query.memoryType,
+                createdAfter: query.createdAfter,
+                createdBefore: query.createdBefore,
+                updatedAfter: query.updatedAfter,
+                updatedBefore: query.updatedBefore,
+                limit: query.limit,
+                offset: query.offset,
+            };
+            return this.queryHybrid({
+                semantic: semanticQuery,
+                structured: structuredQuery,
+                combineStrategy: 'semantic-first',
+            });
+        }
+        // Otherwise, route to structured
+        return this.autoRoute(query);
+    }
+    /**
+     * Combine results using union (all unique results)
+     */
+    combineUnion(semanticResults, structuredResults) {
+        const seen = new Set();
+        const combined = [];
+        for (const entry of [...semanticResults, ...structuredResults]) {
+            if (!seen.has(entry.id)) {
+                seen.add(entry.id);
+                combined.push(entry);
+            }
+        }
+        return combined;
+    }
+    /**
+     * Combine results using intersection (only common results)
+     */
+    combineIntersection(semanticResults, structuredResults) {
+        const semanticIds = new Set(semanticResults.map((e) => e.id));
+        return structuredResults.filter((e) => semanticIds.has(e.id));
+    }
+    /**
+     * Semantic-first: Prefer semantic results, add structured if not present
+     */
+    combineSemanticFirst(semanticResults, structuredResults) {
+        const semanticIds = new Set(semanticResults.map((e) => e.id));
+        const additional = structuredResults.filter((e) => !semanticIds.has(e.id));
+        return [...semanticResults, ...additional];
+    }
+    /**
+     * Structured-first: Prefer structured results, add semantic if not present
+     */
+    combineStructuredFirst(semanticResults, structuredResults) {
+        const structuredIds = new Set(structuredResults.map((e) => e.id));
+        const additional = semanticResults.filter((e) => !structuredIds.has(e.id));
+        return [...structuredResults, ...additional];
+    }
+    // ===== Proxy Methods for AgentDB v3 Controllers (ADR-053 #1212) =====
+    /**
+     * Record feedback for a memory entry.
+     * Delegates to AgentDB's recordFeedback when available.
+     * Gracefully degrades to a no-op when AgentDB is unavailable.
+     */
+    async recordFeedback(entryId, feedback) {
+        const agentdbInstance = this.agentdb.getAgentDB?.();
+        if (agentdbInstance && typeof agentdbInstance.recordFeedback === 'function') {
+            try {
+                await agentdbInstance.recordFeedback(entryId, feedback);
+                this.emit('feedback:recorded', { entryId, score: feedback.score });
+                return true;
+            }
+            catch {
+                // AgentDB feedback recording failed — degrade silently
+            }
+        }
+        return false;
+    }
+    /**
+     * Verify a witness chain for a memory entry.
+     * Delegates to AgentDB's verifyWitnessChain when available.
+     */
+    async verifyWitnessChain(entryId) {
+        const agentdbInstance = this.agentdb.getAgentDB?.();
+        if (agentdbInstance && typeof agentdbInstance.verifyWitnessChain === 'function') {
+            try {
+                return await agentdbInstance.verifyWitnessChain(entryId);
+            }
+            catch {
+                // Verification failed — return degraded result
+            }
+        }
+        return { valid: false, chainLength: 0, errors: ['AgentDB not available'] };
+    }
+    /**
+     * Get the witness chain for a memory entry.
+     * Delegates to AgentDB's getWitnessChain when available.
+     */
+    async getWitnessChain(entryId) {
+        const agentdbInstance = this.agentdb.getAgentDB?.();
+        if (agentdbInstance && typeof agentdbInstance.getWitnessChain === 'function') {
+            try {
+                return await agentdbInstance.getWitnessChain(entryId);
+            }
+            catch {
+                // Chain retrieval failed
+            }
+        }
+        return [];
+    }
+    // ===== Backend Access =====
+    /**
+     * Get underlying backends for advanced operations
+     */
+    getSQLiteBackend() {
+        return this.sqlite;
+    }
+    getAgentDBBackend() {
+        return this.agentdb;
+    }
+}
+export default HybridBackend;
+//# sourceMappingURL=hybrid-backend.js.map

package/src/@claude-flow/memory/dist/hybrid-backend.test.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Tests for HybridBackend (ADR-009)
+ *
+ * Verifies that the hybrid backend correctly routes queries between
+ * SQLite (structured) and AgentDB (semantic) backends.
+ */
+export {};
+//# sourceMappingURL=hybrid-backend.test.d.ts.map