cozo-memory 1.1.2 → 1.1.4

package/dist/dynamic-fusion.js

@@ -0,0 +1,602 @@
+"use strict";
+/**
+ * Dynamic Fusion Framework for CozoDB Memory
+ *
+ * Inspired by Allan-Poe (arXiv:2511.00855) - All-in-one Graph-based Indexing
+ *
+ * Combines 4 retrieval paths with dynamic weighting:
+ * 1. Dense Vector Search (HNSW semantic similarity)
+ * 2. Sparse Vector Search (TF-IDF/BM25 keyword matching)
+ * 3. Full-Text Search (FTS exact/fuzzy matching)
+ * 4. Graph Traversal (relationship-based retrieval)
+ *
+ * Key Innovation: Dynamic fusion without index reconstruction
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DynamicFusionSearch = exports.DEFAULT_FUSION_CONFIG = void 0;
+/**
+ * Default fusion configuration
+ */
+exports.DEFAULT_FUSION_CONFIG = {
+    vector: {
+        enabled: true,
+        weight: 0.4,
+        topK: 20,
+        efSearch: 100
+    },
+    sparse: {
+        enabled: true,
+        weight: 0.3,
+        topK: 20,
+        minScore: 0.1
+    },
+    fts: {
+        enabled: true,
+        weight: 0.2,
+        topK: 20,
+        fuzzy: true
+    },
+    graph: {
+        enabled: true,
+        weight: 0.1,
+        maxDepth: 2,
+        maxResults: 20
+    },
+    fusion: {
+        strategy: 'rrf',
+        rrfK: 60,
+        minScore: 0.0,
+        deduplication: true
+    }
+};
+/**
+ * Dynamic Fusion Search Engine
+ *
+ * Combines multiple retrieval paths with configurable weights
+ * without requiring index reconstruction
+ */
+class DynamicFusionSearch {
+    db;
+    embeddings;
+    constructor(db, embeddings) {
+        this.db = db;
+        this.embeddings = embeddings;
+    }
+    /**
+     * Execute dynamic fusion search
+     */
+    async search(query, config = {}) {
+        const startTime = Date.now();
+        // Merge with defaults
+        const fullConfig = this.mergeConfig(config);
+        console.log('[DynamicFusion] Starting search with config:', {
+            query,
+            enabledPaths: this.getEnabledPaths(fullConfig)
+        });
+        // Execute all enabled paths in parallel
+        const pathResults = await this.executeAllPaths(query, fullConfig);
+        // Fuse results based on strategy
+        const fusedResults = this.fuseResults(pathResults, fullConfig);
+        // Calculate statistics
+        const stats = this.calculateStats(pathResults, fusedResults, startTime);
+        console.log('[DynamicFusion] Search completed:', {
+            totalResults: fusedResults.length,
+            pathContributions: stats.pathContributions,
+            fusionTime: stats.fusionTime
+        });
+        return { results: fusedResults, stats };
+    }
+    /**
+     * Execute all enabled retrieval paths in parallel
+     */
+    async executeAllPaths(query, config) {
+        const promises = [];
+        const pathNames = [];
+        // Vector search
+        if (config.vector?.enabled) {
+            pathNames.push('vector');
+            promises.push(this.executeVectorSearch(query, config.vector));
+        }
+        // Sparse search
+        if (config.sparse?.enabled) {
+            pathNames.push('sparse');
+            promises.push(this.executeSparseSearch(query, config.sparse));
+        }
+        // FTS search
+        if (config.fts?.enabled) {
+            pathNames.push('fts');
+            promises.push(this.executeFTSSearch(query, config.fts));
+        }
+        // Graph search
+        if (config.graph?.enabled) {
+            pathNames.push('graph');
+            promises.push(this.executeGraphSearch(query, config.graph));
+        }
+        const results = await Promise.all(promises);
+        // Map results back to path names
+        const pathResults = {};
+        pathNames.forEach((name, idx) => {
+            pathResults[name] = results[idx];
+        });
+        return pathResults;
+    }
+    /**
+     * Execute dense vector search (HNSW)
+     */
+    async executeVectorSearch(query, config) {
+        const startTime = Date.now();
+        try {
+            // Generate query embedding
+            const embedding = await this.embeddings.embed(query);
+            // HNSW vector search using correct CozoDB syntax
+            const datalogQuery = `
+        ?[id, name, type, score, metadata] := 
+          ~entity:semantic{
+            id | 
+            query: vec($embedding), 
+            k: ${config.topK},
+            ef: ${config.efSearch || 100},
+            bind_distance: dist
+          },
+          *entity{
+            id,
+            name,
+            type,
+            metadata,
+            @ "NOW"
+          },
+          score = 1.0 - dist
+        
+        :order -score
+        :limit ${config.topK}
+      `;
+            const dbResult = await this.db.run(datalogQuery, { embedding });
+            const results = dbResult.rows.map((row) => ({
+                id: row[0],
+                name: row[1],
+                type: row[2],
+                entity_id: row[0],
+                score: row[3],
+                metadata: row[4],
+                source: 'vector',
+                rawScore: row[3]
+            }));
+            return {
+                results,
+                time: Date.now() - startTime
+            };
+        }
+        catch (error) {
+            console.error('[DynamicFusion] Vector search error:', error);
+            return { results: [], time: Date.now() - startTime };
+        }
+    }
+    /**
+     * Execute sparse vector search (keyword-based)
+     */
+    async executeSparseSearch(query, config) {
+        const startTime = Date.now();
+        try {
+            // Extract keywords (simple tokenization)
+            const keywords = query.toLowerCase()
+                .split(/\s+/)
+                .filter(w => w.length > 2);
+            if (keywords.length === 0) {
+                return { results: [], time: Date.now() - startTime };
+            }
+            // Pad keywords to always have 3 (for simpler query)
+            while (keywords.length < 3) {
+                keywords.push('');
+            }
+            // Keyword matching with TF-IDF-like scoring using str_includes
+            const datalogQuery = `
+        ?[id, name, type, score, metadata] := 
+          *entity{
+            id,
+            name,
+            type,
+            metadata
+          },
+          name_lower = lowercase(name),
+          match_count = if(str_includes(name_lower, $kw1), 1, 0) + if(str_includes(name_lower, $kw2), 1, 0) + if(str_includes(name_lower, $kw3), 1, 0),
+          match_count > 0,
+          score = to_float(match_count) / to_float(length(name_lower))
+        
+        :order -score
+        :limit ${config.topK}
+      `;
+            const params = {
+                kw1: keywords[0],
+                kw2: keywords[1],
+                kw3: keywords[2]
+            };
+            const dbResult = await this.db.run(datalogQuery, params);
+            return {
+                results: dbResult.rows
+                    .filter((row) => row[3] >= (config.minScore || 0.1))
+                    .map((row) => ({
+                    id: row[0],
+                    name: row[1],
+                    type: row[2],
+                    entity_id: row[0],
+                    score: row[3],
+                    metadata: row[4],
+                    source: 'sparse',
+                    rawScore: row[3]
+                })),
+                time: Date.now() - startTime
+            };
+        }
+        catch (error) {
+            console.error('[DynamicFusion] Sparse search error:', error);
+            return { results: [], time: Date.now() - startTime };
+        }
+    }
+    /**
+     * Execute full-text search
+     */
+    async executeFTSSearch(query, config) {
+        const startTime = Date.now();
+        try {
+            // FTS search on entity names using correct CozoDB syntax
+            // According to CozoDB v0.7 docs: bind_score (not bind_score_bm_25)
+            // score_kind can be 'tf_idf' or 'tf' (default is 'tf_idf')
+            const datalogQuery = `
+        ?[id, name, type, score, metadata] := 
+          ~entity:fts{
+            id | 
+            query: $query,
+            k: ${config.topK},
+            score_kind: 'tf_idf',
+            bind_score: score
+          },
+          *entity{
+            id,
+            name,
+            type,
+            metadata,
+            @ "NOW"
+          }
+        
+        :order -score
+        :limit ${config.topK}
+      `;
+            const dbResult = await this.db.run(datalogQuery, { query });
+            return {
+                results: dbResult.rows.map((row) => ({
+                    id: row[0],
+                    name: row[1],
+                    type: row[2],
+                    entity_id: row[0],
+                    score: row[3],
+                    metadata: row[4],
+                    source: 'fts',
+                    rawScore: row[3]
+                })),
+                time: Date.now() - startTime
+            };
+        }
+        catch (error) {
+            console.error('[DynamicFusion] FTS search error:', error);
+            return { results: [], time: Date.now() - startTime };
+        }
+    }
+    /**
+     * Execute graph traversal search
+     */
+    async executeGraphSearch(query, config) {
+        const startTime = Date.now();
+        try {
+            // First, find seed nodes via vector search
+            const embedding = await this.embeddings.embed(query);
+            // HNSW index returns 'id' not 'entity_id'
+            const seedQuery = `
+        ?[id] := 
+          ~entity:semantic{
+            id | 
+            query: vec($embedding), 
+            k: 5,
+            ef: 100
+          }
+      `;
+            const seedResult = await this.db.run(seedQuery, { embedding });
+            if (seedResult.rows.length === 0) {
+                return { results: [], time: Date.now() - startTime };
+            }
+            // Graph traversal from seeds
+            const relationFilter = config.relationTypes && config.relationTypes.length > 0
+                ? `is_in(relation_type, [${config.relationTypes.map(t => `"${t}"`).join(', ')}])`
+                : 'true';
+            const seedIds = seedResult.rows.map((row) => `"${row[0]}"`).join(', ');
+            const graphQuery = `
+        seed[id] := id in [${seedIds}]
+        
+        reachable[to_id, depth] := 
+          seed[from_id],
+          *relationship{from_id, to_id, relation_type, @ "NOW"},
+          ${relationFilter},
+          depth = 1
+        
+        reachable[to_id, depth] := 
+          reachable[from_id, prev_depth],
+          prev_depth < ${config.maxDepth},
+          *relationship{from_id, to_id, relation_type, @ "NOW"},
+          ${relationFilter},
+          depth = prev_depth + 1
+        
+        ?[id, name, type, score, metadata] := 
+          reachable[id, depth],
+          *entity{
+            id,
+            name,
+            type,
+            metadata,
+            @ "NOW"
+          },
+          score = 1.0 / to_float(depth)
+        
+        :order -score
+        :limit ${config.maxResults || 20}
+      `;
+            const graphResult = await this.db.run(graphQuery, {});
+            return {
+                results: graphResult.rows.map((row) => ({
+                    id: row[0],
+                    name: row[1],
+                    type: row[2],
+                    entity_id: row[0],
+                    score: row[3],
+                    metadata: row[4],
+                    source: 'graph',
+                    rawScore: row[3]
+                })),
+                time: Date.now() - startTime
+            };
+        }
+        catch (error) {
+            console.error('[DynamicFusion] Graph search error:', error);
+            return { results: [], time: Date.now() - startTime };
+        }
+    }
+    /**
+     * Fuse results from multiple paths
+     */
+    fuseResults(pathResults, config) {
+        const strategy = config.fusion?.strategy || 'rrf';
+        switch (strategy) {
+            case 'rrf':
+                return this.fuseRRF(pathResults, config);
+            case 'weighted_sum':
+                return this.fuseWeightedSum(pathResults, config);
+            case 'max':
+                return this.fuseMax(pathResults, config);
+            case 'adaptive':
+                return this.fuseAdaptive(pathResults, config);
+            default:
+                return this.fuseRRF(pathResults, config);
+        }
+    }
+    /**
+     * Reciprocal Rank Fusion (RRF)
+     */
+    fuseRRF(pathResults, config) {
+        const k = config.fusion?.rrfK || 60;
+        const entityScores = new Map();
+        // Process each path
+        for (const [pathName, pathData] of Object.entries(pathResults)) {
+            if (!pathData || !pathData.results)
+                continue;
+            const weight = config[pathName]?.weight || 1.0;
+            pathData.results.forEach((result, rank) => {
+                const entityId = result.entity_id;
+                const rrfScore = weight / (k + rank + 1);
+                if (!entityScores.has(entityId)) {
+                    entityScores.set(entityId, {
+                        score: 0,
+                        sources: new Set(),
+                        pathScores: {},
+                        entity: result
+                    });
+                }
+                const entry = entityScores.get(entityId);
+                entry.score += rrfScore;
+                entry.sources.add(pathName);
+                entry.pathScores[pathName] = rrfScore;
+            });
+        }
+        // Convert to array and sort
+        const results = Array.from(entityScores.entries())
+            .map(([entityId, data]) => ({
+            id: data.entity.id,
+            entity_id: entityId,
+            name: data.entity.name,
+            type: data.entity.type,
+            score: data.score,
+            source: Array.from(data.sources).join('+'),
+            metadata: data.entity.metadata,
+            pathScores: data.pathScores,
+            explanation: `RRF fusion from ${data.sources.size} path(s)`
+        }))
+            .sort((a, b) => b.score - a.score);
+        // Apply deduplication and min score filter
+        const minScore = config.fusion?.minScore || 0.0;
+        const filtered = results.filter(r => r.score >= minScore);
+        return config.fusion?.deduplication
+            ? this.deduplicateResults(filtered)
+            : filtered;
+    }
+    /**
+     * Weighted sum fusion
+     */
+    fuseWeightedSum(pathResults, config) {
+        const entityScores = new Map();
+        // Process each path
+        for (const [pathName, pathData] of Object.entries(pathResults)) {
+            if (!pathData || !pathData.results)
+                continue;
+            const weight = config[pathName]?.weight || 1.0;
+            pathData.results.forEach((result) => {
+                const entityId = result.entity_id;
+                const weightedScore = result.rawScore * weight;
+                if (!entityScores.has(entityId)) {
+                    entityScores.set(entityId, {
+                        score: 0,
+                        sources: new Set(),
+                        pathScores: {},
+                        entity: result
+                    });
+                }
+                const entry = entityScores.get(entityId);
+                entry.score += weightedScore;
+                entry.sources.add(pathName);
+                entry.pathScores[pathName] = weightedScore;
+            });
+        }
+        // Convert and sort
+        const results = Array.from(entityScores.entries())
+            .map(([entityId, data]) => ({
+            id: data.entity.id,
+            entity_id: entityId,
+            name: data.entity.name,
+            type: data.entity.type,
+            score: data.score,
+            source: Array.from(data.sources).join('+'),
+            metadata: data.entity.metadata,
+            pathScores: data.pathScores,
+            explanation: `Weighted sum from ${data.sources.size} path(s)`
+        }))
+            .sort((a, b) => b.score - a.score);
+        const minScore = config.fusion?.minScore || 0.0;
+        const filtered = results.filter(r => r.score >= minScore);
+        return config.fusion?.deduplication
+            ? this.deduplicateResults(filtered)
+            : filtered;
+    }
+    /**
+     * Max score fusion
+     */
+    fuseMax(pathResults, config) {
+        const entityScores = new Map();
+        // Process each path
+        for (const [pathName, pathData] of Object.entries(pathResults)) {
+            if (!pathData || !pathData.results)
+                continue;
+            const weight = config[pathName]?.weight || 1.0;
+            pathData.results.forEach((result) => {
+                const entityId = result.entity_id;
+                const weightedScore = result.rawScore * weight;
+                if (!entityScores.has(entityId)) {
+                    entityScores.set(entityId, {
+                        score: weightedScore,
+                        sources: new Set([pathName]),
+                        pathScores: { [pathName]: weightedScore },
+                        entity: result
+                    });
+                }
+                else {
+                    const entry = entityScores.get(entityId);
+                    if (weightedScore > entry.score) {
+                        entry.score = weightedScore;
+                    }
+                    entry.sources.add(pathName);
+                    entry.pathScores[pathName] = weightedScore;
+                }
+            });
+        }
+        // Convert and sort
+        const results = Array.from(entityScores.entries())
+            .map(([entityId, data]) => ({
+            id: data.entity.id,
+            entity_id: entityId,
+            name: data.entity.name,
+            type: data.entity.type,
+            score: data.score,
+            source: Array.from(data.sources).join('+'),
+            metadata: data.entity.metadata,
+            pathScores: data.pathScores,
+            explanation: `Max score from ${data.sources.size} path(s)`
+        }))
+            .sort((a, b) => b.score - a.score);
+        const minScore = config.fusion?.minScore || 0.0;
+        const filtered = results.filter(r => r.score >= minScore);
+        return config.fusion?.deduplication
+            ? this.deduplicateResults(filtered)
+            : filtered;
+    }
+    /**
+     * Adaptive fusion (query-dependent weighting)
+     */
+    fuseAdaptive(pathResults, config) {
+        // Analyze query characteristics to adjust weights
+        // For now, fall back to RRF
+        // TODO: Implement adaptive weighting based on query analysis
+        console.log('[DynamicFusion] Adaptive fusion not yet implemented, using RRF');
+        return this.fuseRRF(pathResults, config);
+    }
+    /**
+     * Remove duplicate results
+     */
+    deduplicateResults(results) {
+        const seen = new Set();
+        return results.filter(r => {
+            if (seen.has(r.entity_id)) {
+                return false;
+            }
+            seen.add(r.entity_id);
+            return true;
+        });
+    }
+    /**
+     * Calculate search statistics
+     */
+    calculateStats(pathResults, fusedResults, startTime) {
+        const pathContributions = {
+            vector: 0,
+            sparse: 0,
+            fts: 0,
+            graph: 0
+        };
+        const pathTimes = {};
+        // Count contributions
+        for (const [pathName, pathData] of Object.entries(pathResults)) {
+            if (pathData && pathData.results) {
+                pathContributions[pathName] = pathData.results.length;
+                pathTimes[pathName] = pathData.time;
+            }
+        }
+        return {
+            totalResults: fusedResults.length,
+            pathContributions,
+            fusionTime: Date.now() - startTime,
+            pathTimes
+        };
+    }
+    /**
+     * Merge user config with defaults
+     */
+    mergeConfig(config) {
+        return {
+            vector: { ...exports.DEFAULT_FUSION_CONFIG.vector, ...config.vector },
+            sparse: { ...exports.DEFAULT_FUSION_CONFIG.sparse, ...config.sparse },
+            fts: { ...exports.DEFAULT_FUSION_CONFIG.fts, ...config.fts },
+            graph: { ...exports.DEFAULT_FUSION_CONFIG.graph, ...config.graph },
+            fusion: { ...exports.DEFAULT_FUSION_CONFIG.fusion, ...config.fusion }
+        };
+    }
+    /**
+     * Get list of enabled paths
+     */
+    getEnabledPaths(config) {
+        const paths = [];
+        if (config.vector?.enabled)
+            paths.push('vector');
+        if (config.sparse?.enabled)
+            paths.push('sparse');
+        if (config.fts?.enabled)
+            paths.push('fts');
+        if (config.graph?.enabled)
+            paths.push('graph');
+        return paths;
+    }
+}
+exports.DynamicFusionSearch = DynamicFusionSearch;

package/dist/hybrid-search.js

@@ -204,7 +204,7 @@ class HybridSearch {
             semanticCall += `, filter: ${hnswFilters.join(" && ")}`;
         }
         semanticCall += `}`;
-        let bodyConstraints = [semanticCall, `*entity{id, name, type, metadata, created_at}`];
+        let bodyConstraints = [semanticCall, `*entity{id, name, type, metadata, created_at, @ "NOW"}`];
         if (metaJoins.length > 0) {
             bodyConstraints.push(...metaJoins);
         }
@@ -232,10 +232,10 @@ class HybridSearch {
             `rank_val[id, r] := *entity{id, @ "NOW"}, not *entity_rank{entity_id: id}, r = 0.0`
         ];
         if (graphConstraints?.requiredRelations && graphConstraints.requiredRelations.length > 0) {
-            helperRules.push(`rel_match[id, rel_type] := *relationship{from_id: id, relation_type: rel_type}`, `rel_match[id, rel_type] := *relationship{to_id: id, relation_type: rel_type}`);
+            helperRules.push(`rel_match[id, rel_type] := *relationship{from_id: id, relation_type: rel_type, @ "NOW"}`, `rel_match[id, rel_type] := *relationship{to_id: id, relation_type: rel_type, @ "NOW"}`);
         }
         if (graphConstraints?.targetEntityIds && graphConstraints.targetEntityIds.length > 0) {
-            helperRules.push(`target_match[id, target_id] := *relationship{from_id: id, to_id: target_id}`, `target_match[id, target_id] := *relationship{to_id: id, from_id: target_id}`);
+            helperRules.push(`target_match[id, target_id] := *relationship{from_id: id, to_id: target_id, @ "NOW"}`, `target_match[id, target_id] := *relationship{to_id: id, from_id: target_id, @ "NOW"}`);
         }
         const datalogQuery = [
             ...helperRules,
@@ -360,7 +360,7 @@ class HybridSearch {
 
       result_entities[id, final_score, depth] := path[seed_id, id, depth], seeds[seed_id, seed_score], rank_val[id, pr], final_score = seed_score * (1.0 - 0.2 * depth)
       
-      ?[id, name, type, metadata, created_at, score, source, text] := result_entities[id, score, depth], *entity{id, name, type, metadata, created_at}, source = 'graph_rag_entity', text = ''
+      ?[id, name, type, metadata, created_at, score, source, text] := result_entities[id, score, depth], *entity{id, name, type, metadata, created_at, @ "NOW"}, source = 'graph_rag_entity', text = ''
       
       :sort -score
       :limit $limit