cozo-memory 1.0.0

package/dist/inference-engine.js

@@ -0,0 +1,348 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.InferenceEngine = void 0;
+class InferenceEngine {
+    db;
+    embeddingService;
+    constructor(db, embeddingService) {
+        this.db = db;
+        this.embeddingService = embeddingService;
+    }
+    /**
+     * Infers relations for a specific entity based on various strategies
+     */
+    async inferRelations(entityId) {
+        const results = [];
+        // 1. Co-occurrence in Observations (same entity mentioned in texts)
+        const coOccurrences = await this.findCoOccurrences(entityId);
+        results.push(...coOccurrences);
+        // 2. Vector Proximity (Semantic Similarity)
+        const similar = await this.findSimilarEntities(entityId);
+        results.push(...similar);
+        // 3. Transitive Relations (A -> B, B -> C => A -> C)
+        const transitive = await this.findTransitiveRelations(entityId);
+        results.push(...transitive);
+        const expertise = await this.findTransitiveExpertise(entityId);
+        results.push(...expertise);
+        const custom = await this.applyCustomRules(entityId);
+        results.push(...custom);
+        return results;
+    }
+    async inferImplicitRelations(entityId) {
+        const expertise = await this.findTransitiveExpertise(entityId);
+        const custom = await this.applyCustomRules(entityId);
+        return [...expertise, ...custom];
+    }
+    async findCoOccurrences(entityId) {
+        try {
+            // Search for entities whose names appear in the observations of the target entity
+            // Or entities that reference the same observation (if we had multi-entity observations)
+            // Here: Search for entities whose name appears in the text of the observations of entityId
+            const query = `
+        ?[other_id, other_name, confidence, reason] :=
+          *observation{entity_id: $target, text, @ "NOW"},
+          *entity{id: other_id, name: other_name, @ "NOW"},
+          other_id != $target,
+          str_includes(text, other_name),
+          confidence = 0.7,
+          reason = concat('Mention of ', other_name, ' in observations')
+      `;
+            const res = await this.db.run(query, { target: entityId });
+            return res.rows.map((row) => ({
+                from_id: entityId,
+                to_id: row[0],
+                relation_type: "related_to",
+                confidence: row[2],
+                reason: row[3]
+            }));
+        }
+        catch (e) {
+            console.error("Inference: Co-occurrence failed", e);
+            return [];
+        }
+    }
+    async findSimilarEntities(entityId) {
+        try {
+            // Get embedding of the target entity
+            const entityRes = await this.db.run('?[embedding] := *entity{id: $id, embedding, @ "NOW"}', { id: entityId });
+            if (entityRes.rows.length === 0)
+                return [];
+            const embedding = entityRes.rows[0][0];
+            // Search semantically similar entities via HNSW
+            const query = `
+        ?[other_id, other_name, confidence, reason] :=
+          ~entity:semantic { id: other_id, embedding |
+            query: vec($emb),
+            k: 5,
+            ef: 64,
+            bind_distance: dist
+          },
+          *entity{id: other_id, name: other_name, @ "NOW"},
+          other_id != $id,
+          dist < 0.2,
+          confidence = (1.0 - dist) * 0.9,
+          reason = 'Semantic Similarity'
+      `;
+            const res = await this.db.run(query, { id: entityId, emb: embedding });
+            return res.rows.map((row) => ({
+                from_id: entityId,
+                to_id: row[0],
+                relation_type: "similar_to",
+                confidence: row[2],
+                reason: row[3]
+            }));
+        }
+        catch (e) {
+            console.error("Inference: Similarity failed", e);
+            return [];
+        }
+    }
+    async findTransitiveRelations(entityId) {
+        try {
+            // Finds A -> B -> C and proposes A -> C
+            const query = `
+        ?[target_id, target_name, confidence, reason] :=
+          *relationship{from_id: $id, to_id: mid_id, relation_type: r1, @ "NOW"},
+          *relationship{from_id: mid_id, to_id: target_id, relation_type: r2, @ "NOW"},
+          *entity{id: target_id, name: target_name, @ "NOW"},
+          target_id != $id,
+          confidence = 0.5,
+          reason = concat('Indirect connection via ', r1, ' and ', r2)
+      `;
+            const res = await this.db.run(query, { id: entityId });
+            return res.rows.map((row) => ({
+                from_id: entityId,
+                to_id: row[0],
+                relation_type: "potentially_related",
+                confidence: row[2],
+                reason: row[3]
+            }));
+        }
+        catch (e) {
+            console.error("Inference: Transitive failed", e);
+            return [];
+        }
+    }
+    async findTransitiveExpertise(entityId) {
+        try {
+            const query = `
+        ?[tech_id, tech_name, tech_type, confidence, reason] :=
+          *entity{id: $id, type: 'Person', @ "NOW"},
+          *relationship{from_id: $id, to_id: proj_id, relation_type: 'works_on', @ "NOW"},
+          *relationship{from_id: proj_id, to_id: tech_id, relation_type: 'uses_tech', @ "NOW"},
+          *entity{id: tech_id, name: tech_name, type: tech_type, @ "NOW"},
+          confidence = 0.7,
+          reason = concat('Transitive expertise via works_on -> uses_tech: ', tech_name)
+      `;
+            const res = await this.db.run(query, { id: entityId });
+            const bestByTarget = new Map();
+            for (const row of res.rows) {
+                const to_id = row[0];
+                const confidence = Number(row[3]);
+                const candidate = {
+                    from_id: entityId,
+                    to_id,
+                    relation_type: "expert_in",
+                    confidence,
+                    reason: row[4],
+                    target_name: row[1],
+                    target_type: row[2],
+                };
+                const current = bestByTarget.get(to_id);
+                if (!current || confidence > current.confidence)
+                    bestByTarget.set(to_id, candidate);
+            }
+            return Array.from(bestByTarget.values());
+        }
+        catch (e) {
+            console.error("Inference: Transitive expertise failed", e);
+            return [];
+        }
+    }
+    /**
+     * Performs a recursive "Graph Walk" that follows both explicit relations and
+     * semantic similarity. This enables finding paths like:
+     * A -> (explicit) -> B -> (semantically similar) -> C -> (explicit) -> D
+     *
+     * @param startEntityId The starting entity
+     * @param maxDepth Maximum depth of the search (Default: 3)
+     * @param minSimilarity Minimum similarity for semantic jumps (0.0 - 1.0, Default: 0.7)
+     */
+    async semanticGraphWalk(startEntityId, maxDepth = 3, minSimilarity = 0.7) {
+        try {
+            // Get embedding of the start entity for the first semantic jump
+            const entityRes = await this.db.run('?[embedding] := *entity{id: $id, embedding, @ "NOW"}', { id: startEntityId });
+            if (entityRes.rows.length === 0)
+                return [];
+            const startEmbedding = entityRes.rows[0][0];
+            // Recursive Datalog query
+            // We avoid complex aggregation of strings in Datalog as this can cause errors.
+            // Instead, we implicitly group by 'type' as well and filter later in JS.
+            const query = `
+        # 1. Start point
+        path[id, depth, score, type] := 
+          id = $startId, 
+          depth = 0, 
+          score = 1.0, 
+          type = 'start'
+
+        # 2. Recursion: Follow explicit relations
+        path[next_id, new_depth, new_score, new_type] :=
+          path[curr_id, depth, score, curr_type],
+          depth < $maxDepth,
+          *relationship{from_id: curr_id, to_id: next_id, relation_type, strength, @ "NOW"},
+          new_depth = depth + 1,
+          new_score = score * strength,
+          new_type = if(curr_type == 'start', 'explicit', if(curr_type == 'explicit', 'explicit', 'mixed'))
+
+        # 3. Recursion: Follow semantic similarity (via HNSW Index)
+        path[next_id, new_depth, new_score, new_type] :=
+          path[curr_id, depth, score, curr_type],
+          depth < $maxDepth,
+          *entity{id: curr_id, embedding: curr_emb, @ "NOW"}, # Load embedding
+          # Search for the K nearest neighbors to the current embedding
+          ~entity:semantic { id: next_id |
+            query: curr_emb,
+            k: 5,
+            ef: 20,
+            bind_distance: dist
+          },
+          next_id != curr_id, # No self-reference
+          sim = 1.0 - dist,
+          sim >= $minSim,
+          new_depth = depth + 1,
+          new_score = score * sim * 0.8, # Penalize semantic jumps slightly (damping)
+          new_type = if(curr_type == 'start', 'semantic', if(curr_type == 'semantic', 'semantic', 'mixed'))
+
+        # Aggregate result (Grouping by ID and Type)
+        ?[id, min_depth, max_score, type] := 
+          path[id, d, s, type],
+          id != $startId,
+          min_depth = min(d),
+          max_score = max(s)
+          :limit 100
+      `;
+            const res = await this.db.run(query, {
+                startId: startEntityId,
+                maxDepth: maxDepth,
+                minSim: minSimilarity
+            });
+            // Post-processing in JS: Select best path type per ID
+            const bestPaths = new Map();
+            for (const row of res.rows) {
+                const [id, depth, score, type] = row;
+                // Cozo sometimes returns arrays or raw values, ensure we have Strings/Numbers
+                const cleanId = String(id);
+                const cleanDepth = Number(depth);
+                const cleanScore = Number(score);
+                const cleanType = String(type);
+                if (!bestPaths.has(cleanId) || cleanScore > bestPaths.get(cleanId).path_score) {
+                    bestPaths.set(cleanId, {
+                        entity_id: cleanId,
+                        distance: cleanDepth,
+                        path_score: cleanScore,
+                        path_type: cleanType
+                    });
+                }
+            }
+            return Array.from(bestPaths.values());
+        }
+        catch (e) {
+            console.error("Semantic Graph Walk Failed:", e.message);
+            return [];
+        }
+    }
+    /**
+     * Analyzes the cluster structure directly on the HNSW graph (Layer 0).
+     * This is extremely fast because no vector calculations (K-Means) are necessary,
+     * but the already indexed neighborhood topology is used.
+     */
+    async analyzeHnswClusters() {
+        try {
+            const query = `
+        # 1. Extract edges from HNSW Layer 0 (Base Layer)
+        # We use unweighted edges (weight=1.0) for pure topology analysis
+        hnsw_edges[from, to, weight] := 
+            *entity:semantic{layer: 0, fr_id: from, to_id: to},
+            from != to,
+            weight = 1.0
+
+        # 2. Run Label Propagation on the HNSW graph
+        # This finds natural density clusters in the vector space
+        communities[cluster_id, node_id] <~ LabelPropagation(hnsw_edges[from, to, weight])
+
+        # 3. Get names for the nodes
+        cluster_nodes[cluster_id, name] :=
+            communities[cluster_id, node_id],
+            *entity{id: node_id, name, @ "NOW"}
+        
+        # 4. Aggregate results: size and examples per cluster
+        # We get the raw data and aggregate in JS to avoid string aggregation problems
+        ?[cluster_id, name] :=
+            cluster_nodes[cluster_id, name]
+      `;
+            const res = await this.db.run(query);
+            // Post-Processing: Aggregation in JS
+            const clusterMap = new Map();
+            for (const row of res.rows) {
+                const clusterId = Number(row[0]);
+                const name = row[1];
+                if (!clusterMap.has(clusterId)) {
+                    clusterMap.set(clusterId, { size: 0, examples: [] });
+                }
+                const entry = clusterMap.get(clusterId);
+                entry.size++;
+                if (entry.examples.length < 5) {
+                    entry.examples.push(name);
+                }
+            }
+            const clusters = Array.from(clusterMap.entries()).map(([id, data]) => ({
+                cluster_id: id,
+                size: data.size,
+                examples: data.examples
+            }));
+            // Sort by cluster size descending
+            return clusters.sort((a, b) => b.size - a.size);
+        }
+        catch (e) {
+            console.error("HNSW Cluster Analysis Failed:", e.message);
+            return [];
+        }
+    }
+    async applyCustomRules(entityId) {
+        try {
+            const rulesRes = await this.db.run('?[id, name, datalog] := *inference_rule{id, name, datalog}');
+            if (rulesRes.rows.length === 0)
+                return [];
+            const results = [];
+            for (const row of rulesRes.rows) {
+                const ruleId = String(row[0]);
+                const ruleName = String(row[1]);
+                const datalog = String(row[2]);
+                try {
+                    const res = await this.db.run(datalog, { id: entityId });
+                    for (const r of res.rows) {
+                        if (!r || r.length < 5)
+                            continue;
+                        results.push({
+                            from_id: String(r[0]),
+                            to_id: String(r[1]),
+                            relation_type: String(r[2]),
+                            confidence: Number(r[3]),
+                            reason: String(r[4]),
+                        });
+                    }
+                }
+                catch (e) {
+                    console.error("Inference: Custom rule failed", ruleId, ruleName, e);
+                }
+            }
+            return results;
+        }
+        catch (e) {
+            console.error("Inference: Custom rules load failed", e);
+            return [];
+        }
+    }
+}
+exports.InferenceEngine = InferenceEngine;

package/dist/memory-service.js

@@ -0,0 +1,215 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MemoryService = void 0;
+const uuid_1 = require("uuid");
+class MemoryService {
+    db;
+    embeddings;
+    constructor(db, embeddings) {
+        this.db = db;
+        this.embeddings = embeddings;
+    }
+    async createEntity(name, type, metadata = {}) {
+        const id = (0, uuid_1.v4)();
+        const nameEmbedding = await this.embeddings.embed(name);
+        const contentText = metadata.description ? name + '. ' + metadata.description : name;
+        const contentEmbedding = await this.embeddings.embed(contentText);
+        const entity = {
+            id: id,
+            name: name,
+            type: type,
+            embedding: contentEmbedding,
+            name_embedding: nameEmbedding,
+            metadata: metadata,
+            created_at: Date.now(),
+        };
+        await this.db.createEntity(entity);
+        return entity;
+    }
+    async getEntity(id, asOf) {
+        return this.db.getEntity(id, asOf);
+    }
+    async updateEntity(id, updates) {
+        await this.db.updateEntity(id, updates);
+    }
+    async deleteEntity(id) {
+        await this.db.deleteEntity(id);
+    }
+    async addObservation(entityId, text, metadata = {}) {
+        const id = (0, uuid_1.v4)();
+        const embedding = await this.embeddings.embed(text);
+        const observation = {
+            id: id,
+            entity_id: entityId,
+            text: text,
+            embedding: embedding,
+            metadata: metadata,
+            created_at: Date.now(),
+        };
+        await this.db.addObservation(observation);
+        return observation;
+    }
+    async getObservations(entityId) {
+        return this.db.getObservationsForEntity(entityId);
+    }
+    async createRelation(fromId, toId, relationType, strength = 1.0, metadata = {}) {
+        if (fromId === toId) {
+            throw new Error('Self-references are not allowed');
+        }
+        const relation = {
+            from_id: fromId,
+            to_id: toId,
+            relation_type: relationType,
+            strength: strength,
+            metadata: metadata,
+            created_at: Date.now(),
+        };
+        await this.db.createRelation(relation);
+        return relation;
+    }
+    async getRelations(fromId, toId) {
+        return this.db.getRelations(fromId, toId);
+    }
+    async search(query, limit = 10, entityTypes) {
+        const queryEmbedding = await this.embeddings.embed(query);
+        const vectorResults = await this.db.vectorSearchEntity(queryEmbedding, limit * 2);
+        const keywordResults = await this.db.fullTextSearchEntity(query, limit * 2);
+        const combined = this.reciprocalRankFusion(vectorResults, keywordResults, limit);
+        if (entityTypes && entityTypes.length > 0) {
+            return combined.filter(r => entityTypes.includes(r.entity.type));
+        }
+        return combined;
+    }
+    reciprocalRankFusion(vectorResults, keywordResults, limit, k = 60) {
+        const scores = new Map();
+        vectorResults.forEach((row, index) => {
+            const id = row[0];
+            const rank = index + 1;
+            const rrfScore = 1 / (k + rank);
+            if (!scores.has(id)) {
+                scores.set(id, {
+                    entity: {
+                        id: row[0],
+                        name: row[1],
+                        type: row[2],
+                        metadata: row[3],
+                    },
+                    score: 0,
+                    sources: new Set(),
+                });
+            }
+            const entry = scores.get(id);
+            if (entry) {
+                entry.score += rrfScore;
+                entry.sources.add('vector');
+            }
+        });
+        keywordResults.forEach((row, index) => {
+            const id = row[0];
+            const rank = index + 1;
+            const rrfScore = 1 / (k + rank);
+            if (!scores.has(id)) {
+                scores.set(id, {
+                    entity: {
+                        id: row[0],
+                        name: row[1],
+                        type: row[2],
+                        metadata: row[3],
+                    },
+                    score: 0,
+                    sources: new Set(),
+                });
+            }
+            const entry = scores.get(id);
+            if (entry) {
+                entry.score += rrfScore;
+                entry.sources.add('keyword');
+            }
+        });
+        const results = Array.from(scores.entries()).map(([id, data]) => ({
+            entity: data.entity,
+            score: data.score,
+            source: data.sources.has('vector') && data.sources.has('keyword')
+                ? 'vector'
+                : (data.sources.has('vector') ? 'vector' : 'keyword'),
+        }));
+        results.sort((a, b) => b.score - a.score);
+        return results.slice(0, limit);
+    }
+    async getContext(query, contextWindow = 20) {
+        const searchResults = await this.search(query, contextWindow);
+        const entitiesWithObservations = await Promise.all(searchResults.map(async (result) => {
+            const observations = await this.getObservations(result.entity.id);
+            const relations = await this.getRelations(result.entity.id);
+            return {
+                ...result,
+                observations: observations.slice(0, 5),
+                relations: relations.slice(0, 5),
+            };
+        }));
+        return {
+            query: query,
+            entities: entitiesWithObservations,
+            total_entities: searchResults.length,
+        };
+    }
+    async health() {
+        const stats = await this.db.getStats();
+        const cacheStats = this.embeddings.getCacheStats();
+        return {
+            status: 'healthy',
+            database: stats,
+            cache: cacheStats,
+            timestamp: new Date().toISOString(),
+        };
+    }
+    async createSnapshot(metadata = {}) {
+        const snapshotId = (0, uuid_1.v4)();
+        const stats = await this.db.getStats();
+        console.error('[MemoryService] Snapshot created:', snapshotId, stats);
+        return snapshotId;
+    }
+    async ingestFile(content, format, entityName, entityType = 'Document', chunking = 'paragraphs') {
+        const searchResults = await this.search(entityName, 1);
+        let entity;
+        if (searchResults.length > 0 && searchResults[0].entity.name.toLowerCase() === entityName.toLowerCase()) {
+            entity = searchResults[0].entity;
+        }
+        else {
+            entity = await this.createEntity(entityName, entityType, { format: format });
+        }
+        let chunks = [];
+        if (format === 'markdown' && chunking === 'paragraphs') {
+            chunks = content.split(/\n\s*\n/).filter((c) => c.trim().length > 0);
+        }
+        else if (format === 'json') {
+            try {
+                const data = JSON.parse(content);
+                chunks = [JSON.stringify(data, null, 2)];
+            }
+            catch (e) {
+                chunks = [content];
+            }
+        }
+        else {
+            chunks = [content];
+        }
+        const observations = [];
+        const maxChunks = Math.min(chunks.length, 50);
+        for (let i = 0; i < maxChunks; i++) {
+            const chunk = chunks[i];
+            const obs = await this.addObservation(entity.id, chunk, {
+                chunk_index: i,
+                total_chunks: chunks.length,
+            });
+            observations.push(obs);
+        }
+        return {
+            entity_id: entity.id,
+            entity_name: entity.name,
+            chunks_processed: observations.length,
+            total_chunks: chunks.length,
+        };
+    }
+}
+exports.MemoryService = MemoryService;

package/dist/test-advanced-filters.js

@@ -0,0 +1,64 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const cozo_node_1 = require("cozo-node");
+const hybrid_search_1 = require("./hybrid-search");
+const embedding_service_1 = require("./embedding-service");
+const fs_1 = __importDefault(require("fs"));
+async function testAdvancedSearchFilters() {
+    const dbPath = "test-advanced-filters.db";
+    if (fs_1.default.existsSync(dbPath)) {
+        try {
+            fs_1.default.unlinkSync(dbPath);
+        }
+        catch (e) { }
+    }
+    const db = new cozo_node_1.CozoDb("sqlite", dbPath);
+    const embeddingService = new embedding_service_1.EmbeddingService();
+    // Setup schema
+    const EMBEDDING_DIM = 1024;
+    await db.run(`{:create entity {id: String, created_at: Validity => name: String, type: String, embedding: <F32; ${EMBEDDING_DIM}>, metadata: Json}}`);
+    await db.run(`{::hnsw create entity:semantic {dim: ${EMBEDDING_DIM}, m: 16, dtype: F32, fields: [embedding], distance: Cosine, ef_construction: 200}}`);
+    await db.run(`{:create entity_rank {entity_id: String => pagerank: Float}}`);
+    const hybridSearch = new hybrid_search_1.HybridSearch(db, embeddingService);
+    // Insert some data
+    const now = Date.now();
+    const vec = new Array(EMBEDDING_DIM).fill(0.1);
+    const data = [
+        ['id1', [now, true], 'Alice', 'Person', vec, { role: 'Developer', level: 5 }],
+        ['id2', [now, true], 'Bob', 'Person', vec, { role: 'Manager', level: 8 }],
+        ['id3', [now, true], 'Project X', 'Project', vec, { status: 'active' }]
+    ];
+    await db.run(`?[id, created_at, name, type, embedding, metadata] <- $data :put entity {id, created_at => name, type, embedding, metadata}`, { data });
+    await db.run(`?[entity_id, pagerank] <- [['id1', 0.5], ['id2', 0.5], ['id3', 0.5]] :put entity_rank {entity_id => pagerank}`);
+    console.log("--- Testing Filter: Entity Type 'Person' ---");
+    const res1 = await hybridSearch.advancedSearch({
+        query: "test",
+        filters: { entityTypes: ["Person"] }
+    });
+    console.log("Results 1 (Person):", res1.map(r => r.id));
+    console.log("\n--- Testing Filter: Metadata role='Manager' ---");
+    const res2 = await hybridSearch.advancedSearch({
+        query: "test",
+        filters: { metadata: { role: "Manager" } }
+    });
+    console.log("Results 2 (Manager):", res2.map(r => r.id));
+    console.log("\n--- Testing Filter: Metadata level=5 ---");
+    const res3 = await hybridSearch.advancedSearch({
+        query: "test",
+        filters: { metadata: { level: 5 } }
+    });
+    console.log("Results 3 (Level 5):", res3.map(r => r.id));
+    console.log("\n--- Testing Multi-Filter: Type 'Person' + role='Developer' ---");
+    const res4 = await hybridSearch.advancedSearch({
+        query: "test",
+        filters: {
+            entityTypes: ["Person"],
+            metadata: { role: "Developer" }
+        }
+    });
+    console.log("Results 4 (Dev Person):", res4.map(r => r.id));
+}
+testAdvancedSearchFilters();