cozo-memory 1.1.2 → 1.1.4

package/dist/index.js

@@ -16,6 +16,8 @@ const fs_1 = __importDefault(require("fs"));
 const pdf_mjs_1 = require("pdfjs-dist/legacy/build/pdf.mjs");
 const hybrid_search_1 = require("./hybrid-search");
 const inference_engine_1 = require("./inference-engine");
+const dynamic_fusion_1 = require("./dynamic-fusion");
+const adaptive_retrieval_1 = require("./adaptive-retrieval");
 exports.DB_PATH = path_1.default.resolve(__dirname, "..", "memory_db.cozo");
 const DB_ENGINE = process.env.DB_ENGINE || "sqlite"; // "sqlite" or "rocksdb"
 exports.USER_ENTITY_ID = "global_user_profile";
@@ -26,8 +28,11 @@ class MemoryServer {
     mcp;
     embeddingService;
     hybridSearch;
+    dynamicFusion;
+    adaptiveRetrieval;
     inferenceEngine;
     initPromise;
+    compactionLocks = new Set();
     // Metrics tracking
     metrics = {
         operations: {
@@ -75,6 +80,8 @@ class MemoryServer {
         console.error(`[DB] Using backend: ${DB_ENGINE}, path: ${fullDbPath}`);
         this.embeddingService = new embedding_service_1.EmbeddingService();
         this.hybridSearch = new hybrid_search_1.HybridSearch(this.db, this.embeddingService);
+        this.dynamicFusion = new dynamic_fusion_1.DynamicFusionSearch(this.db, this.embeddingService);
+        this.adaptiveRetrieval = new adaptive_retrieval_1.AdaptiveGraphRetrieval(this.db, this.embeddingService);
         this.inferenceEngine = new inference_engine_1.InferenceEngine(this.db, this.embeddingService);
         this.mcp = new fastmcp_1.FastMCP({
             name: "cozo-memory-server",
@@ -1422,13 +1429,13 @@ class MemoryServer {
             if (session_id) {
                 try {
                     await this.db.run(`
-            ?[id, last_active, status, metadata] :=
-              *session_state{id, metadata},
-              id = $id,
+            ?[session_id, last_active, status, metadata] :=
+              *session_state{session_id, metadata},
+              session_id = $session_id,
               last_active = $now,
               status = "active"
-            :update session_state {id => last_active, status, metadata}
-          `, { id: session_id, now: Math.floor(now / 1000000) });
+            :update session_state {session_id => last_active, status, metadata}
+          `, { session_id: session_id, now: Math.floor(now / 1000000) });
                 }
                 catch (e) {
                     console.warn(`[AddObservation] Session activity update failed for ${session_id}:`, e.message);
@@ -1437,6 +1444,12 @@ class MemoryServer {
             // Optional: Automatic inference after new observation (in background)
             const suggestionsRaw = await this.inferenceEngine.inferRelations(entityId);
             const suggestions = await this.formatInferredRelationsForContext(suggestionsRaw);
+            // Automatic Compaction (Threshold-based)
+            // Skip for compaction-generated observations to prevent recursive loop
+            const metadataKind = args.metadata?.kind;
+            if (metadataKind !== "compaction" && metadataKind !== "session_summary") {
+                this.compactEntity({ entity_id: entityId }).catch(e => console.error(`[AutoCompact] Error for ${entityId}:`, e));
+            }
             const created_at_iso = new Date(Math.floor(now / 1000)).toISOString();
             return {
                 id,
@@ -1472,6 +1485,13 @@ class MemoryServer {
     }
     async stopSession(args) {
         await this.initPromise;
+        // Automatic Session Compaction
+        try {
+            await this.compactSession({ session_id: args.id });
+        }
+        catch (e) {
+            console.warn(`[AutoCompact] Session compaction failed for ${args.id}:`, e.message);
+        }
         await this.db.run(`
       ?[session_id, last_active, status, metadata] :=
         *session_state{session_id, last_active, metadata},
@@ -1796,9 +1816,9 @@ ids[id] <- $ids
         else {
             // Select top 5 entities with the most observations
             const res = await this.db.run(`
-        ?[id, name, type, count(id)] := 
-          *entity{id, name, type, @ "NOW"}, 
-          *observation{entity_id: id, @ "NOW"}
+        ?[eid, ename, etype, count(oid)] := 
+          *entity{id: eid, name: ename, type: etype, @ "NOW"}, 
+          *observation{entity_id: eid, id: oid, @ "NOW"}
       `);
             entitiesToReflect = res.rows
                 .map((r) => ({ id: String(r[0]), name: String(r[1]), type: String(r[2]), count: Number(r[3]) }))
@@ -1859,8 +1879,8 @@ If no special patterns are recognizable, answer with "No new insights".`;
                 for (const candidate of candidates) {
                     // Check if relationship already exists
                     const existing = await this.db.run(`
-            ?[count(from_id)] := *relationship{from_id, to_id, relation_type, @ "NOW"},
-              from_id = $from, to_id = $to, relation_type = $rel
+            ?[count(fid)] := *relationship{from_id: fid, to_id: tid, relation_type: rel, @ "NOW"},
+              fid = $from, tid = $to, rel = $rel
           `, { from: candidate.from_id, to: candidate.to_id, rel: candidate.relation_type });
                     if (Number(existing.rows[0][0]) > 0)
                         continue;
@@ -2191,6 +2211,132 @@ ids[id] <- $ids
             return { error: error.message || "Error during ingest" };
         }
     }
+    async compactSession(args) {
+        await this.initPromise;
+        const model = args.model || "demyagent-4b-i1:Q6_K";
+        try {
+            // 1. Get all observations from this session
+            const obsRes = await this.db.run('?[text, ts] := *observation{session_id: $sid, text, created_at, @ "NOW"}, ts = to_int(created_at) :order ts', {
+                sid: args.session_id
+            });
+            if (obsRes.rows.length < 3) {
+                return { status: "skipped", reason: "Too few observations for session compaction" };
+            }
+            const observations = obsRes.rows.map((r) => r[0]);
+            const systemPrompt = `You are a memory compaction module. Summarize the following session observations into exactly 2-3 concise bullet points. 
+Focus only on core facts, preferences, or important events. Skip conversational filler.
+Format:
+- Bullet Point 1
+- Bullet Point 2
+(- Bullet Point 3)`;
+            const userPrompt = `Session ID: ${args.session_id}\n\nObservations:\n${observations.join("\n")}`;
+            const response = await this.callOllama(model, systemPrompt, userPrompt);
+            if (!response)
+                throw new Error("Ollama summary failed");
+            // 2. Add as preference/long-term info to user profile
+            await this.addObservation({
+                entity_id: exports.USER_ENTITY_ID,
+                text: `Session Summary (${args.session_id}): ${response}`,
+                metadata: { kind: "session_summary", session_id: args.session_id, model },
+                deduplicate: true
+            });
+            return { status: "session_compacted", session_id: args.session_id, summary: response };
+        }
+        catch (error) {
+            console.error(`[CompactSession] Error:`, error);
+            return { error: error.message };
+        }
+    }
+    async compactEntity(args) {
+        if (this.compactionLocks.has(args.entity_id))
+            return { status: "already_compacting" };
+        this.compactionLocks.add(args.entity_id);
+        await this.initPromise;
+        const threshold = args.threshold || 20;
+        const model = args.model || "demyagent-4b-i1:Q6_K";
+        try {
+            // 1. Check count of active observations
+            const countRes = await this.db.run('?[count(oid)] := *observation{entity_id: $eid, id: oid, @ "NOW"}', { eid: args.entity_id });
+            const currentCount = Number(countRes.rows[0][0]);
+            if (currentCount <= threshold) {
+                return { entity_id: args.entity_id, status: "ok", count: currentCount };
+            }
+            console.error(`[CompactEntity] Compacting ${args.entity_id} (Count: ${currentCount})`);
+            // 2. Get older observations (limit to currentCount - threshold/2 to keep some recent context)
+            const toKeep = Math.floor(threshold / 2);
+            const toCompact = currentCount - toKeep;
+            const obsRes = await this.db.run(`
+        ?[oid, txt, ts] := *observation{entity_id: $eid, id: oid, text: txt, created_at, @ "NOW"}, ts = to_int(created_at)
+        :order ts
+        :limit $limit
+      `, { eid: args.entity_id, limit: toCompact });
+            if (obsRes.rows.length === 0)
+                return { entity_id: args.entity_id, status: "no_older_obs_found" };
+            const idsToInvalidate = obsRes.rows.map((r) => String(r[0]));
+            const textsToCompact = obsRes.rows.map((r) => r[1]);
+            // 3. Check for existing ExecutiveSummary
+            const existingSummaryRes = await this.db.run('?[sid, stxt] := *observation{entity_id: $eid, text: stxt, id: sid, @ "NOW"}, regex_matches(stxt, "(?i).*(Executive\\s*Summary|ExecutiveSummary).*") :limit 1', { eid: args.entity_id });
+            let summaryText = "";
+            let oldSummaryId = "";
+            if (existingSummaryRes.rows.length > 0) {
+                oldSummaryId = String(existingSummaryRes.rows[0][0]);
+                summaryText = String(existingSummaryRes.rows[0][1]).replace(/Executive\s*Summary:\s*/i, "").replace(/ExecutiveSummary:\s*/i, "");
+                console.error(`[CompactEntity] Found existing summary to merge: ${oldSummaryId}`);
+            }
+            // 4. Summarize / Merge
+            const systemPrompt = `You are a memory compaction module. You are maintaining an "Executive Summary" for an entity.
+Merge the NEW observations into the EXISTING summary (if present). 
+The summary should be dense, data-rich, and capture all relevant historical facts.
+Keep it structured and concise (max 5-8 sentences).
+Format MUST start with "ExecutiveSummary: " followed by the consolidated content.`;
+            const userPrompt = `Entity ID: ${args.entity_id}\nExisting Summary: ${summaryText || "None"}\n\nNew Info to integrate:\n${textsToCompact.join("\n")}`;
+            console.error(`[CompactEntity] Calling Ollama for ${idsToInvalidate.length} observations...`);
+            const newSummaryText = await this.callOllama(model, systemPrompt, userPrompt);
+            if (!newSummaryText)
+                throw new Error("Ollama compaction failed");
+            // 5. Atomic Update
+            // a) Invalidate old observations
+            for (const oid of idsToInvalidate) {
+                await this.invalidateObservation({ observation_id: oid });
+            }
+            // b) Invalidate old summary if exists
+            if (oldSummaryId) {
+                await this.invalidateObservation({ observation_id: oldSummaryId });
+            }
+            // c) Add new summary
+            await this.addObservation({
+                entity_id: args.entity_id,
+                text: newSummaryText,
+                metadata: { kind: "compaction", model, compacted_count: idsToInvalidate.length }
+            });
+            return { status: "entity_compacted", entity_id: args.entity_id, observations_compacted: idsToInvalidate.length };
+        }
+        catch (error) {
+            console.error(`[CompactEntity] Error for ${args.entity_id}:`, error);
+            return { error: error.message };
+        }
+        finally {
+            this.compactionLocks.delete(args.entity_id);
+        }
+    }
+    async callOllama(model, system, user) {
+        try {
+            const ollamaMod = await import("ollama");
+            const ollamaClient = ollamaMod?.default ?? ollamaMod;
+            const response = await ollamaClient.chat({
+                model,
+                messages: [
+                    { role: "system", content: system },
+                    { role: "user", content: user },
+                ],
+            });
+            return response?.message?.content?.trim?.() ?? null;
+        }
+        catch (e) {
+            console.error(`[OllamaCall] Error:`, e);
+            return null;
+        }
+    }
     async deleteEntity(args) {
         const startTime = Date.now();
         try {
@@ -2203,9 +2349,9 @@ ids[id] <- $ids
             }
             console.error(`[Delete] Entity found: ${entityRes.rows[0][0]}`);
             // 2. Count related data before deletion
-            const obsCount = await this.db.run('?[count(id)] := *observation{id, entity_id, @ "NOW"}, entity_id = $id', { id: args.entity_id });
-            const relOutCount = await this.db.run('?[count(from_id)] := *relationship{from_id, to_id, @ "NOW"}, from_id = $id', { id: args.entity_id });
-            const relInCount = await this.db.run('?[count(to_id)] := *relationship{from_id, to_id, @ "NOW"}, to_id = $id', { id: args.entity_id });
+            const obsCount = await this.db.run('?[count(oid)] := *observation{id: oid, entity_id: $id, @ "NOW"}', { id: args.entity_id });
+            const relOutCount = await this.db.run('?[count(fid)] := *relationship{from_id: fid, to_id, @ "NOW"}, from_id = $id', { id: args.entity_id });
+            const relInCount = await this.db.run('?[count(tid)] := *relationship{from_id, to_id: tid, @ "NOW"}, to_id = $id', { id: args.entity_id });
             console.error(`[Delete] Related data: ${obsCount.rows[0][0]} observations, ${relOutCount.rows[0][0]} outgoing relations, ${relInCount.rows[0][0]} incoming relations`);
             // 3. Delete all related data in a transaction (block)
             console.error(`[Delete] Executing deletion transaction...`);
@@ -2243,6 +2389,280 @@ ids[id] <- $ids
             return { error: "Deletion failed", message: error.message };
         }
     }
+    /**
+     * Memory Defragmentation
+     * Reorganizes memory structure by:
+     * 1. Detecting and merging duplicate/near-duplicate observations using LSH MinHash
+     * 2. Connecting fragmented knowledge islands (small connected components) to main graph
+     * 3. Removing orphaned entities (no observations or relations)
+     *
+     * Inspired by Letta MemFS defragmentation
+     */
+    async defragMemory(args) {
+        await this.initPromise;
+        const startTime = Date.now();
+        try {
+            console.error(`[Defrag] Starting memory defragmentation (confirm=${args.confirm})`);
+            const similarity_threshold = args.similarity_threshold || 0.95; // High threshold for near-duplicates
+            const min_island_size = args.min_island_size || 3; // Islands with <= 3 nodes
+            const stats = {
+                duplicates_found: 0,
+                duplicates_merged: 0,
+                islands_found: 0,
+                islands_connected: 0,
+                orphans_found: 0,
+                orphans_removed: 0,
+            };
+            // Step 1: Find duplicate observations using semantic similarity
+            console.error(`[Defrag] Step 1: Detecting duplicate observations (threshold=${similarity_threshold})`);
+            const allObs = await this.db.run(`
+        ?[id, entity_id, text, embedding] := *observation{id, entity_id, text, embedding, @ "NOW"}
+      `);
+            const duplicatePairs = [];
+            // Compare embeddings for similarity (using existing embeddings)
+            for (let i = 0; i < allObs.rows.length; i++) {
+                for (let j = i + 1; j < allObs.rows.length; j++) {
+                    const emb1 = allObs.rows[i][3];
+                    const emb2 = allObs.rows[j][3];
+                    if (!emb1 || !emb2 || emb1.length === 0 || emb2.length === 0)
+                        continue;
+                    // Cosine similarity
+                    const similarity = this.cosineSimilarity(emb1, emb2);
+                    if (similarity >= similarity_threshold) {
+                        duplicatePairs.push([
+                            String(allObs.rows[i][0]), // id1
+                            String(allObs.rows[j][0]), // id2
+                            similarity
+                        ]);
+                    }
+                }
+            }
+            stats.duplicates_found = duplicatePairs.length;
+            console.error(`[Defrag] Found ${stats.duplicates_found} duplicate observation pairs`);
+            // Step 2: Find fragmented knowledge islands (small connected components)
+            console.error(`[Defrag] Step 2: Detecting fragmented knowledge islands`);
+            const components = await this.recomputeConnectedComponents();
+            const smallIslands = Object.entries(components.components || {})
+                .filter(([_, entities]) => entities.length > 0 && entities.length <= min_island_size)
+                .map(([compId, entities]) => ({ compId, entities: entities, size: entities.length }));
+            stats.islands_found = smallIslands.length;
+            console.error(`[Defrag] Found ${stats.islands_found} small knowledge islands (size <= ${min_island_size})`);
+            // Step 3: Find orphaned entities (no observations, no relations)
+            console.error(`[Defrag] Step 3: Detecting orphaned entities`);
+            // Simplified approach: Get all entities, then filter in JavaScript
+            const allEntities = await this.db.run(`
+        ?[id, name, type] := *entity{id, name, type, @ "NOW"}, id != "global_user_profile"
+      `);
+            const orphanedEntities = { rows: [] };
+            for (const row of allEntities.rows) {
+                const entityId = String(row[0]);
+                // Check if entity has observations
+                const obsCheck = await this.db.run(`
+          ?[count(id)] := *observation{id, entity_id, @ "NOW"}, entity_id = $eid
+        `, { eid: entityId });
+                const hasObs = Number(obsCheck.rows[0]?.[0] || 0) > 0;
+                if (hasObs)
+                    continue;
+                // Check if entity has relationships
+                const relCheck = await this.db.run(`
+          out[count(from_id)] := *relationship{from_id, @ "NOW"}, from_id = $eid
+          in[count(to_id)] := *relationship{to_id, @ "NOW"}, to_id = $eid
+          ?[total] := out[out_count], in[in_count], total = out_count + in_count
+        `, { eid: entityId });
+                const hasRel = Number(relCheck.rows[0]?.[0] || 0) > 0;
+                if (hasRel)
+                    continue;
+                // This entity is orphaned
+                orphanedEntities.rows.push(row);
+            }
+            stats.orphans_found = orphanedEntities.rows.length;
+            console.error(`[Defrag] Found ${stats.orphans_found} orphaned entities`);
+            // If not confirmed, return dry-run results
+            if (!args.confirm) {
+                return {
+                    status: "dry_run",
+                    message: "Defragmentation analysis complete. Set confirm=true to execute.",
+                    statistics: stats,
+                    preview: {
+                        duplicate_samples: duplicatePairs.slice(0, 5).map(([id1, id2, sim]) => ({
+                            observation_id_1: id1,
+                            observation_id_2: id2,
+                            similarity: sim
+                        })),
+                        island_samples: smallIslands.slice(0, 5),
+                        orphan_samples: orphanedEntities.rows.slice(0, 5).map((r) => ({
+                            id: String(r[0]),
+                            name: String(r[1]),
+                            type: String(r[2])
+                        }))
+                    }
+                };
+            }
+            // Execute defragmentation
+            console.error(`[Defrag] Executing defragmentation...`);
+            // Merge duplicates: Keep first, delete second
+            for (const [id1, id2, similarity] of duplicatePairs) {
+                try {
+                    // Delete the duplicate observation
+                    await this.db.run(`
+            ?[id, created_at] := *observation{id, created_at}, id = $id2
+            :rm observation {id, created_at}
+          `, { id2 });
+                    stats.duplicates_merged++;
+                }
+                catch (e) {
+                    console.error(`[Defrag] Failed to merge duplicate ${id2}:`, e.message);
+                }
+            }
+            // Connect islands: Create semantic relations to main graph
+            for (const island of smallIslands) {
+                try {
+                    // Find the most central entity in the island
+                    const islandEntityIds = island.entities.map((e) => e.id);
+                    // Find semantically similar entities in the main graph (not in this island)
+                    for (const entityId of islandEntityIds) {
+                        const entityData = await this.db.run(`
+              ?[embedding] := *entity{id: $id, embedding, @ "NOW"}
+            `, { id: entityId });
+                        if (entityData.rows.length === 0)
+                            continue;
+                        const embedding = entityData.rows[0][0];
+                        if (!embedding || embedding.length === 0)
+                            continue;
+                        // Find similar entity in main graph
+                        const similarEntities = await this.db.run(`
+              ~entity:semantic { id: target_id | query: $emb, k: 1, bind_distance: dist }
+              ?[target_id, dist] := ~entity:semantic { id: target_id | query: $emb, k: 1, bind_distance: dist },
+                                     target_id != $id
+            `, { emb: embedding, id: entityId });
+                        if (similarEntities.rows.length > 0) {
+                            const targetId = String(similarEntities.rows[0][0]);
+                            const distance = Number(similarEntities.rows[0][1]);
+                            const similarity = 1 - distance;
+                            // Create bridge relation
+                            if (similarity >= 0.7) {
+                                await this.createRelation({
+                                    from_id: entityId,
+                                    to_id: targetId,
+                                    relation_type: "semantically_related",
+                                    strength: similarity,
+                                    metadata: { created_by: "defrag", reason: "island_connection" }
+                                });
+                                stats.islands_connected++;
+                                break; // One connection per island is enough
+                            }
+                        }
+                    }
+                }
+                catch (e) {
+                    console.error(`[Defrag] Failed to connect island ${island.compId}:`, e.message);
+                }
+            }
+            // Remove orphaned entities
+            for (const row of orphanedEntities.rows) {
+                try {
+                    const entityId = String(row[0]);
+                    await this.deleteEntity({ entity_id: entityId });
+                    stats.orphans_removed++;
+                }
+                catch (e) {
+                    console.error(`[Defrag] Failed to remove orphan ${row[0]}:`, e.message);
+                }
+            }
+            this.trackOperation('defrag', startTime);
+            return {
+                status: "completed",
+                message: `Defragmentation complete: ${stats.duplicates_merged} duplicates merged, ${stats.islands_connected} islands connected, ${stats.orphans_removed} orphans removed`,
+                statistics: stats,
+                duration_ms: Date.now() - startTime
+            };
+        }
+        catch (error) {
+            console.error("[Defrag] Error during defragmentation:", error);
+            this.trackError('defrag');
+            return { error: "Defragmentation failed", message: error.message };
+        }
+    }
+    cosineSimilarity(a, b) {
+        if (a.length !== b.length)
+            return 0;
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dotProduct += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+        if (normA === 0 || normB === 0)
+            return 0;
+        return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+    }
+    async invalidateObservation(args) {
+        await this.initPromise;
+        const startTime = Date.now();
+        try {
+            const now = Date.now() * 1000;
+            // Invalidate by putting a retraction (assertive=false) at current time
+            const existing = await this.db.run(`?[oid, eid, sid, tid, txt, emb, meta] := *observation{id: oid, entity_id: eid, session_id: sid, task_id: tid, text: txt, embedding: emb, metadata: meta, @ "NOW"}, oid = $id`, { id: args.observation_id });
+            if (existing.rows.length === 0) {
+                return { error: "Observation not found or already invalidated" };
+            }
+            const row = existing.rows[0];
+            const params = {
+                id: row[0],
+                v: [now, false],
+                entity_id: row[1],
+                session_id: row[2],
+                task_id: row[3],
+                text: row[4],
+                embedding: row[5],
+                metadata: row[6]
+            };
+            await this.db.run(`
+        ?[id, created_at, entity_id, session_id, task_id, text, embedding, metadata] <- [[$id, $v, $entity_id, $session_id, $task_id, $text, $embedding, $metadata]]
+        :put observation {id, created_at => entity_id, session_id, task_id, text, embedding, metadata}
+      `, params);
+            this.trackOperation('invalidate_observation', startTime);
+            return { status: "Observation invalidated", id: args.observation_id, invalidated_at: now };
+        }
+        catch (error) {
+            console.error("[InvalidateObs] Error:", error);
+            this.trackError('invalidate_observation');
+            return { error: "Invalidation failed", message: error.message };
+        }
+    }
+    async invalidateRelationship(args) {
+        await this.initPromise;
+        const startTime = Date.now();
+        try {
+            const now = Date.now() * 1000;
+            const existing = await this.db.run(`?[f, t, type, strength, metadata] := *relationship{from_id: f, to_id: t, relation_type: type, strength, metadata, @ "NOW"}, f = $f, t = $t, type = $type`, { f: args.from_id, t: args.to_id, type: args.relation_type });
+            if (existing.rows.length === 0) {
+                return { error: "Relationship not found or already invalidated" };
+            }
+            const row = existing.rows[0];
+            const params = {
+                f: row[0],
+                t: row[1],
+                type: row[2],
+                v: [now, false],
+                strength: row[3],
+                metadata: row[4]
+            };
+            await this.db.run(`
+        ?[from_id, to_id, relation_type, created_at, strength, metadata] <- [[$f, $t, $type, $v, $strength, $metadata]]
+        :put relationship {from_id, to_id, relation_type, created_at => strength, metadata}
+      `, params);
+            this.trackOperation('invalidate_relationship', startTime);
+            return { status: "Relationship invalidated", from_id: args.from_id, to_id: args.to_id, relation_type: args.relation_type, invalidated_at: now };
+        }
+        catch (error) {
+            console.error("[InvalidateRel] Error:", error);
+            this.trackError('invalidate_relationship');
+            return { error: "Invalidation failed", message: error.message };
+        }
+    }
     async runTransaction(args) {
         await this.initPromise;
         try {
@@ -2444,6 +2864,46 @@ ids[id] <- $ids
                         results.push({ action: "delete_entity", id: entity_id });
                         break;
                     }
+                    case "invalidate_observation": {
+                        const { observation_id } = params;
+                        if (!observation_id) {
+                            return { error: `Missing observation_id for invalidate_observation in operation ${i}` };
+                        }
+                        const now = Date.now() * 1000;
+                        allParams[`inv_obs_id${suffix}`] = observation_id;
+                        allParams[`inv_obs_v${suffix}`] = [now, false];
+                        statements.push(`
+              {
+                ?[oid, cat, eid, sid, tid, txt, emb, meta] := 
+                  *observation{id: oid, entity_id: eid, session_id: sid, task_id: tid, text: txt, embedding: emb, metadata: meta, @ "NOW"}, 
+                  oid = $inv_obs_id${suffix}, cat = $inv_obs_v${suffix}
+                :put observation {id: oid, created_at: cat => entity_id: eid, session_id: sid, task_id: tid, text: txt, embedding: emb, metadata: meta}
+              }
+            `);
+                        results.push({ action: "invalidate_observation", id: observation_id });
+                        break;
+                    }
+                    case "invalidate_relation": {
+                        const { from_id, to_id, relation_type } = params;
+                        if (!from_id || !to_id || !relation_type) {
+                            return { error: `Missing from_id, to_id, or relation_type for invalidate_relation in operation ${i}` };
+                        }
+                        const now = Date.now() * 1000;
+                        allParams[`inv_rel_f${suffix}`] = from_id;
+                        allParams[`inv_rel_t${suffix}`] = to_id;
+                        allParams[`inv_rel_type${suffix}`] = relation_type;
+                        allParams[`inv_rel_v${suffix}`] = [now, false];
+                        statements.push(`
+              {
+                ?[f, t, type, cat, str, meta] := 
+                  *relationship{from_id: f, to_id: t, relation_type: type, strength: str, metadata: meta, @ "NOW"}, 
+                  f = $inv_rel_f${suffix}, t = $inv_rel_t${suffix}, type = $inv_rel_type${suffix}, cat = $inv_rel_v${suffix}
+                :put relationship {from_id: f, to_id: t, relation_type: type, created_at: cat => strength: str, metadata: meta}
+              }
+            `);
+                        results.push({ action: "invalidate_relation", from_id, to_id, relation_type });
+                        break;
+                    }
                     default:
                         return { error: `Unknown operation: ${op.action}` };
                 }
@@ -2623,6 +3083,16 @@ ids[id] <- $ids
                 action: zod_1.z.literal("delete_entity"),
                 entity_id: zod_1.z.string().describe("ID of the entity to delete"),
             }).passthrough(),
+            zod_1.z.object({
+                action: zod_1.z.literal("invalidate_observation"),
+                observation_id: zod_1.z.string().describe("ID of the observation to invalidate"),
+            }).passthrough(),
+            zod_1.z.object({
+                action: zod_1.z.literal("invalidate_relation"),
+                from_id: zod_1.z.string().describe("Source entity ID"),
+                to_id: zod_1.z.string().describe("Target entity ID"),
+                relation_type: zod_1.z.string().describe("Type of the relationship"),
+            }).passthrough(),
             zod_1.z.object({
                 action: zod_1.z.literal("add_observation"),
                 entity_id: zod_1.z.string().optional().describe("ID of the entity"),
@@ -2746,7 +3216,7 @@ ids[id] <- $ids
         ]);
         const MutateMemoryParameters = zod_1.z.object({
             action: zod_1.z
-                .enum(["create_entity", "update_entity", "delete_entity", "add_observation", "create_relation", "run_transaction", "add_inference_rule", "ingest_file", "start_session", "stop_session", "start_task", "stop_task"])
+                .enum(["create_entity", "update_entity", "delete_entity", "add_observation", "create_relation", "run_transaction", "add_inference_rule", "ingest_file", "start_session", "stop_session", "start_task", "stop_task", "invalidate_observation", "invalidate_relation"])
                 .describe("Action (determines which fields are required)"),
             name: zod_1.z.string().optional().describe("For create_entity (required) or add_inference_rule (required)"),
             type: zod_1.z.string().optional().describe("For create_entity (required)"),
@@ -2768,6 +3238,7 @@ ids[id] <- $ids
             relation_type: zod_1.z.string().optional().describe("For create_relation (required)"),
             strength: zod_1.z.number().min(0).max(1).optional().describe("Optional for create_relation"),
             metadata: MetadataSchema.optional().describe("Optional for create_entity/update_entity/add_observation/create_relation/ingest_file"),
+            observation_id: zod_1.z.string().optional().describe("For invalidate_observation (required)"),
             operations: zod_1.z.array(zod_1.z.object({
                 action: zod_1.z.enum(["create_entity", "add_observation", "create_relation", "delete_entity"]),
                 params: zod_1.z.any().describe("Parameters for the operation as an object")
@@ -2798,6 +3269,8 @@ Supported actions:
 - 'stop_session': Closes a session. Params: { id: string }.
 - 'start_task': Initializes a new task within a session. Params: { name: string, session_id?: string, metadata?: object }.
 - 'stop_task': Marks a task as completed. Params: { id: string }.
+- 'invalidate_observation': Invalidates (soft-deletes) an observation at the current time. Params: { observation_id: string }.
+- 'invalidate_relation': Invalidates (soft-deletes) a relationship at the current time. Params: { from_id: string, to_id: string, relation_type: string }.
 
 Validation: Invalid syntax or missing columns in inference rules will result in errors.`,
             parameters: MutateMemoryParameters,
@@ -2825,6 +3298,10 @@ Validation: Invalid syntax or missing columns in inference rules will result in
                     return JSON.stringify(await this.createRelation(rest));
                 if (action === "run_transaction")
                     return JSON.stringify(await this.runTransaction(rest));
+                if (action === "invalidate_observation")
+                    return JSON.stringify(await this.invalidateObservation(rest));
+                if (action === "invalidate_relation")
+                    return JSON.stringify(await this.invalidateRelationship(rest));
                 if (action === "delete_entity")
                     return JSON.stringify(await this.deleteEntity({ entity_id: rest.entity_id }));
                 if (action === "add_inference_rule")
@@ -2921,13 +3398,56 @@ Validation: Invalid syntax or missing columns in inference rules will result in
                 session_id: zod_1.z.string().optional().describe("Prioritize results from this session"),
                 task_id: zod_1.z.string().optional().describe("Prioritize results from this task"),
             }),
+            zod_1.z.object({
+                action: zod_1.z.literal("dynamic_fusion"),
+                query: zod_1.z.string().describe("Search query"),
+                config: zod_1.z.object({
+                    vector: zod_1.z.object({
+                        enabled: zod_1.z.boolean().optional().default(true),
+                        weight: zod_1.z.number().min(0).max(1).optional().default(0.4),
+                        topK: zod_1.z.number().optional().default(20),
+                        efSearch: zod_1.z.number().optional().default(100),
+                    }).optional(),
+                    sparse: zod_1.z.object({
+                        enabled: zod_1.z.boolean().optional().default(true),
+                        weight: zod_1.z.number().min(0).max(1).optional().default(0.3),
+                        topK: zod_1.z.number().optional().default(20),
+                        minScore: zod_1.z.number().optional().default(0.1),
+                    }).optional(),
+                    fts: zod_1.z.object({
+                        enabled: zod_1.z.boolean().optional().default(true),
+                        weight: zod_1.z.number().min(0).max(1).optional().default(0.2),
+                        topK: zod_1.z.number().optional().default(20),
+                        fuzzy: zod_1.z.boolean().optional().default(true),
+                    }).optional(),
+                    graph: zod_1.z.object({
+                        enabled: zod_1.z.boolean().optional().default(true),
+                        weight: zod_1.z.number().min(0).max(1).optional().default(0.1),
+                        maxDepth: zod_1.z.number().min(1).max(3).optional().default(2),
+                        maxResults: zod_1.z.number().optional().default(20),
+                        relationTypes: zod_1.z.array(zod_1.z.string()).optional(),
+                    }).optional(),
+                    fusion: zod_1.z.object({
+                        strategy: zod_1.z.enum(['rrf', 'weighted_sum', 'max', 'adaptive']).optional().default('rrf'),
+                        rrfK: zod_1.z.number().optional().default(60),
+                        minScore: zod_1.z.number().optional().default(0.0),
+                        deduplication: zod_1.z.boolean().optional().default(true),
+                    }).optional(),
+                }).optional().describe("Dynamic fusion configuration (all paths optional)"),
+                limit: zod_1.z.number().optional().default(10).describe("Maximum number of results to return"),
+            }),
+            zod_1.z.object({
+                action: zod_1.z.literal("adaptive_retrieval"),
+                query: zod_1.z.string().describe("Search query for adaptive retrieval"),
+                limit: zod_1.z.number().optional().default(10).describe("Maximum number of results"),
+            }),
         ]);
         const QueryMemoryParameters = zod_1.z.object({
             action: zod_1.z
-                .enum(["search", "advancedSearch", "context", "entity_details", "history", "graph_rag", "graph_walking", "agentic_search"])
+                .enum(["search", "advancedSearch", "context", "entity_details", "history", "graph_rag", "graph_walking", "agentic_search", "dynamic_fusion", "adaptive_retrieval"])
                 .describe("Action (determines which fields are required)"),
-            query: zod_1.z.string().optional().describe("Required for search/advancedSearch/context/graph_rag/graph_walking/agentic_search"),
-            limit: zod_1.z.number().optional().describe("Only for search/advancedSearch/graph_rag/graph_walking"),
+            query: zod_1.z.string().optional().describe("Required for search/advancedSearch/context/graph_rag/graph_walking/agentic_search/dynamic_fusion/adaptive_retrieval"),
+            limit: zod_1.z.number().optional().describe("Only for search/advancedSearch/graph_rag/graph_walking/dynamic_fusion/adaptive_retrieval"),
             session_id: zod_1.z.string().optional().describe("Optional session ID for context boosting"),
             task_id: zod_1.z.string().optional().describe("Optional task ID for context boosting"),
             filters: zod_1.z.any().optional().describe("Only for advancedSearch"),
@@ -2943,6 +3463,7 @@ Validation: Invalid syntax or missing columns in inference rules will result in
             max_depth: zod_1.z.number().optional().describe("Only for graph_rag/graph_walking: Maximum expansion depth"),
             start_entity_id: zod_1.z.string().optional().describe("Only for graph_walking: Start entity"),
             rerank: zod_1.z.boolean().optional().describe("Only for search/advancedSearch/agentic_search: Enable Cross-Encoder reranking"),
+            config: zod_1.z.any().optional().describe("Only for dynamic_fusion: Fusion configuration object"),
         });
         this.mcp.addTool({
             name: "query_memory",
@@ -2958,8 +3479,10 @@ Supported actions:
 - 'graph_rag': Graph-based reasoning (Hybrid RAG). Finds semantic vector seeds first, then expands via graph traversals. Ideal for multi-hop reasoning. Params: { query: string, max_depth?: number, limit?: number }.
 - 'graph_walking': Recursive semantic graph search. Starts at vector seeds or an entity and follows relationships to other semantically relevant entities. Params: { query: string, start_entity_id?: string, max_depth?: number, limit?: number }.
 - 'agentic_search': Auto-Routing Search. Uses local LLM to analyze intent and routes the query automatically to the best strategy (Vector, Graph, or Community Summaries). Params: { query: string, limit?: number }.
+- 'adaptive_retrieval': GraphRAG-R1 inspired adaptive retrieval with Progressive Retrieval Attenuation (PRA) and Cost-Aware F1 (CAF) scoring. Automatically selects optimal strategy based on query complexity and historical performance. Params: { query: string, limit?: number }.
+- 'dynamic_fusion': Advanced multi-path fusion search combining Vector (HNSW), Sparse (keyword), FTS (full-text), and Graph traversal with configurable weights and strategies. Params: { query: string, config?: { vector?, sparse?, fts?, graph?, fusion? }, limit?: number }. Each path can be enabled/disabled and weighted independently. Fusion strategies: 'rrf' (Reciprocal Rank Fusion), 'weighted_sum', 'max', 'adaptive'. Returns results with path contribution details and performance stats.
 
-Notes: 'agentic_search' is the most powerful and adaptable, 'context' is ideal for exploratory questions. 'search' and 'advancedSearch' are better for targeted fact retrieval.`,
+Notes: 'adaptive_retrieval' learns from usage and optimizes over time. 'dynamic_fusion' provides the most control and transparency over retrieval paths. 'agentic_search' is the most adaptive. 'context' is ideal for exploratory questions. 'search' and 'advancedSearch' are better for targeted fact retrieval.`,
             parameters: QueryMemoryParameters,
             execute: async (args) => {
                 await this.initPromise;
@@ -3130,6 +3653,101 @@ Notes: 'agentic_search' is the most powerful and adaptable, 'context' is ideal f
                     });
                     return JSON.stringify(results);
                 }
+                if (input.action === "dynamic_fusion") {
+                    const startTime = Date.now();
+                    if (!input.query || input.query.trim().length === 0) {
+                        return JSON.stringify({ error: "Search query must not be empty." });
+                    }
+                    try {
+                        console.log('[query_memory] Dynamic Fusion search:', {
+                            query: input.query,
+                            config: input.config ? 'custom' : 'default',
+                            limit: input.limit
+                        });
+                        // Execute dynamic fusion search
+                        const { results, stats } = await this.dynamicFusion.search(input.query, input.config || {});
+                        // Apply limit
+                        const limitedResults = results.slice(0, input.limit || 10);
+                        const response = {
+                            results: limitedResults,
+                            stats: {
+                                ...stats,
+                                totalTime: Date.now() - startTime,
+                                resultsReturned: limitedResults.length,
+                                resultsTotal: results.length
+                            },
+                            metadata: {
+                                query: input.query,
+                                fusionStrategy: input.config?.fusion?.strategy || 'rrf',
+                                enabledPaths: {
+                                    vector: input.config?.vector?.enabled !== false,
+                                    sparse: input.config?.sparse?.enabled !== false,
+                                    fts: input.config?.fts?.enabled !== false,
+                                    graph: input.config?.graph?.enabled !== false
+                                }
+                            }
+                        };
+                        console.log('[query_memory] Dynamic Fusion completed:', {
+                            resultsReturned: limitedResults.length,
+                            totalTime: response.stats.totalTime,
+                            pathContributions: stats.pathContributions
+                        });
+                        return JSON.stringify(response);
+                    }
+                    catch (error) {
+                        console.error('[query_memory] Dynamic Fusion error:', error);
+                        return JSON.stringify({
+                            error: "Dynamic fusion search failed",
+                            details: error.message
+                        });
+                    }
+                }
+                if (input.action === "adaptive_retrieval") {
+                    const startTime = Date.now();
+                    if (!input.query || input.query.trim().length === 0) {
+                        return JSON.stringify({ error: "Search query must not be empty." });
+                    }
+                    try {
+                        console.log('[query_memory] Adaptive Retrieval search:', {
+                            query: input.query,
+                            limit: input.limit
+                        });
+                        // Execute adaptive retrieval
+                        const result = await this.adaptiveRetrieval.retrieve(input.query, input.limit || 10);
+                        const response = {
+                            results: result.results,
+                            metadata: {
+                                query: input.query,
+                                strategy: result.strategy,
+                                retrievalCount: result.retrievalCount,
+                                latency: result.latency,
+                                cafScore: result.cafScore,
+                                totalTime: Date.now() - startTime
+                            },
+                            performance: {
+                                strategyUsed: result.strategy,
+                                retrievalCalls: result.retrievalCount,
+                                costAwareF1: result.cafScore,
+                                latencyMs: result.latency
+                            }
+                        };
+                        console.log('[query_memory] Adaptive Retrieval completed:', {
+                            strategy: result.strategy,
+                            resultsReturned: result.results.length,
+                            retrievalCount: result.retrievalCount,
+                            cafScore: result.cafScore?.toFixed(3),
+                            totalTime: response.metadata.totalTime
+                        });
+                        return JSON.stringify(response);
+                    }
+                    catch (error) {
+                        console.error('[query_memory] Adaptive Retrieval error:', error);
+                        return JSON.stringify({
+                            error: "Adaptive retrieval search failed",
+                            details: error.message
+                        });
+                    }
+                }
                 if (input.action === "graph_rag") {
                     if (!input.query || input.query.trim().length === 0) {
                         return JSON.stringify({ error: "Search query must not be empty." });
@@ -3603,6 +4221,12 @@ Supported actions:
                 min_entity_degree: zod_1.z.number().min(0).max(100).optional().default(2),
                 model: zod_1.z.string().optional().default("demyagent-4b-i1:Q6_K"),
             }),
+            zod_1.z.object({
+                action: zod_1.z.literal("defrag"),
+                confirm: zod_1.z.boolean().describe("Must be true to confirm defragmentation"),
+                similarity_threshold: zod_1.z.number().min(0.8).max(1.0).optional().default(0.95).describe("Similarity threshold for duplicate detection (0.8-1.0)"),
+                min_island_size: zod_1.z.number().min(1).max(10).optional().default(3).describe("Maximum size of knowledge islands to connect"),
+            }),
             zod_1.z.object({
                 action: zod_1.z.literal("reflect"),
                 entity_id: zod_1.z.string().optional().describe("Optional entity ID for targeted reflection"),
@@ -3618,10 +4242,16 @@ Supported actions:
                 model: zod_1.z.string().optional().default("demyagent-4b-i1:Q6_K"),
                 min_community_size: zod_1.z.number().min(2).max(100).optional().default(3),
             }),
+            zod_1.z.object({
+                action: zod_1.z.literal("compact"),
+                session_id: zod_1.z.string().optional().describe("Session ID to compact"),
+                entity_id: zod_1.z.string().optional().describe("Entity ID to compact"),
+                model: zod_1.z.string().optional().default("demyagent-4b-i1:Q6_K"),
+            }),
         ]);
         const ManageSystemParameters = zod_1.z.object({
             action: zod_1.z
-                .enum(["health", "metrics", "export_memory", "import_memory", "snapshot_create", "snapshot_list", "snapshot_diff", "cleanup", "reflect", "clear_memory", "summarize_communities"])
+                .enum(["health", "metrics", "export_memory", "import_memory", "snapshot_create", "snapshot_list", "snapshot_diff", "cleanup", "defrag", "reflect", "clear_memory", "summarize_communities", "compact"])
                 .describe("Action (determines which fields are required)"),
             format: zod_1.z.enum(["json", "markdown", "obsidian"]).optional().describe("Export format (for export_memory)"),
             includeMetadata: zod_1.z.boolean().optional().describe("Include metadata (for export_memory)"),
@@ -3636,10 +4266,12 @@ Supported actions:
             snapshot_id_a: zod_1.z.string().optional().describe("Required for snapshot_diff"),
             snapshot_id_b: zod_1.z.string().optional().describe("Required for snapshot_diff"),
             metadata: MetadataSchema.optional().describe("Optional for snapshot_create"),
-            confirm: zod_1.z.boolean().optional().describe("Required for cleanup/clear_memory and must be true"),
+            confirm: zod_1.z.boolean().optional().describe("Required for cleanup/defrag/clear_memory and must be true"),
             older_than_days: zod_1.z.number().optional().describe("Optional for cleanup"),
             max_observations: zod_1.z.number().optional().describe("Optional for cleanup"),
             min_entity_degree: zod_1.z.number().optional().describe("Optional for cleanup"),
+            similarity_threshold: zod_1.z.number().optional().describe("Optional for defrag (0.8-1.0, default 0.95)"),
+            min_island_size: zod_1.z.number().optional().describe("Optional for defrag (1-10, default 3)"),
             model: zod_1.z.string().optional().describe("Optional for cleanup/reflect/summarize_communities"),
             entity_id: zod_1.z.string().optional().describe("Optional for reflect"),
             min_community_size: zod_1.z.number().optional().describe("Optional for summarize_communities"),
@@ -3666,6 +4298,9 @@ Supported actions:
 - 'cleanup': Janitor service for consolidation. Params: { confirm: boolean, older_than_days?: number, max_observations?: number, min_entity_degree?: number, model?: string }.
   * With confirm=false: Dry-Run (shows candidates).
   * With confirm=true: Merges old/isolated fragments using LLM (Executive Summary) and removes noise.
+- 'defrag': Memory defragmentation. Reorganizes memory structure by detecting/merging duplicates, connecting fragmented knowledge islands, and removing orphaned entities. Params: { confirm: boolean, similarity_threshold?: number (0.8-1.0, default 0.95), min_island_size?: number (1-10, default 3) }.
+  * With confirm=false: Dry-Run (shows analysis and candidates).
+  * With confirm=true: Executes defragmentation and returns statistics.
 - 'reflect': Reflection service. Analyzes memory for contradictions and insights. Params: { entity_id?: string, model?: string }.
 - 'clear_memory': Resets the entire database. Params: { confirm: boolean (must be true) }.
 - 'summarize_communities': Hierarchical GraphRAG. Generates summaries for entity clusters. Params: { model?: string, min_community_size?: number }.`,
@@ -3764,9 +4399,9 @@ Supported actions:
                 if (input.action === "snapshot_create") {
                     try {
                         // Optimization: Sequential execution and count aggregation instead of full fetch
-                        const entityResult = await this.db.run('?[count(id)] := *entity{id, @ "NOW"}');
-                        const obsResult = await this.db.run('?[count(id)] := *observation{id, @ "NOW"}');
-                        const relResult = await this.db.run('?[count(from_id)] := *relationship{from_id, to_id, @ "NOW"}');
+                        const entityResult = await this.db.run('?[count(eid)] := *entity{id: eid, @ "NOW"}');
+                        const obsResult = await this.db.run('?[count(oid)] := *observation{id: oid, @ "NOW"}');
+                        const relResult = await this.db.run('?[count(fid)] := *relationship{from_id: fid, to_id, @ "NOW"}');
                         const snapshot_id = (0, uuid_1.v4)();
                         const counts = {
                             entities: Number(entityResult.rows[0]?.[0] || 0),
@@ -3856,6 +4491,19 @@ Supported actions:
                         return JSON.stringify({ error: error.message || "Error during cleanup" });
                     }
                 }
+                if (input.action === "defrag") {
+                    try {
+                        const result = await this.defragMemory({
+                            confirm: Boolean(input.confirm),
+                            similarity_threshold: input.similarity_threshold,
+                            min_island_size: input.min_island_size,
+                        });
+                        return JSON.stringify(result);
+                    }
+                    catch (error) {
+                        return JSON.stringify({ error: error.message || "Error during defragmentation" });
+                    }
+                }
                 if (input.action === "reflect") {
                     try {
                         const result = await this.reflectMemory({
@@ -3897,6 +4545,34 @@ Supported actions:
                         return JSON.stringify({ error: error.message || "Error clearing memory" });
                     }
                 }
+                if (input.action === "compact") {
+                    try {
+                        if (input.session_id) {
+                            return JSON.stringify(await this.compactSession({ session_id: input.session_id, model: input.model }));
+                        }
+                        else if (input.entity_id) {
+                            return JSON.stringify(await this.compactEntity({ entity_id: input.entity_id, model: input.model }));
+                        }
+                        else {
+                            // Compact all entities that exceed threshold
+                            const res = await this.db.run(`
+                ?[eid, count(oid)] := *observation{entity_id: eid, id: oid, @ "NOW"}
+              `);
+                            const threshold = 20;
+                            const entitiesToCompact = res.rows
+                                .filter((r) => Number(r[1]) > threshold)
+                                .map((r) => String(r[0]));
+                            const results = [];
+                            for (const eid of entitiesToCompact) {
+                                results.push(await this.compactEntity({ entity_id: eid, model: input.model }));
+                            }
+                            return JSON.stringify({ status: "global_compaction_completed", entities_processed: results.length, results });
+                        }
+                    }
+                    catch (error) {
+                        return JSON.stringify({ error: error.message || "Error during compaction" });
+                    }
+                }
                 return JSON.stringify({ error: "Unknown action" });
             },
         });