@gethmy/mcp 2.0.0 → 2.1.1

package/dist/lib/consolidation.js

@@ -0,0 +1,243 @@
+/**
+ * Smart Memory Consolidation
+ *
+ * Clusters similar draft/episode memories and merges them into
+ * consolidated reference entities to reduce noise and improve retrieval.
+ */
+import { findSimilarEntities } from "./graph-expansion.js";
+/**
+ * Consolidate similar draft/episode memories into reference entities.
+ *
+ * 1. Lists all draft and episode tier entities in scope
+ * 2. Groups by entity type
+ * 3. For each type group, finds clusters via embedding similarity
+ * 4. Merges clusters into new reference entities with part_of relations
+ */
+export async function consolidateMemories(client, workspaceId, projectId, options) {
+    const dryRun = options?.dryRun !== false; // default true
+    const minClusterSize = options?.minClusterSize ?? 2;
+    const result = {
+        consolidated: 0,
+        clustersFound: 0,
+        entitiesProcessed: 0,
+        details: [],
+    };
+    // Step 1: Fetch all draft and episode entities
+    const listResult = await client.listMemoryEntities({
+        workspace_id: workspaceId,
+        project_id: projectId,
+        limit: 100,
+    });
+    const allEntities = (listResult.entities || []).filter((e) => e.memory_tier === "draft" || e.memory_tier === "episode");
+    result.entitiesProcessed = allEntities.length;
+    if (allEntities.length < minClusterSize)
+        return result;
+    // Step 2: Group by type
+    const typeGroups = new Map();
+    for (const entity of allEntities) {
+        const group = typeGroups.get(entity.type) || [];
+        group.push(entity);
+        typeGroups.set(entity.type, group);
+    }
+    // Step 3: Find clusters within each type group
+    for (const [type, entities] of typeGroups) {
+        if (entities.length < minClusterSize)
+            continue;
+        const clustered = new Set();
+        const clusters = [];
+        for (const entity of entities) {
+            if (clustered.has(entity.id))
+                continue;
+            // Search for similar entities using embedding-based search
+            const similar = await findSimilarEntities(client, entity.title, entity.content, workspaceId, {
+                projectId,
+                limit: 20,
+                minRrfScore: 0.01,
+                excludeIds: [...clustered],
+            });
+            // Filter to only entities in our current type group that aren't yet clustered
+            const entityIdSet = new Set(entities.map((e) => e.id));
+            const clusterMembers = similar.filter((s) => entityIdSet.has(s.id) &&
+                !clustered.has(s.id) &&
+                s.id !== entity.id &&
+                s.type === type);
+            if (clusterMembers.length >= minClusterSize - 1) {
+                const cluster = [
+                    entity,
+                    ...clusterMembers.slice(0, 5).map((s) => {
+                        // Map back to full entity from our list
+                        return entities.find((e) => e.id === s.id) || entity;
+                    }),
+                ];
+                // Deduplicate by id
+                const uniqueCluster = [];
+                const seen = new Set();
+                for (const member of cluster) {
+                    if (!seen.has(member.id)) {
+                        seen.add(member.id);
+                        uniqueCluster.push(member);
+                    }
+                }
+                if (uniqueCluster.length >= minClusterSize) {
+                    clusters.push(uniqueCluster);
+                    for (const member of uniqueCluster) {
+                        clustered.add(member.id);
+                    }
+                }
+            }
+        }
+        // Step 4: Create consolidated entities for each cluster
+        for (const cluster of clusters) {
+            result.clustersFound++;
+            // Derive title from most common words across cluster titles
+            const mergedTitle = deriveClusterTitle(cluster, type);
+            const memberTitles = cluster.map((e) => e.title);
+            // Merge content as bullet points
+            const mergedContent = [
+                `Consolidated from ${cluster.length} ${type} memories:\n`,
+                ...cluster.map((e) => `- **${e.title}**: ${e.content.slice(0, 200)}`),
+            ].join("\n");
+            // Max confidence from cluster members
+            const maxConfidence = Math.max(...cluster.map((e) => e.confidence));
+            // Union of all tags (deduped)
+            const allTags = [...new Set(cluster.flatMap((e) => e.tags || []))];
+            const detail = {
+                clusterSize: cluster.length,
+                mergedTitle,
+                memberTitles,
+            };
+            if (!dryRun) {
+                try {
+                    // Create consolidated reference entity
+                    const createResult = await client.createMemoryEntity({
+                        workspace_id: workspaceId,
+                        project_id: projectId,
+                        type,
+                        scope: "project",
+                        memory_tier: "reference",
+                        title: mergedTitle,
+                        content: mergedContent,
+                        confidence: maxConfidence,
+                        tags: [...allTags.slice(0, 15), "consolidated"],
+                        metadata: {
+                            source: "consolidation",
+                            member_ids: cluster.map((e) => e.id),
+                            consolidated_at: new Date().toISOString(),
+                        },
+                    });
+                    const newEntity = createResult.entity;
+                    if (newEntity?.id) {
+                        detail.entityId = newEntity.id;
+                        // Create part_of relations from members → consolidated entity
+                        for (const member of cluster) {
+                            try {
+                                await client.createMemoryRelation({
+                                    source_id: member.id,
+                                    target_id: newEntity.id,
+                                    relation_type: "part_of",
+                                    confidence: 0.8,
+                                });
+                            }
+                            catch {
+                                // Skip duplicate relations
+                            }
+                        }
+                        // Downgrade member confidence by 0.3 (min 0.1)
+                        for (const member of cluster) {
+                            try {
+                                const newConf = Math.max(member.confidence - 0.3, 0.1);
+                                await client.updateMemoryEntity(member.id, {
+                                    confidence: newConf,
+                                    metadata: {
+                                        consolidated_into: newEntity.id,
+                                        original_confidence: member.confidence,
+                                    },
+                                });
+                            }
+                            catch {
+                                // Non-fatal
+                            }
+                        }
+                        result.consolidated++;
+                    }
+                }
+                catch {
+                    // Non-fatal: consolidation failure for one cluster shouldn't block others
+                }
+            }
+            else {
+                result.consolidated++;
+            }
+            result.details.push(detail);
+        }
+    }
+    return result;
+}
+/**
+ * Derive a cluster title from the most common meaningful words across member titles.
+ */
+function deriveClusterTitle(cluster, type) {
+    const stopWords = new Set([
+        "the",
+        "a",
+        "an",
+        "is",
+        "are",
+        "was",
+        "were",
+        "be",
+        "been",
+        "being",
+        "have",
+        "has",
+        "had",
+        "do",
+        "does",
+        "did",
+        "will",
+        "shall",
+        "would",
+        "should",
+        "may",
+        "might",
+        "can",
+        "could",
+        "of",
+        "in",
+        "to",
+        "for",
+        "with",
+        "on",
+        "at",
+        "from",
+        "by",
+        "and",
+        "or",
+        "but",
+        "not",
+        "session",
+        "blocker",
+        "pattern",
+        "solution",
+        "error",
+        "task",
+        "mid-session",
+    ]);
+    const wordCounts = new Map();
+    for (const entity of cluster) {
+        const words = entity.title
+            .toLowerCase()
+            .split(/\W+/)
+            .filter((w) => w.length > 2 && !stopWords.has(w));
+        for (const word of words) {
+            wordCounts.set(word, (wordCounts.get(word) || 0) + 1);
+        }
+    }
+    // Sort by frequency, take top 3
+    const topWords = [...wordCounts.entries()]
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, 3)
+        .map(([word]) => word);
+    const suffix = topWords.length > 0 ? topWords.join(", ") : "various";
+    return `Consolidated ${type}: ${suffix}`;
+}