@gethmy/mcp 1.0.0 → 2.1.0

package/src/consolidation.ts

@@ -0,0 +1,314 @@
+/**
+ * Smart Memory Consolidation
+ *
+ * Clusters similar draft/episode memories and merges them into
+ * consolidated reference entities to reduce noise and improve retrieval.
+ */
+
+import type { HarmonyApiClient } from "./api-client.js";
+import { findSimilarEntities } from "./graph-expansion.js";
+
+interface MemoryEntity {
+  id: string;
+  type: string;
+  title: string;
+  content: string;
+  confidence: number;
+  memory_tier: string;
+  tags: string[];
+  metadata?: Record<string, unknown>;
+  updated_at?: string;
+}
+
+export interface ConsolidationResult {
+  consolidated: number;
+  clustersFound: number;
+  entitiesProcessed: number;
+  details: Array<{
+    clusterSize: number;
+    mergedTitle: string;
+    memberTitles: string[];
+    entityId?: string; // set when not dry run
+  }>;
+}
+
+export interface ConsolidationOptions {
+  dryRun?: boolean;
+  minClusterSize?: number;
+}
+
+/**
+ * Consolidate similar draft/episode memories into reference entities.
+ *
+ * 1. Lists all draft and episode tier entities in scope
+ * 2. Groups by entity type
+ * 3. For each type group, finds clusters via embedding similarity
+ * 4. Merges clusters into new reference entities with part_of relations
+ */
+export async function consolidateMemories(
+  client: HarmonyApiClient,
+  workspaceId: string,
+  projectId?: string,
+  options?: ConsolidationOptions,
+): Promise<ConsolidationResult> {
+  const dryRun = options?.dryRun !== false; // default true
+  const minClusterSize = options?.minClusterSize ?? 2;
+
+  const result: ConsolidationResult = {
+    consolidated: 0,
+    clustersFound: 0,
+    entitiesProcessed: 0,
+    details: [],
+  };
+
+  // Step 1: Fetch all draft and episode entities
+  const listResult = await client.listMemoryEntities({
+    workspace_id: workspaceId,
+    project_id: projectId,
+    limit: 100,
+  });
+
+  const allEntities = ((listResult.entities || []) as MemoryEntity[]).filter(
+    (e) => e.memory_tier === "draft" || e.memory_tier === "episode",
+  );
+
+  result.entitiesProcessed = allEntities.length;
+  if (allEntities.length < minClusterSize) return result;
+
+  // Step 2: Group by type
+  const typeGroups = new Map<string, MemoryEntity[]>();
+  for (const entity of allEntities) {
+    const group = typeGroups.get(entity.type) || [];
+    group.push(entity);
+    typeGroups.set(entity.type, group);
+  }
+
+  // Step 3: Find clusters within each type group
+  for (const [type, entities] of typeGroups) {
+    if (entities.length < minClusterSize) continue;
+
+    const clustered = new Set<string>();
+    const clusters: MemoryEntity[][] = [];
+
+    for (const entity of entities) {
+      if (clustered.has(entity.id)) continue;
+
+      // Search for similar entities using embedding-based search
+      const similar = await findSimilarEntities(
+        client,
+        entity.title,
+        entity.content,
+        workspaceId,
+        {
+          projectId,
+          limit: 20,
+          minRrfScore: 0.01,
+          excludeIds: [...clustered],
+        },
+      );
+
+      // Filter to only entities in our current type group that aren't yet clustered
+      const entityIdSet = new Set(entities.map((e) => e.id));
+      const clusterMembers = similar.filter(
+        (s) =>
+          entityIdSet.has(s.id) &&
+          !clustered.has(s.id) &&
+          s.id !== entity.id &&
+          s.type === type,
+      );
+
+      if (clusterMembers.length >= minClusterSize - 1) {
+        const cluster = [
+          entity,
+          ...clusterMembers.slice(0, 5).map((s) => {
+            // Map back to full entity from our list
+            return entities.find((e) => e.id === s.id) || entity;
+          }),
+        ];
+
+        // Deduplicate by id
+        const uniqueCluster: MemoryEntity[] = [];
+        const seen = new Set<string>();
+        for (const member of cluster) {
+          if (!seen.has(member.id)) {
+            seen.add(member.id);
+            uniqueCluster.push(member);
+          }
+        }
+
+        if (uniqueCluster.length >= minClusterSize) {
+          clusters.push(uniqueCluster);
+          for (const member of uniqueCluster) {
+            clustered.add(member.id);
+          }
+        }
+      }
+    }
+
+    // Step 4: Create consolidated entities for each cluster
+    for (const cluster of clusters) {
+      result.clustersFound++;
+
+      // Derive title from most common words across cluster titles
+      const mergedTitle = deriveClusterTitle(cluster, type);
+      const memberTitles = cluster.map((e) => e.title);
+
+      // Merge content as bullet points
+      const mergedContent = [
+        `Consolidated from ${cluster.length} ${type} memories:\n`,
+        ...cluster.map((e) => `- **${e.title}**: ${e.content.slice(0, 200)}`),
+      ].join("\n");
+
+      // Max confidence from cluster members
+      const maxConfidence = Math.max(...cluster.map((e) => e.confidence));
+
+      // Union of all tags (deduped)
+      const allTags = [...new Set(cluster.flatMap((e) => e.tags || []))];
+
+      const detail: ConsolidationResult["details"][0] = {
+        clusterSize: cluster.length,
+        mergedTitle,
+        memberTitles,
+      };
+
+      if (!dryRun) {
+        try {
+          // Create consolidated reference entity
+          const createResult = await client.createMemoryEntity({
+            workspace_id: workspaceId,
+            project_id: projectId,
+            type,
+            scope: "project",
+            memory_tier: "reference",
+            title: mergedTitle,
+            content: mergedContent,
+            confidence: maxConfidence,
+            tags: [...allTags.slice(0, 15), "consolidated"],
+            metadata: {
+              source: "consolidation",
+              member_ids: cluster.map((e) => e.id),
+              consolidated_at: new Date().toISOString(),
+            },
+          });
+
+          const newEntity = createResult.entity as { id: string };
+          if (newEntity?.id) {
+            detail.entityId = newEntity.id;
+
+            // Create part_of relations from members → consolidated entity
+            for (const member of cluster) {
+              try {
+                await client.createMemoryRelation({
+                  source_id: member.id,
+                  target_id: newEntity.id,
+                  relation_type: "part_of",
+                  confidence: 0.8,
+                });
+              } catch {
+                // Skip duplicate relations
+              }
+            }
+
+            // Downgrade member confidence by 0.3 (min 0.1)
+            for (const member of cluster) {
+              try {
+                const newConf = Math.max(member.confidence - 0.3, 0.1);
+                await client.updateMemoryEntity(member.id, {
+                  confidence: newConf,
+                  metadata: {
+                    consolidated_into: newEntity.id,
+                    original_confidence: member.confidence,
+                  },
+                });
+              } catch {
+                // Non-fatal
+              }
+            }
+
+            result.consolidated++;
+          }
+        } catch {
+          // Non-fatal: consolidation failure for one cluster shouldn't block others
+        }
+      } else {
+        result.consolidated++;
+      }
+
+      result.details.push(detail);
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Derive a cluster title from the most common meaningful words across member titles.
+ */
+function deriveClusterTitle(cluster: MemoryEntity[], type: string): string {
+  const stopWords = new Set([
+    "the",
+    "a",
+    "an",
+    "is",
+    "are",
+    "was",
+    "were",
+    "be",
+    "been",
+    "being",
+    "have",
+    "has",
+    "had",
+    "do",
+    "does",
+    "did",
+    "will",
+    "shall",
+    "would",
+    "should",
+    "may",
+    "might",
+    "can",
+    "could",
+    "of",
+    "in",
+    "to",
+    "for",
+    "with",
+    "on",
+    "at",
+    "from",
+    "by",
+    "and",
+    "or",
+    "but",
+    "not",
+    "session",
+    "blocker",
+    "pattern",
+    "solution",
+    "error",
+    "task",
+    "mid-session",
+  ]);
+
+  const wordCounts = new Map<string, number>();
+  for (const entity of cluster) {
+    const words = entity.title
+      .toLowerCase()
+      .split(/\W+/)
+      .filter((w) => w.length > 2 && !stopWords.has(w));
+    for (const word of words) {
+      wordCounts.set(word, (wordCounts.get(word) || 0) + 1);
+    }
+  }
+
+  // Sort by frequency, take top 3
+  const topWords = [...wordCounts.entries()]
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, 3)
+    .map(([word]) => word);
+
+  const suffix = topWords.length > 0 ? topWords.join(", ") : "various";
+  return `Consolidated ${type}: ${suffix}`;
+}