@gethmy/mcp 2.4.6 → 2.5.0

package/src/consolidation.ts

@@ -1,383 +0,0 @@
-/**
- * Smart Memory Consolidation
- *
- * Clusters similar draft/episode memories and merges them into
- * consolidated reference entities to reduce noise and improve retrieval.
- */
-
-import type { HarmonyApiClient } from "./api-client.js";
-import { findSimilarEntities } from "./graph-expansion.js";
-
-interface MemoryEntity {
-  id: string;
-  type: string;
-  title: string;
-  content: string;
-  confidence: number;
-  memory_tier: string;
-  tags: string[];
-  metadata?: Record<string, unknown>;
-  updated_at?: string;
-}
-
-export interface ConsolidationResult {
-  consolidated: number;
-  clustersFound: number;
-  entitiesProcessed: number;
-  details: Array<{
-    clusterSize: number;
-    mergedTitle: string;
-    memberTitles: string[];
-    entityId?: string; // set when not dry run
-  }>;
-}
-
-export interface ConsolidationOptions {
-  dryRun?: boolean;
-  minClusterSize?: number; // Default: 3 (was 2 — raised to avoid premature merging)
-}
-
-/**
- * Consolidate similar draft/episode memories into reference entities.
- *
- * 1. Lists all draft and episode tier entities in scope
- * 2. Groups by entity type
- * 3. For each type group, finds clusters via embedding similarity
- * 4. Merges clusters into new reference entities with part_of relations
- */
-export async function consolidateMemories(
-  client: HarmonyApiClient,
-  workspaceId: string,
-  projectId?: string,
-  options?: ConsolidationOptions,
-): Promise<ConsolidationResult> {
-  const dryRun = options?.dryRun !== false; // default true
-  const minClusterSize = options?.minClusterSize ?? 3; // raised from 2 to reduce noise
-
-  const result: ConsolidationResult = {
-    consolidated: 0,
-    clustersFound: 0,
-    entitiesProcessed: 0,
-    details: [],
-  };
-
-  // Step 1: Fetch all draft and episode entities
-  const listResult = await client.listMemoryEntities({
-    workspace_id: workspaceId,
-    project_id: projectId,
-    limit: 100,
-  });
-
-  const allEntities = ((listResult.entities || []) as MemoryEntity[]).filter(
-    (e) => e.memory_tier === "draft" || e.memory_tier === "episode",
-  );
-
-  result.entitiesProcessed = allEntities.length;
-  if (allEntities.length < minClusterSize) return result;
-
-  // Step 2: Group by type
-  const typeGroups = new Map<string, MemoryEntity[]>();
-  for (const entity of allEntities) {
-    const group = typeGroups.get(entity.type) || [];
-    group.push(entity);
-    typeGroups.set(entity.type, group);
-  }
-
-  // Step 3: Find clusters within each type group
-  for (const [type, entities] of typeGroups) {
-    if (entities.length < minClusterSize) continue;
-
-    const clustered = new Set<string>();
-    const clusters: MemoryEntity[][] = [];
-
-    for (const entity of entities) {
-      if (clustered.has(entity.id)) continue;
-
-      // Search for similar entities using embedding-based search
-      const similar = await findSimilarEntities(
-        client,
-        entity.title,
-        entity.content,
-        workspaceId,
-        {
-          projectId,
-          limit: 20,
-          minRrfScore: 0.01,
-          excludeIds: [...clustered],
-        },
-      );
-
-      // Filter to only entities in our current type group that aren't yet clustered
-      const entityIdSet = new Set(entities.map((e) => e.id));
-      const clusterMembers = similar.filter(
-        (s) =>
-          entityIdSet.has(s.id) &&
-          !clustered.has(s.id) &&
-          s.id !== entity.id &&
-          s.type === type,
-      );
-
-      if (clusterMembers.length >= minClusterSize - 1) {
-        const cluster = [
-          entity,
-          ...clusterMembers.slice(0, 5).map((s) => {
-            // Map back to full entity from our list
-            return entities.find((e) => e.id === s.id) || entity;
-          }),
-        ];
-
-        // Deduplicate by id
-        const uniqueCluster: MemoryEntity[] = [];
-        const seen = new Set<string>();
-        for (const member of cluster) {
-          if (!seen.has(member.id)) {
-            seen.add(member.id);
-            uniqueCluster.push(member);
-          }
-        }
-
-        if (uniqueCluster.length >= minClusterSize) {
-          clusters.push(uniqueCluster);
-          for (const member of uniqueCluster) {
-            clustered.add(member.id);
-          }
-        }
-      }
-    }
-
-    // Step 4: Create consolidated entities for each cluster
-    for (const cluster of clusters) {
-      result.clustersFound++;
-
-      // Derive title from most common words across cluster titles
-      const mergedTitle = deriveClusterTitle(cluster, type);
-      const memberTitles = cluster.map((e) => e.title);
-
-      // Synthesize content: extract unique knowledge from each member,
-      // not just a bullet list of titles. Each member's content is trimmed
-      // to its first meaningful paragraph (skipping headers and metadata).
-      const mergedContent = synthesizeClusterContent(cluster, type);
-
-      // Max confidence from cluster members
-      const maxConfidence = Math.max(...cluster.map((e) => e.confidence));
-
-      // Union of all tags (deduped)
-      const allTags = [...new Set(cluster.flatMap((e) => e.tags || []))];
-
-      const detail: ConsolidationResult["details"][0] = {
-        clusterSize: cluster.length,
-        mergedTitle,
-        memberTitles,
-      };
-
-      if (!dryRun) {
-        try {
-          // Create consolidated reference entity
-          const createResult = await client.createMemoryEntity({
-            workspace_id: workspaceId,
-            project_id: projectId,
-            type,
-            scope: "project",
-            memory_tier: "reference",
-            title: mergedTitle,
-            content: mergedContent,
-            confidence: maxConfidence,
-            tags: [...allTags.slice(0, 15), "consolidated"],
-            metadata: {
-              source: "consolidation",
-              member_ids: cluster.map((e) => e.id),
-              consolidated_at: new Date().toISOString(),
-            },
-          });
-
-          const newEntity = createResult.entity as { id: string };
-          if (newEntity?.id) {
-            detail.entityId = newEntity.id;
-
-            // Create part_of relations from members → consolidated entity
-            for (const member of cluster) {
-              try {
-                await client.createMemoryRelation({
-                  source_id: member.id,
-                  target_id: newEntity.id,
-                  relation_type: "part_of",
-                  confidence: 0.8,
-                });
-              } catch {
-                // Skip duplicate relations
-              }
-            }
-
-            // Downgrade member confidence by 0.3 (min 0.1)
-            for (const member of cluster) {
-              try {
-                const newConf = Math.max(member.confidence - 0.3, 0.1);
-                await client.updateMemoryEntity(member.id, {
-                  confidence: newConf,
-                  metadata: {
-                    consolidated_into: newEntity.id,
-                    original_confidence: member.confidence,
-                  },
-                });
-              } catch {
-                // Non-fatal
-              }
-            }
-
-            result.consolidated++;
-          }
-        } catch {
-          // Non-fatal: consolidation failure for one cluster shouldn't block others
-        }
-      } else {
-        result.consolidated++;
-      }
-
-      result.details.push(detail);
-    }
-  }
-
-  return result;
-}
-
-/**
- * Synthesize cluster content by extracting unique, actionable knowledge
- * from each member entity. Skips boilerplate (headers, metadata, agent names)
- * and deduplicates similar lines across members.
- */
-function synthesizeClusterContent(
-  cluster: MemoryEntity[],
-  type: string,
-): string {
-  // Lines to skip: headers, agent metadata, timestamps, progress percentages
-  const SKIP_PATTERNS = [
-    /^##\s/,
-    /^Agent:/,
-    /^Duration:/,
-    /^Labels:/,
-    /^Progress:/,
-    /^Session status:/,
-    /^Completed at/,
-    /^Final state:/,
-    /^Related:/,
-    /^When working on:/,
-    /^\d+\.\s+.+\(\d+%,\s*\+\d+%\)/, // procedure step with progress percentages
-    /^Last updated:/,
-    /^Recurring pattern:/,
-    /^Consolidated from/,
-  ];
-
-  const seenLines = new Set<string>();
-  const knowledgeLines: string[] = [];
-
-  for (const entity of cluster) {
-    const lines = entity.content.split("\n").map((l) => l.trim());
-
-    for (const line of lines) {
-      if (!line || line.length < 20) continue;
-      if (SKIP_PATTERNS.some((p) => p.test(line))) continue;
-
-      // Normalize for dedup: lowercase, strip markdown formatting
-      const normalized = line
-        .toLowerCase()
-        .replace(/[*_`#[\]]/g, "")
-        .trim();
-      if (seenLines.has(normalized)) continue;
-      seenLines.add(normalized);
-
-      knowledgeLines.push(line);
-    }
-  }
-
-  if (knowledgeLines.length === 0) {
-    // Fallback: if no knowledge was extractable, use a compact summary
-    return `${cluster.length} related ${type} entities consolidated. Original titles:\n${cluster.map((e) => `- ${e.title}`).join("\n")}`;
-  }
-
-  // Cap at ~400 tokens worth of content (1600 chars)
-  const MAX_CHARS = 1600;
-  const result: string[] = [
-    `Consolidated knowledge from ${cluster.length} ${type} entities:\n`,
-  ];
-  let charCount = result[0].length;
-
-  for (const line of knowledgeLines) {
-    if (charCount + line.length + 3 > MAX_CHARS) break;
-    result.push(`- ${line}`);
-    charCount += line.length + 3;
-  }
-
-  return result.join("\n");
-}
-
-/**
- * Derive a cluster title from the most common meaningful words across member titles.
- */
-function deriveClusterTitle(cluster: MemoryEntity[], type: string): string {
-  const stopWords = new Set([
-    "the",
-    "a",
-    "an",
-    "is",
-    "are",
-    "was",
-    "were",
-    "be",
-    "been",
-    "being",
-    "have",
-    "has",
-    "had",
-    "do",
-    "does",
-    "did",
-    "will",
-    "shall",
-    "would",
-    "should",
-    "may",
-    "might",
-    "can",
-    "could",
-    "of",
-    "in",
-    "to",
-    "for",
-    "with",
-    "on",
-    "at",
-    "from",
-    "by",
-    "and",
-    "or",
-    "but",
-    "not",
-    "session",
-    "blocker",
-    "pattern",
-    "solution",
-    "error",
-    "task",
-    "mid-session",
-  ]);
-
-  const wordCounts = new Map<string, number>();
-  for (const entity of cluster) {
-    const words = entity.title
-      .toLowerCase()
-      .split(/\W+/)
-      .filter((w) => w.length > 2 && !stopWords.has(w));
-    for (const word of words) {
-      wordCounts.set(word, (wordCounts.get(word) || 0) + 1);
-    }
-  }
-
-  // Sort by frequency, take top 4 for more descriptive titles
-  const topWords = [...wordCounts.entries()]
-    .sort((a, b) => b[1] - a[1])
-    .slice(0, 4)
-    .map(([word]) => word[0].toUpperCase() + word.slice(1));
-
-  const suffix = topWords.length > 0 ? topWords.join(" / ") : "Various";
-  return `${type[0].toUpperCase() + type.slice(1)}: ${suffix}`;
-}