@gethmy/mcp 2.2.4 → 2.3.1

package/src/memory-cleanup.ts

@@ -0,0 +1,658 @@
+/**
+ * Unified Memory Cleanup
+ *
+ * Orchestrates a 5-stage cleanup pipeline: prune stale drafts, consolidate
+ * similar memories, detect orphans, detect duplicates, and backfill embeddings.
+ *
+ * All stages are non-fatal — individual failures are collected but never block
+ * the remaining stages. Defaults to dry-run mode (preview only).
+ */
+
+import { evaluateLifecycle } from "@harmony/memory";
+import type { HarmonyApiClient } from "./api-client.js";
+import {
+  type ConsolidationResult,
+  consolidateMemories,
+} from "./consolidation.js";
+import { findSimilarEntities } from "./graph-expansion.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+interface MemoryEntity {
+  id: string;
+  type: string;
+  title: string;
+  content: string;
+  confidence: number;
+  memory_tier: "draft" | "episode" | "reference";
+  access_count: number;
+  last_accessed_at: string | null;
+  created_at: string;
+  updated_at?: string;
+  tags?: string[];
+}
+
+export type CleanupStep =
+  | "prune"
+  | "consolidate"
+  | "orphans"
+  | "duplicates"
+  | "backfill";
+
+export interface CleanupOptions {
+  dryRun?: boolean;
+  steps?: CleanupStep[];
+  maxAgeDays?: number;
+  minClusterSize?: number;
+  orphanAgeDays?: number;
+}
+
+interface PruneStepResult {
+  staleDraftsFound: number;
+  pruned: number;
+  items: Array<{
+    id: string;
+    title: string;
+    ageDays: number;
+    decayScore: number;
+  }>;
+}
+
+interface ConsolidateStepResult {
+  clustersFound: number;
+  entitiesProcessed: number;
+  consolidated: number;
+  details: ConsolidationResult["details"];
+}
+
+interface OrphanStepResult {
+  orphansFound: number;
+  removed: number;
+  items: Array<{
+    id: string;
+    title: string;
+    type: string;
+    tier: string;
+    ageDays: number;
+    accessCount: number;
+  }>;
+}
+
+interface DuplicateStepResult {
+  duplicatePairsFound: number;
+  resolved: number;
+  pairs: Array<{
+    keepId: string;
+    keepTitle: string;
+    removeId: string;
+    removeTitle: string;
+    similarity: number;
+  }>;
+}
+
+interface BackfillStepResult {
+  processed: number;
+  remaining: number;
+  errors: Array<{ entity_id: string; error: string }>;
+}
+
+export interface CleanupReport {
+  success: boolean;
+  dryRun: boolean;
+  timestamp: string;
+  workspace: { id: string; projectId?: string };
+
+  summary: {
+    totalEntities: number;
+    issuesFound: number;
+    actionsTaken: number;
+  };
+
+  steps: {
+    prune?: PruneStepResult;
+    consolidate?: ConsolidateStepResult;
+    orphans?: OrphanStepResult;
+    duplicates?: DuplicateStepResult;
+    backfill?: BackfillStepResult;
+  };
+
+  errors: Array<{ step: string; message: string }>;
+  healthReport: string;
+}
+
+const ALL_STEPS: CleanupStep[] = [
+  "prune",
+  "consolidate",
+  "orphans",
+  "duplicates",
+  "backfill",
+];
+
+const MS_PER_DAY = 1000 * 60 * 60 * 24;
+const MAX_ENTITIES_FETCH = 200;
+const DUPLICATE_SIMILARITY_THRESHOLD = 0.85;
+const CONCURRENCY_LIMIT = 5;
+
+// ---------------------------------------------------------------------------
+// Main orchestrator
+// ---------------------------------------------------------------------------
+
+export async function runMemoryCleanup(
+  client: HarmonyApiClient,
+  workspaceId: string,
+  projectId?: string,
+  options?: CleanupOptions,
+): Promise<CleanupReport> {
+  const dryRun = options?.dryRun !== false;
+  const steps = options?.steps ?? ALL_STEPS;
+  const maxAgeDays = options?.maxAgeDays ?? 30;
+  const minClusterSize = options?.minClusterSize ?? 3;
+  const orphanAgeDays = options?.orphanAgeDays ?? 14;
+
+  const report: CleanupReport = {
+    success: true,
+    dryRun,
+    timestamp: new Date().toISOString(),
+    workspace: { id: workspaceId, projectId },
+    summary: { totalEntities: 0, issuesFound: 0, actionsTaken: 0 },
+    steps: {},
+    errors: [],
+    healthReport: "",
+  };
+
+  // Fetch all entities once (shared across steps)
+  let entities: MemoryEntity[] = [];
+  try {
+    const listResult = await client.listMemoryEntities({
+      workspace_id: workspaceId,
+      project_id: projectId,
+      limit: MAX_ENTITIES_FETCH,
+    });
+    entities = (listResult.entities || []) as MemoryEntity[];
+    report.summary.totalEntities = entities.length;
+  } catch (err) {
+    report.errors.push({
+      step: "init",
+      message: `Failed to fetch entities: ${(err as Error).message}`,
+    });
+    report.success = false;
+    report.healthReport = generateHealthReport(report);
+    return report;
+  }
+
+  // Stage 1: Prune stale drafts
+  if (steps.includes("prune")) {
+    try {
+      report.steps.prune = runPruneStep(entities, maxAgeDays);
+      if (!dryRun) {
+        for (const item of report.steps.prune.items) {
+          try {
+            await client.deleteMemoryEntity(item.id);
+            report.steps.prune.pruned++;
+          } catch (err) {
+            report.errors.push({
+              step: "prune",
+              message: `Failed to delete ${item.id}: ${(err as Error).message}`,
+            });
+          }
+        }
+        report.summary.actionsTaken += report.steps.prune.pruned;
+      }
+      report.summary.issuesFound += report.steps.prune.staleDraftsFound;
+    } catch (err) {
+      report.errors.push({
+        step: "prune",
+        message: (err as Error).message,
+      });
+    }
+  }
+
+  // Stage 2: Consolidate similar memories
+  if (steps.includes("consolidate")) {
+    try {
+      const result = await consolidateMemories(client, workspaceId, projectId, {
+        dryRun,
+        minClusterSize,
+      });
+      report.steps.consolidate = {
+        clustersFound: result.clustersFound,
+        entitiesProcessed: result.entitiesProcessed,
+        consolidated: result.consolidated,
+        details: result.details,
+      };
+      report.summary.issuesFound += result.clustersFound;
+      if (!dryRun) report.summary.actionsTaken += result.consolidated;
+    } catch (err) {
+      report.errors.push({
+        step: "consolidate",
+        message: (err as Error).message,
+      });
+    }
+  }
+
+  // Stage 3: Detect orphans
+  if (steps.includes("orphans")) {
+    try {
+      report.steps.orphans = await runOrphanStep(
+        client,
+        entities,
+        orphanAgeDays,
+      );
+      if (!dryRun) {
+        for (const item of report.steps.orphans.items) {
+          try {
+            await client.deleteMemoryEntity(item.id);
+            report.steps.orphans.removed++;
+          } catch (err) {
+            report.errors.push({
+              step: "orphans",
+              message: `Failed to delete ${item.id}: ${(err as Error).message}`,
+            });
+          }
+        }
+        report.summary.actionsTaken += report.steps.orphans.removed;
+      }
+      report.summary.issuesFound += report.steps.orphans.orphansFound;
+    } catch (err) {
+      report.errors.push({
+        step: "orphans",
+        message: (err as Error).message,
+      });
+    }
+  }
+
+  // Stage 4: Detect duplicates
+  if (steps.includes("duplicates")) {
+    try {
+      report.steps.duplicates = await runDuplicateStep(
+        client,
+        entities,
+        workspaceId,
+        projectId,
+      );
+      if (!dryRun) {
+        for (const pair of report.steps.duplicates.pairs) {
+          try {
+            await client.deleteMemoryEntity(pair.removeId);
+            report.steps.duplicates.resolved++;
+          } catch (err) {
+            report.errors.push({
+              step: "duplicates",
+              message: `Failed to delete ${pair.removeId}: ${(err as Error).message}`,
+            });
+          }
+        }
+        report.summary.actionsTaken += report.steps.duplicates.resolved;
+      }
+      report.summary.issuesFound += report.steps.duplicates.duplicatePairsFound;
+    } catch (err) {
+      report.errors.push({
+        step: "duplicates",
+        message: (err as Error).message,
+      });
+    }
+  }
+
+  // Stage 5: Backfill embeddings
+  if (steps.includes("backfill")) {
+    try {
+      if (dryRun) {
+        // In dry-run, just report that backfill would run
+        report.steps.backfill = {
+          processed: 0,
+          remaining: -1,
+          errors: [],
+        };
+      } else {
+        const result = await client.backfillEmbeddings(workspaceId);
+        report.steps.backfill = {
+          processed: result.processed,
+          remaining: result.remaining,
+          errors: result.errors || [],
+        };
+        report.summary.actionsTaken += result.processed;
+      }
+    } catch (err) {
+      report.errors.push({
+        step: "backfill",
+        message: (err as Error).message,
+      });
+    }
+  }
+
+  report.healthReport = generateHealthReport(report);
+  return report;
+}
+
+// ---------------------------------------------------------------------------
+// Step implementations
+// ---------------------------------------------------------------------------
+
+function runPruneStep(
+  entities: MemoryEntity[],
+  maxAgeDays: number,
+): PruneStepResult {
+  const now = Date.now();
+  const drafts = entities.filter((e) => e.memory_tier === "draft");
+  const stale: PruneStepResult["items"] = [];
+
+  for (const entity of drafts) {
+    const ageDays =
+      (now - new Date(entity.created_at).getTime()) / MS_PER_DAY;
+    if (ageDays < maxAgeDays) continue;
+
+    const lifecycle = evaluateLifecycle(entity);
+    stale.push({
+      id: entity.id,
+      title: entity.title,
+      ageDays: Math.round(ageDays),
+      decayScore: Math.round(lifecycle.decay.score * 100) / 100,
+    });
+  }
+
+  return { staleDraftsFound: stale.length, pruned: 0, items: stale };
+}
+
+async function runOrphanStep(
+  client: HarmonyApiClient,
+  entities: MemoryEntity[],
+  orphanAgeDays: number,
+): Promise<OrphanStepResult> {
+  const now = Date.now();
+  const result: OrphanStepResult = { orphansFound: 0, removed: 0, items: [] };
+
+  // Pre-filter: only check entities that look like orphan candidates
+  const candidates = entities.filter((e) => {
+    if (e.memory_tier === "reference") return false;
+    if (e.access_count >= 2) return false;
+    const ageDays =
+      (now - new Date(e.created_at).getTime()) / MS_PER_DAY;
+    return ageDays >= orphanAgeDays;
+  });
+
+  // Check relations in concurrent batches
+  for (let i = 0; i < candidates.length; i += CONCURRENCY_LIMIT) {
+    const batch = candidates.slice(i, i + CONCURRENCY_LIMIT);
+    const results = await Promise.allSettled(
+      batch.map(async (entity) => {
+        const related = await client.getRelatedEntities(entity.id);
+        const totalRelations =
+          (related.outgoing?.length || 0) + (related.incoming?.length || 0);
+        if (totalRelations > 0) return null;
+
+        const ageDays =
+          (now - new Date(entity.created_at).getTime()) / MS_PER_DAY;
+        return {
+          id: entity.id,
+          title: entity.title,
+          type: entity.type,
+          tier: entity.memory_tier,
+          ageDays: Math.round(ageDays),
+          accessCount: entity.access_count,
+        };
+      }),
+    );
+
+    for (const r of results) {
+      if (r.status === "fulfilled" && r.value) {
+        result.items.push(r.value);
+        result.orphansFound++;
+      }
+    }
+  }
+
+  return result;
+}
+
+async function runDuplicateStep(
+  client: HarmonyApiClient,
+  entities: MemoryEntity[],
+  workspaceId: string,
+  projectId?: string,
+): Promise<DuplicateStepResult> {
+  const result: DuplicateStepResult = {
+    duplicatePairsFound: 0,
+    resolved: 0,
+    pairs: [],
+  };
+
+  const seenPairs = new Set<string>();
+  const flaggedForRemoval = new Set<string>();
+  const entityMap = new Map(entities.map((e) => [e.id, e]));
+
+  // Pre-fetch similarities in concurrent batches
+  type SimilarMatch = {
+    id: string;
+    type: string;
+    title: string;
+    content: string;
+    confidence: number;
+  };
+  const similarityMap = new Map<string, SimilarMatch[]>();
+  for (let i = 0; i < entities.length; i += CONCURRENCY_LIMIT) {
+    const batch = entities.slice(i, i + CONCURRENCY_LIMIT);
+    const results = await Promise.allSettled(
+      batch.map(async (entity) => {
+        const similar = await findSimilarEntities(
+          client,
+          entity.title,
+          entity.content,
+          workspaceId,
+          { projectId, limit: 5, minRrfScore: 0.05, excludeIds: [entity.id] },
+        );
+        return { entityId: entity.id, similar };
+      }),
+    );
+    for (const r of results) {
+      if (r.status === "fulfilled") {
+        similarityMap.set(r.value.entityId, r.value.similar);
+      }
+    }
+  }
+
+  // Process pairs sequentially (flaggedForRemoval creates dependencies)
+  for (const entity of entities) {
+    if (flaggedForRemoval.has(entity.id)) continue;
+    const similar = similarityMap.get(entity.id) || [];
+
+    for (const match of similar) {
+      if (flaggedForRemoval.has(match.id)) continue;
+
+      const pairKey = [entity.id, match.id].sort().join(":");
+      if (seenPairs.has(pairKey)) continue;
+      seenPairs.add(pairKey);
+
+      const sim = titleSimilarity(entity.title, match.title);
+      if (sim < DUPLICATE_SIMILARITY_THRESHOLD) continue;
+
+      // Keep the one with higher confidence, more accesses, or higher tier
+      const entityScore = entityQualityScore(entity);
+      const matchEntity = entityMap.get(match.id);
+      const matchScore = matchEntity
+        ? entityQualityScore(matchEntity)
+        : match.confidence;
+
+      const [keep, remove] =
+        entityScore >= matchScore
+          ? [entity, { id: match.id, title: match.title }]
+          : [{ id: match.id, title: match.title }, entity];
+
+      flaggedForRemoval.add(remove.id);
+      result.pairs.push({
+        keepId: keep.id,
+        keepTitle: keep.title,
+        removeId: remove.id,
+        removeTitle: remove.title,
+        similarity: Math.round(sim * 100) / 100,
+      });
+      result.duplicatePairsFound++;
+    }
+  }
+
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+const TIER_WEIGHTS: Record<string, number> = {
+  reference: 3,
+  episode: 2,
+  draft: 1,
+};
+
+function entityQualityScore(entity: MemoryEntity): number {
+  return (
+    entity.confidence +
+    (TIER_WEIGHTS[entity.memory_tier] || 0) +
+    Math.min(entity.access_count, 10) * 0.1
+  );
+}
+
+function titleSimilarity(a: string, b: string): number {
+  const na = a.toLowerCase().trim();
+  const nb = b.toLowerCase().trim();
+  if (na === nb) return 1;
+
+  const wordsA = new Set(na.split(/\W+/).filter(Boolean));
+  const wordsB = new Set(nb.split(/\W+/).filter(Boolean));
+  if (wordsA.size === 0 || wordsB.size === 0) return 0;
+
+  let intersection = 0;
+  for (const w of wordsA) {
+    if (wordsB.has(w)) intersection++;
+  }
+  // Jaccard similarity
+  const union = wordsA.size + wordsB.size - intersection;
+  return union > 0 ? intersection / union : 0;
+}
+
+// ---------------------------------------------------------------------------
+// Health report renderer
+// ---------------------------------------------------------------------------
+
+function generateHealthReport(report: CleanupReport): string {
+  const mode = report.dryRun ? "Dry Run (preview)" : "Executed";
+  const lines: string[] = [
+    "# Memory Health Report\n",
+    `**Mode:** ${mode} | **Entities:** ${report.summary.totalEntities} | **Issues:** ${report.summary.issuesFound} | **Actions:** ${report.summary.actionsTaken}`,
+    "",
+  ];
+
+  if (report.summary.totalEntities >= MAX_ENTITIES_FETCH) {
+    lines.push(
+      `> **Note:** Entity count hit the ${MAX_ENTITIES_FETCH} fetch limit. Some entities may not have been analyzed.\n`,
+    );
+  }
+
+  // Prune
+  if (report.steps.prune) {
+    const p = report.steps.prune;
+    lines.push("## Stale Drafts");
+    if (p.staleDraftsFound === 0) {
+      lines.push("No stale drafts found.\n");
+    } else {
+      lines.push(
+        `Found **${p.staleDraftsFound}** stale drafts${!report.dryRun ? ` (pruned ${p.pruned})` : ""}:`,
+      );
+      lines.push("| Title | Age | Decay |");
+      lines.push("|-------|-----|-------|");
+      for (const item of p.items.slice(0, 20)) {
+        lines.push(`| ${item.title} | ${item.ageDays}d | ${item.decayScore} |`);
+      }
+      lines.push("");
+    }
+  }
+
+  // Consolidate
+  if (report.steps.consolidate) {
+    const c = report.steps.consolidate;
+    lines.push("## Consolidation");
+    if (c.clustersFound === 0) {
+      lines.push(
+        `Scanned ${c.entitiesProcessed} draft/episode entities — no clusters found.\n`,
+      );
+    } else {
+      lines.push(
+        `Found **${c.clustersFound}** clusters across ${c.entitiesProcessed} entities:`,
+      );
+      for (const d of c.details.slice(0, 10)) {
+        lines.push(`- **${d.mergedTitle}** — ${d.clusterSize} entities`);
+      }
+      lines.push("");
+    }
+  }
+
+  // Orphans
+  if (report.steps.orphans) {
+    const o = report.steps.orphans;
+    lines.push("## Orphaned Entities");
+    if (o.orphansFound === 0) {
+      lines.push("No orphans found.\n");
+    } else {
+      lines.push(
+        `Found **${o.orphansFound}** orphans${!report.dryRun ? ` (removed ${o.removed})` : ""}:`,
+      );
+      lines.push("| Title | Type | Tier | Age | Accesses |");
+      lines.push("|-------|------|------|-----|----------|");
+      for (const item of o.items.slice(0, 20)) {
+        lines.push(
+          `| ${item.title} | ${item.type} | ${item.tier} | ${item.ageDays}d | ${item.accessCount} |`,
+        );
+      }
+      lines.push("");
+    }
+  }
+
+  // Duplicates
+  if (report.steps.duplicates) {
+    const d = report.steps.duplicates;
+    lines.push("## Near-Duplicates");
+    if (d.duplicatePairsFound === 0) {
+      lines.push("No duplicates found.\n");
+    } else {
+      lines.push(
+        `Found **${d.duplicatePairsFound}** duplicate pairs${!report.dryRun ? ` (resolved ${d.resolved})` : ""}:`,
+      );
+      for (const pair of d.pairs.slice(0, 20)) {
+        lines.push(
+          `- "${pair.keepTitle}" ~ "${pair.removeTitle}" (${Math.round(pair.similarity * 100)}% similar, keep first)`,
+        );
+      }
+      lines.push("");
+    }
+  }
+
+  // Backfill
+  if (report.steps.backfill) {
+    const b = report.steps.backfill;
+    lines.push("## Embedding Coverage");
+    if (report.dryRun) {
+      lines.push("Backfill will run when executed with `dryRun: false`.\n");
+    } else if (b.remaining === 0) {
+      lines.push(`All embeddings up to date (processed ${b.processed}).\n`);
+    } else {
+      lines.push(
+        `Processed ${b.processed} entities. ${b.remaining} still need embeddings.\n`,
+      );
+    }
+  }
+
+  // Errors
+  if (report.errors.length > 0) {
+    lines.push("## Errors");
+    for (const e of report.errors) {
+      lines.push(`- **${e.step}:** ${e.message}`);
+    }
+    lines.push("");
+  }
+
+  if (report.dryRun) {
+    lines.push("---\n*Run with `dryRun: false` to execute cleanup.*");
+  }
+
+  return lines.join("\n");
+}

package/src/prompt-builder.ts

@@ -439,7 +439,19 @@ export function generatePrompt(
     sections.push(`- ${f}`);
   });
   sections.push(
-    `- **Memory:** When you discover important domain knowledge, architectural decisions, or infrastructure details, store them via \`harmony_remember\`. Focus on durable knowledge that future agents would benefit from — not ephemeral task details (those are auto-extracted from your session).`,
+    `- **Memory:** Store reusable knowledge via \`harmony_remember\`. Only store what a future agent couldn't easily discover from the code itself, applies beyond this specific card, and includes a "because" (not just what, but why).`,
+  );
+  sections.push(
+    `  - GOOD: "BoardContext card state must use moveCard action, never direct setState — optimistic updates depend on action ordering"`,
+  );
+  sections.push(
+    `  - GOOD: "Mobile bottom bar is 64px, overlaps fixed-position drawers — always add pb-16 to drawer content"`,
+  );
+  sections.push(
+    `  - BAD: "Fixed the login button" (no reusable knowledge — the fix is in the code)`,
+  );
+  sections.push(
+    `  - BAD: "Completed card #42" (ephemeral, auto-tracked by session)`,
   );
 
   // Output suggestions