@gethmy/mcp 2.3.3 → 2.4.0

package/src/memory-audit.ts

@@ -0,0 +1,485 @@
+/**
+ * Memory Quality Audit
+ *
+ * Scores every memory entity against modern quality standards and buckets
+ * them into keep / review / archive / delete. Designed to catch legacy
+ * memories that pre-date tier/decay/embedding optimizations.
+ *
+ * Composite score (0-100): confidence (25) + decay (20) + structural (15) +
+ * content (15) + tier-age-fit (15) + access (10). Legacy signals (default
+ * confidence, missing embedding, stuck draft, no graph presence) are reported
+ * but don't change the score — they provide explanation.
+ */
+
+import { evaluateLifecycle } from "@harmony/memory";
+import type { HarmonyApiClient } from "./api-client.js";
+
+// Embeddings migration landed 2026-02-18. Entities older than this without
+// embeddings are pre-vector and legacy by construction.
+const EMBEDDINGS_MIGRATION_AT = Date.parse("2026-02-18T00:00:00Z");
+const MS_PER_DAY = 1000 * 60 * 60 * 24;
+const BATCH_SIZE = 100;
+const CONCURRENCY_LIMIT = 5;
+
+interface AuditEntity {
+  id: string;
+  type: string;
+  title: string;
+  content: string;
+  confidence: number;
+  memory_tier: "draft" | "episode" | "reference";
+  access_count: number;
+  last_accessed_at: string | null;
+  created_at: string;
+  updated_at?: string;
+  tags?: string[];
+  metadata?: Record<string, unknown>;
+  embedding?: unknown;
+  promoted_from_id?: string | null;
+}
+
+export type AuditBucket = "keep" | "review" | "archive" | "delete";
+
+export interface AuditOptions {
+  dryRun?: boolean;
+  archiveBelow?: number;
+  deleteBelow?: number;
+  includeLegacyFlag?: boolean;
+  limit?: number;
+}
+
+interface EntityAudit {
+  id: string;
+  title: string;
+  type: string;
+  tier: string;
+  ageDays: number;
+  score: number;
+  bucket: AuditBucket;
+  reasons: string[];
+  legacy: boolean;
+  legacyReasons: string[];
+  subScores: {
+    confidence: number;
+    decay: number;
+    structural: number;
+    content: number;
+    tierAgeFit: number;
+    access: number;
+  };
+}
+
+export interface AuditReport {
+  success: boolean;
+  dryRun: boolean;
+  timestamp: string;
+  workspace: { id: string; projectId?: string };
+  summary: {
+    totalEntities: number;
+    scanned: number;
+    keep: number;
+    review: number;
+    archive: number;
+    delete: number;
+    legacyCount: number;
+  };
+  actionsTaken: {
+    flaggedReview: number;
+    archived: number;
+    deleted: number;
+  };
+  distribution: {
+    "0-20": number;
+    "20-40": number;
+    "40-70": number;
+    "70-100": number;
+  };
+  legacyBreakdown: {
+    defaultConfidence: number;
+    missingEmbedding: number;
+    stuckDraft: number;
+    noGraphPresence: number;
+  };
+  lowest: EntityAudit[];
+  errors: Array<{ entityId?: string; step: string; message: string }>;
+  healthReport: string;
+}
+
+const BOILERPLATE_PATTERNS = [
+  /^todo:?$/i,
+  /^placeholder/i,
+  /^\.\.\.$/,
+  /^untitled/i,
+  /^(note|memo|draft)\s*\d*$/i,
+];
+
+function isBoilerplate(title: string, content: string): boolean {
+  const t = title.trim();
+  const c = content.trim();
+  if (c.length === 0) return true;
+  for (const pat of BOILERPLATE_PATTERNS) {
+    if (pat.test(t)) return true;
+  }
+  return false;
+}
+
+function scoreEntity(
+  entity: AuditEntity,
+  relationCount: number,
+  archiveBelow: number,
+  deleteBelow: number,
+): EntityAudit {
+  const now = Date.now();
+  const ageDays = (now - Date.parse(entity.created_at)) / MS_PER_DAY;
+  // If an entity was never accessed, decay should start from creation time,
+  // not from "now" (which would falsely yield a fresh decay score of 1.0).
+  const effectiveLastAccess = entity.last_accessed_at ?? entity.created_at;
+  const lifecycle = evaluateLifecycle({
+    memory_tier: entity.memory_tier,
+    confidence: entity.confidence,
+    access_count: entity.access_count,
+    last_accessed_at: effectiveLastAccess,
+    created_at: entity.created_at,
+  });
+
+  const reasons: string[] = [];
+  const legacyReasons: string[] = [];
+
+  // Confidence (25)
+  const confidence = Math.max(0, Math.min(1, entity.confidence)) * 25;
+
+  // Decay (20)
+  const decay = Math.max(0, Math.min(1, lifecycle.decay.score)) * 20;
+  if (lifecycle.decay.score < 0.2)
+    reasons.push(`decay score ${lifecycle.decay.score.toFixed(2)}`);
+
+  // Structural completeness (15)
+  const hasEmbedding = entity.embedding != null;
+  const hasTags = (entity.tags?.length || 0) >= 1;
+  const hasRelations = relationCount > 0;
+  let structural = 0;
+  if (hasEmbedding) structural += 6;
+  if (hasTags) structural += 4;
+  if (hasRelations) structural += 5;
+  if (!hasEmbedding) reasons.push("no embedding");
+  if (!hasTags) reasons.push("no tags");
+  if (!hasRelations) reasons.push("no relations");
+
+  // Content quality (15)
+  let content = 0;
+  const contentLen = entity.content?.length || 0;
+  if (contentLen >= 80) content += 8;
+  const titleOk =
+    entity.title.trim().length >= 4 &&
+    !/^(untitled|draft|note)\b/i.test(entity.title.trim());
+  if (titleOk) content += 4;
+  if (!isBoilerplate(entity.title, entity.content)) content += 3;
+  if (contentLen < 80) reasons.push(`thin content (${contentLen} chars)`);
+  if (isBoilerplate(entity.title, entity.content))
+    reasons.push("boilerplate title/content");
+
+  // Tier-age fit (15)
+  let tierAgeFit = 15;
+  if (
+    entity.memory_tier === "draft" &&
+    ageDays > 60 &&
+    !entity.promoted_from_id
+  ) {
+    tierAgeFit = 0;
+    reasons.push("stuck draft >60d never promoted");
+  }
+  if (entity.promoted_from_id) {
+    tierAgeFit = Math.min(15, tierAgeFit + 5);
+  }
+
+  // Access pattern (10)
+  const access = Math.min(10, Math.log10((entity.access_count || 0) + 1) * 5);
+  if (entity.access_count === 0 && ageDays > 14) reasons.push("never accessed");
+
+  const raw = confidence + decay + structural + content + tierAgeFit + access;
+  const score = Math.round(Math.max(0, Math.min(100, raw)));
+
+  // Legacy detection
+  let legacy = false;
+  if (entity.confidence === 1.0 && entity.access_count === 0 && ageDays > 30) {
+    legacy = true;
+    legacyReasons.push("default confidence never validated");
+  }
+  if (
+    !hasEmbedding &&
+    Date.parse(entity.created_at) < EMBEDDINGS_MIGRATION_AT
+  ) {
+    legacy = true;
+    legacyReasons.push("pre-embeddings migration");
+  }
+  if (
+    entity.memory_tier === "draft" &&
+    ageDays > 60 &&
+    !entity.promoted_from_id
+  ) {
+    legacy = true;
+    legacyReasons.push("stuck draft");
+  }
+  if (!hasTags && !hasRelations) {
+    legacy = true;
+    legacyReasons.push("no graph presence");
+  }
+
+  // Bucket
+  let bucket: AuditBucket;
+  if (score < deleteBelow) bucket = "delete";
+  else if (score < archiveBelow) bucket = "archive";
+  else if (score < 70) bucket = "review";
+  else bucket = "keep";
+
+  return {
+    id: entity.id,
+    title: entity.title,
+    type: entity.type,
+    tier: entity.memory_tier,
+    ageDays: Math.round(ageDays),
+    score,
+    bucket,
+    reasons,
+    legacy,
+    legacyReasons,
+    subScores: {
+      confidence: Math.round(confidence),
+      decay: Math.round(decay),
+      structural,
+      content,
+      tierAgeFit,
+      access: Math.round(access),
+    },
+  };
+}
+
+export async function runMemoryAudit(
+  client: HarmonyApiClient,
+  workspaceId: string,
+  projectId?: string,
+  options?: AuditOptions,
+): Promise<AuditReport> {
+  const dryRun = options?.dryRun !== false;
+  const archiveBelow = options?.archiveBelow ?? 40;
+  const deleteBelow = options?.deleteBelow ?? 20;
+  const limit = options?.limit ?? 500;
+
+  const report: AuditReport = {
+    success: true,
+    dryRun,
+    timestamp: new Date().toISOString(),
+    workspace: { id: workspaceId, projectId },
+    summary: {
+      totalEntities: 0,
+      scanned: 0,
+      keep: 0,
+      review: 0,
+      archive: 0,
+      delete: 0,
+      legacyCount: 0,
+    },
+    actionsTaken: { flaggedReview: 0, archived: 0, deleted: 0 },
+    distribution: { "0-20": 0, "20-40": 0, "40-70": 0, "70-100": 0 },
+    legacyBreakdown: {
+      defaultConfidence: 0,
+      missingEmbedding: 0,
+      stuckDraft: 0,
+      noGraphPresence: 0,
+    },
+    lowest: [],
+    errors: [],
+    healthReport: "",
+  };
+
+  // Paginate
+  const entities: AuditEntity[] = [];
+  let offset = 0;
+  try {
+    while (entities.length < limit) {
+      const pageSize = Math.min(BATCH_SIZE, limit - entities.length);
+      const result = await client.listMemoryEntities({
+        workspace_id: workspaceId,
+        project_id: projectId,
+        limit: pageSize,
+        offset,
+      });
+      const page = (result.entities || []) as AuditEntity[];
+      if (page.length === 0) break;
+      entities.push(...page);
+      if (page.length < pageSize) break;
+      offset += pageSize;
+    }
+  } catch (err) {
+    report.errors.push({
+      step: "fetch",
+      message: `Failed to fetch entities: ${(err as Error).message}`,
+    });
+    report.success = false;
+    report.healthReport = renderReport(report);
+    return report;
+  }
+
+  report.summary.totalEntities = entities.length;
+
+  // Fetch relation counts concurrently
+  const relationCounts = new Map<string, number>();
+  for (let i = 0; i < entities.length; i += CONCURRENCY_LIMIT) {
+    const batch = entities.slice(i, i + CONCURRENCY_LIMIT);
+    const results = await Promise.allSettled(
+      batch.map(async (e) => {
+        const related = await client.getRelatedEntities(e.id);
+        const count =
+          (related.outgoing?.length || 0) + (related.incoming?.length || 0);
+        return { id: e.id, count };
+      }),
+    );
+    for (const r of results) {
+      if (r.status === "fulfilled") {
+        relationCounts.set(r.value.id, r.value.count);
+      }
+    }
+  }
+
+  // Score each entity
+  const audits: EntityAudit[] = [];
+  for (const entity of entities) {
+    const relCount = relationCounts.get(entity.id) ?? 0;
+    const audit = scoreEntity(entity, relCount, archiveBelow, deleteBelow);
+    audits.push(audit);
+    report.summary.scanned++;
+    report.summary[audit.bucket]++;
+    if (audit.legacy) report.summary.legacyCount++;
+
+    // Distribution bin
+    if (audit.score < 20) report.distribution["0-20"]++;
+    else if (audit.score < 40) report.distribution["20-40"]++;
+    else if (audit.score < 70) report.distribution["40-70"]++;
+    else report.distribution["70-100"]++;
+
+    // Legacy breakdown
+    for (const reason of audit.legacyReasons) {
+      if (reason.startsWith("default confidence"))
+        report.legacyBreakdown.defaultConfidence++;
+      else if (reason.startsWith("pre-embeddings"))
+        report.legacyBreakdown.missingEmbedding++;
+      else if (reason.startsWith("stuck draft"))
+        report.legacyBreakdown.stuckDraft++;
+      else if (reason.startsWith("no graph"))
+        report.legacyBreakdown.noGraphPresence++;
+    }
+  }
+
+  // Top 10 lowest-scoring
+  report.lowest = [...audits].sort((a, b) => a.score - b.score).slice(0, 10);
+
+  // Execute actions
+  if (!dryRun) {
+    for (const audit of audits) {
+      try {
+        if (audit.bucket === "delete") {
+          await client.deleteMemoryEntity(audit.id);
+          report.actionsTaken.deleted++;
+        } else if (audit.bucket === "archive") {
+          await client.updateMemoryEntity(audit.id, {
+            confidence: 0.25,
+            metadata: {
+              audit_archived_at: new Date().toISOString(),
+              audit_score: audit.score,
+              audit_reasons: audit.reasons,
+            },
+          });
+          report.actionsTaken.archived++;
+        } else if (audit.bucket === "review") {
+          await client.updateMemoryEntity(audit.id, {
+            metadata: {
+              needs_review: true,
+              audit_score: audit.score,
+              audit_reasons: audit.reasons,
+              audit_at: new Date().toISOString(),
+            },
+          });
+          report.actionsTaken.flaggedReview++;
+        }
+      } catch (err) {
+        report.errors.push({
+          entityId: audit.id,
+          step: audit.bucket,
+          message: (err as Error).message,
+        });
+      }
+    }
+  }
+
+  report.healthReport = renderReport(report);
+  return report;
+}
+
+function renderReport(report: AuditReport): string {
+  const mode = report.dryRun ? "Dry Run (preview)" : "Executed";
+  const s = report.summary;
+  const lines: string[] = [
+    "# Memory Quality Audit\n",
+    `**Mode:** ${mode} | **Scanned:** ${s.scanned}/${s.totalEntities} | **Legacy:** ${s.legacyCount}`,
+    "",
+    "## Distribution",
+    `- 70-100 (keep):    ${report.distribution["70-100"]}`,
+    `- 40-69  (review):  ${report.distribution["40-70"]}`,
+    `- 20-39  (archive): ${report.distribution["20-40"]}`,
+    `- 0-19   (delete):  ${report.distribution["0-20"]}`,
+    "",
+    "## Buckets",
+    `- **Keep:**    ${s.keep}`,
+    `- **Review:**  ${s.review}${!report.dryRun ? ` (flagged ${report.actionsTaken.flaggedReview})` : ""}`,
+    `- **Archive:** ${s.archive}${!report.dryRun ? ` (archived ${report.actionsTaken.archived})` : ""}`,
+    `- **Delete:**  ${s.delete}${!report.dryRun ? ` (deleted ${report.actionsTaken.deleted})` : ""}`,
+    "",
+  ];
+
+  const l = report.legacyBreakdown;
+  if (s.legacyCount > 0) {
+    lines.push("## Legacy Breakdown");
+    lines.push(`- Default confidence, never validated: ${l.defaultConfidence}`);
+    lines.push(`- Pre-embeddings migration:            ${l.missingEmbedding}`);
+    lines.push(`- Stuck drafts (>60d, no promotion):   ${l.stuckDraft}`);
+    lines.push(`- No tags + no relations:              ${l.noGraphPresence}`);
+    lines.push("");
+  }
+
+  if (report.lowest.length > 0) {
+    lines.push("## Lowest-Scoring (top 10)");
+    lines.push("| Score | Bucket | Tier | Age | Title | Reasons |");
+    lines.push("|-------|--------|------|-----|-------|---------|");
+    for (const a of report.lowest) {
+      const reasonStr = a.reasons.slice(0, 3).join(", ") || "—";
+      const titleTrunc =
+        a.title.length > 40 ? `${a.title.slice(0, 37)}...` : a.title;
+      lines.push(
+        `| ${a.score} | ${a.bucket} | ${a.tier} | ${a.ageDays}d | ${titleTrunc} | ${reasonStr} |`,
+      );
+    }
+    lines.push("");
+  }
+
+  if (report.errors.length > 0) {
+    lines.push("## Errors");
+    for (const e of report.errors.slice(0, 10)) {
+      lines.push(
+        `- **${e.step}${e.entityId ? ` ${e.entityId}` : ""}:** ${e.message}`,
+      );
+    }
+    lines.push("");
+  }
+
+  if (report.dryRun) {
+    lines.push("---");
+    lines.push(
+      "*Run with `dryRun: false` to flag review entries, archive low-quality memories, and delete worst offenders.*",
+    );
+  }
+
+  return lines.join("\n");
+}
+
+// Exposed for reuse from memory-cleanup.ts
+export { scoreEntity };