@gethmy/mcp 2.2.3 → 2.3.0

package/src/context-assembly.ts

@@ -5,7 +5,8 @@
  * for a given task, producing a manifest of what was included/excluded.
  */
 
-import { checkPromotion } from "@harmony/memory";
+import type { GraphRelation } from "@harmony/memory";
+import { checkPromotion, discoverRelatedContext } from "@harmony/memory";
 import type { HarmonyApiClient } from "./api-client.js";
 
 // Types
@@ -68,6 +69,13 @@ export interface AssembleContextOptions {
   cardId?: string;
   tokenBudget?: number; // Default: 4000 tokens
   client: HarmonyApiClient;
+  graphWalkEnabled?: boolean; // Default: true — enrich candidates via knowledge graph relations
+  queryExpansionEnabled?: boolean; // Default: true — expand query with synonyms/variations
+  enableLlmReranking?: boolean; // Default: false — LLM re-ranking when scores are clustered
+  rerankFn?: (
+    taskContext: string,
+    candidates: Array<{ id: string; title: string; snippet: string }>,
+  ) => Promise<string[]>; // Custom re-rank function
 }
 
 export interface AssembledContext {
@@ -79,7 +87,7 @@ export interface AssembledContext {
 // Constants
 const DEFAULT_TOKEN_BUDGET = 4000;
 const MAX_TOKENS_PER_ENTITY = 500;
-const MIN_RELEVANCE_THRESHOLD = 0.1;
+const MIN_RELEVANCE_THRESHOLD = 0.15; // raised from 0.1 to filter low-signal entities
 
 // Tier weight multipliers for relevance scoring
 const TIER_WEIGHTS: Record<MemoryTier, number> = {
@@ -98,8 +106,55 @@ const TIER_BUDGET_ALLOCATION: Record<MemoryTier, number> = {
   draft: 0.1,
 };
 
-// Minimum guaranteed slots per tier
-const MIN_REFERENCE_SLOTS = 3;
+// Minimum guaranteed slots per tier (reduced from 3 to avoid filling context with noise)
+const MIN_REFERENCE_SLOTS = 1;
+
+// Graph walk configuration
+const GRAPH_WALK_MAX_DEPTH = 1;
+const GRAPH_WALK_MAX_ENTITIES = 10;
+const GRAPH_WALK_MIN_CONFIDENCE = 0.5;
+const GRAPH_WALK_SEED_COUNT = 5;
+
+// Query expansion configuration
+const MAX_QUERY_VARIATIONS = 4;
+
+// LLM re-ranking configuration
+const RERANK_CLUSTER_THRESHOLD = 0.05;
+const RERANK_TOP_N = 10;
+const RERANK_MIN_CANDIDATES = 5;
+
+// Graph walk relation-type bonuses for relevance scoring
+const RELATION_BONUSES: Record<string, number> = {
+  depends_on: 0.15,
+  resolved_by: 0.2,
+  relates_to: 0.1,
+  implements: 0.15,
+  blocks: 0.15,
+  references: 0.1,
+  extends: 0.1,
+  caused_by: 0.15,
+};
+
+// Synonym map for query expansion (common dev term variations)
+// NOTE: Avoid circular references (auth->login, login->auth) — first synonym
+// is used for replacement, so each key should expand to non-overlapping terms.
+const QUERY_SYNONYMS: Record<string, string[]> = {
+  auth: ["authentication", "authorization", "session"],
+  authentication: ["auth", "session", "sign-in"],
+  login: ["sign-in", "authentication", "session"],
+  bug: ["error", "issue", "defect", "problem"],
+  error: ["exception", "failure", "issue"],
+  fix: ["resolve", "patch", "repair", "correct"],
+  deploy: ["deployment", "release", "ship", "publish"],
+  test: ["testing", "spec", "assertion", "verify"],
+  config: ["configuration", "settings", "setup"],
+  db: ["database", "storage", "persistence"],
+  database: ["storage", "persistence", "data store"],
+  api: ["endpoint", "route", "service"],
+  ui: ["frontend", "component", "view"],
+  perf: ["performance", "speed", "latency"],
+  performance: ["speed", "latency", "optimization"],
+};
 
 /**
  * Estimate token count (rough: 1 token per 4 chars)
@@ -108,6 +163,55 @@ function estimateTokens(text: string): number {
   return Math.ceil(text.length / 4);
 }
 
+/**
+ * Content quality gate: filter out entities that waste token budget.
+ * Returns true if the entity passes quality checks.
+ */
+function passesQualityGate(entity: ContextEntity): boolean {
+  const content = entity.content.trim();
+
+  // Gate 1: Minimum content length — entities with <50 chars of content
+  // are too shallow to provide value (e.g., "Resolved bug: Fix login button")
+  if (content.length < 50) return false;
+
+  // Gate 2: Title-content similarity — skip entities where content is just
+  // the title restated. Normalize both and check if content adds anything.
+  const normalizedTitle = entity.title
+    .toLowerCase()
+    .replace(/[^a-z0-9\s]/g, "")
+    .trim();
+  const normalizedContent = content
+    .toLowerCase()
+    .replace(/[^a-z0-9\s]/g, "")
+    .trim();
+  if (normalizedContent.length < normalizedTitle.length * 1.5) {
+    // Content is barely longer than the title — likely just a reformulation
+    return false;
+  }
+
+  // Gate 3: Pattern noise detection — skip "Pattern: recurring X (N instances)"
+  // and "Consolidated from N type memories:" entities that are just catalogs
+  if (
+    entity.type === "pattern" &&
+    /recurring .+ \(\d+ instances\)/i.test(entity.title)
+  ) {
+    // Check if content is just a member list (lines starting with "- ")
+    const lines = content.split("\n").filter((l) => l.trim().length > 0);
+    const bulletLines = lines.filter((l) => l.trim().startsWith("- "));
+    if (bulletLines.length > lines.length * 0.6) return false;
+  }
+
+  // Gate 4: Procedure quality — procedures must contain actual steps,
+  // not just a card title wrapped in a template
+  if (entity.type === "procedure") {
+    // Count numbered steps (1. ..., 2. ..., etc.)
+    const stepCount = (content.match(/^\d+\.\s/gm) || []).length;
+    if (stepCount < 3) return false;
+  }
+
+  return true;
+}
+
 /**
  * Generate a unique assembly ID
  */
@@ -154,6 +258,73 @@ function truncateContent(
   return { text: result, truncated: true };
 }
 
+/**
+ * Escape regex metacharacters in a string for safe use in RegExp constructor.
+ */
+function escapeRegex(str: string): string {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/**
+ * Expand a query into multiple search variations using synonym substitution.
+ * Returns the original query plus up to 3 additional variations (4 total).
+ */
+export function expandQuery(taskContext: string): string[] {
+  const queries = [taskContext];
+  const lowerQueries = [taskContext.toLowerCase()];
+  const words = taskContext
+    .toLowerCase()
+    .split(/\W+/)
+    .filter((w) => w.length > 2);
+
+  // Find words that have synonym expansions
+  const expandableWords = words.filter((w) => QUERY_SYNONYMS[w]);
+
+  for (const word of expandableWords) {
+    const synonyms = QUERY_SYNONYMS[word];
+    if (!synonyms) continue;
+    // Create a variation by replacing the word with its first synonym
+    const variation = taskContext.replace(
+      new RegExp(`\\b${escapeRegex(word)}\\b`, "gi"),
+      synonyms[0],
+    );
+    const lowerVariation = variation.toLowerCase();
+    if (
+      lowerVariation !== taskContext.toLowerCase() &&
+      !lowerQueries.includes(lowerVariation)
+    ) {
+      queries.push(variation);
+      lowerQueries.push(lowerVariation);
+    }
+    if (queries.length >= MAX_QUERY_VARIATIONS) break;
+  }
+
+  // Also extract key noun phrases as a compact query
+  if (words.length >= 3) {
+    const keyPhrases = words
+      .filter(
+        (w) =>
+          ![
+            "the",
+            "and",
+            "for",
+            "with",
+            "this",
+            "that",
+            "from",
+            "into",
+          ].includes(w),
+      )
+      .slice(0, 4)
+      .join(" ");
+    if (!lowerQueries.includes(keyPhrases)) {
+      queries.push(keyPhrases);
+    }
+  }
+
+  return queries.slice(0, MAX_QUERY_VARIATIONS);
+}
+
 /**
  * Compute relevance score for an entity against task context.
  */
@@ -161,6 +332,7 @@ export function computeRelevanceScore(
   entity: ContextEntity,
   taskContext: string,
   cardLabels: string[],
+  graphRelations?: GraphRelation[],
 ): { score: number; reasons: string[] } {
   const reasons: string[] = [];
   let score = 0;
@@ -255,8 +427,29 @@ export function computeRelevanceScore(
     reasons.push("procedure_boost");
   }
 
+  // 7. Graph walk relation bonus: boost entities discovered via knowledge graph
+  if (graphRelations && graphRelations.length > 0) {
+    const entityRelations = graphRelations.filter(
+      (r) => r.source_id === entity.id || r.target_id === entity.id,
+    );
+    if (entityRelations.length > 0) {
+      // Take the highest relation bonus (don't stack all of them)
+      let bestBonus = 0;
+      let bestRelType = "";
+      for (const rel of entityRelations) {
+        const bonus = RELATION_BONUSES[rel.relation_type] ?? 0.1;
+        if (bonus > bestBonus) {
+          bestBonus = bonus;
+          bestRelType = rel.relation_type;
+        }
+      }
+      score += bestBonus;
+      reasons.push(`graph_walk(${bestRelType})`);
+    }
+  }
+
   // Clamp raw score to 0-1 range before applying tier weight
-  score = Math.min(score, 1.0);
+  score = Math.max(0, Math.min(score, 1.0));
 
   // Apply tier weight
   const tierWeight = TIER_WEIGHTS[entity.memory_tier];
@@ -278,6 +471,10 @@ export async function assembleContext(
     cardLabels = [],
     tokenBudget = DEFAULT_TOKEN_BUDGET,
     client,
+    graphWalkEnabled = true,
+    queryExpansionEnabled = true,
+    enableLlmReranking = false,
+    rerankFn,
   } = options;
 
   const assemblyId = generateAssemblyId();
@@ -295,21 +492,35 @@ export async function assembleContext(
     },
   };
 
-  // Fetch candidate entities: search by task context + list by project
-  let candidates: ContextEntity[] = [];
+  // Fetch candidate entities: search by task context (with query expansion) + list by project
+  const candidates: ContextEntity[] = [];
 
-  try {
-    // Full-text search by task context
-    const searchResult = await client.searchMemoryEntities(
-      workspaceId,
-      taskContext,
-      { project_id: projectId, limit: 30 },
-    );
-    if (searchResult.entities?.length > 0) {
-      candidates = searchResult.entities.map(mapToContextEntity);
+  // P1: Query expansion — search with multiple query variations to catch synonym mismatches
+  const queries = queryExpansionEnabled
+    ? expandQuery(taskContext)
+    : [taskContext];
+
+  const searchResults = await Promise.allSettled(
+    queries.map((query) =>
+      client.searchMemoryEntities(workspaceId, query, {
+        project_id: projectId,
+        limit: 30,
+      }),
+    ),
+  );
+
+  const candidateIds = new Set<string>();
+  for (const result of searchResults) {
+    if (result.status !== "fulfilled") continue;
+    if (result.value.entities?.length > 0) {
+      for (const raw of result.value.entities) {
+        const entity = mapToContextEntity(raw);
+        if (!candidateIds.has(entity.id)) {
+          candidateIds.add(entity.id);
+          candidates.push(entity);
+        }
+      }
     }
-  } catch {
-    // Search failed, fall back to listing
   }
 
   // Also fetch by project scope if we have few candidates
@@ -321,11 +532,13 @@ export async function assembleContext(
         limit: 30,
       });
       if (listResult.entities?.length > 0) {
-        const existingIds = new Set(candidates.map((c) => c.id));
-        const additional = listResult.entities
-          .map(mapToContextEntity)
-          .filter((e) => !existingIds.has(e.id));
-        candidates.push(...additional);
+        for (const raw of listResult.entities) {
+          const entity = mapToContextEntity(raw);
+          if (!candidateIds.has(entity.id)) {
+            candidateIds.add(entity.id);
+            candidates.push(entity);
+          }
+        }
       }
     } catch {
       // List failed, continue with what we have
@@ -342,17 +555,61 @@ export async function assembleContext(
         limit: 20,
       });
       if (wsResult.entities?.length > 0) {
-        const existingIds = new Set(candidates.map((c) => c.id));
-        const additional = wsResult.entities
-          .map(mapToContextEntity)
-          .filter((e) => !existingIds.has(e.id));
-        candidates.push(...additional);
+        for (const raw of wsResult.entities) {
+          const entity = mapToContextEntity(raw);
+          if (!candidateIds.has(entity.id)) {
+            candidateIds.add(entity.id);
+            candidates.push(entity);
+          }
+        }
       }
     } catch {
       // Continue with what we have
     }
   }
 
+  // P0: Graph walk enrichment — discover related entities via knowledge graph
+  let graphRelations: GraphRelation[] = [];
+  if (graphWalkEnabled && candidates.length > 0) {
+    try {
+      // Take top candidates by RRF score (or first N if no RRF scores)
+      const seedCandidates = [...candidates]
+        .sort((a, b) => (b.rrf_score ?? 0) - (a.rrf_score ?? 0))
+        .slice(0, GRAPH_WALK_SEED_COUNT);
+      const seedIds = seedCandidates.map((c) => c.id);
+
+      const walkResult = await discoverRelatedContext(
+        client,
+        seedIds,
+        GRAPH_WALK_MAX_DEPTH,
+        GRAPH_WALK_MAX_ENTITIES,
+        GRAPH_WALK_MIN_CONFIDENCE,
+      );
+
+      graphRelations = walkResult.relations;
+
+      // Add discovered entities to candidate pool (skip those already present)
+      const newEntityIds = walkResult.entities
+        .filter((e) => !candidateIds.has(e.id))
+        .map((e) => e.id);
+
+      if (newEntityIds.length > 0) {
+        // Fetch full entity data in parallel (graph walk only returns summary fields)
+        const fetchResults = await Promise.allSettled(
+          newEntityIds.map((id) => client.getMemoryEntity(id)),
+        );
+        for (const result of fetchResults) {
+          if (result.status !== "fulfilled" || !result.value.entity) continue;
+          const mapped = mapToContextEntity(result.value.entity);
+          candidateIds.add(mapped.id);
+          candidates.push(mapped);
+        }
+      }
+    } catch {
+      // Graph walk failed, continue with search-only candidates
+    }
+  }
+
   if (candidates.length === 0) {
     return {
       context: "",
@@ -361,12 +618,35 @@ export async function assembleContext(
     };
   }
 
-  // Score all candidates
-  const scored = candidates.map((entity) => {
+  // Quality gate: filter out low-value entities before scoring
+  const qualityCandidates = candidates.filter((entity) => {
+    if (passesQualityGate(entity)) return true;
+    manifest.excluded.push({
+      entityId: entity.id,
+      title: entity.title,
+      type: entity.type,
+      tier: entity.memory_tier,
+      relevanceScore: 0,
+      reason: "failed_quality_gate",
+    });
+    return false;
+  });
+
+  if (qualityCandidates.length === 0) {
+    return {
+      context: "",
+      manifest,
+      memories: [],
+    };
+  }
+
+  // Score all candidates (pass graph relations for relation-type bonuses)
+  const scored = qualityCandidates.map((entity) => {
     const { score, reasons } = computeRelevanceScore(
       entity,
       taskContext,
       cardLabels,
+      graphRelations.length > 0 ? graphRelations : undefined,
     );
     return { entity, score, reasons };
   });
@@ -374,6 +654,38 @@ export async function assembleContext(
   // Sort by score descending
   scored.sort((a, b) => b.score - a.score);
 
+  // P2: Optional LLM re-ranking when top scores are clustered
+  if (
+    enableLlmReranking &&
+    rerankFn &&
+    scored.length >= RERANK_MIN_CANDIDATES
+  ) {
+    const topN = scored.slice(0, RERANK_TOP_N);
+    const scoreRange = topN[0].score - topN[topN.length - 1].score;
+    // Only re-rank when scores are tightly clustered
+    if (scoreRange <= RERANK_CLUSTER_THRESHOLD) {
+      try {
+        const rerankCandidates = topN.map((s) => ({
+          id: s.entity.id,
+          title: s.entity.title,
+          snippet: s.entity.content.slice(0, 200),
+        }));
+        const rerankedIds = await rerankFn(taskContext, rerankCandidates);
+        // Reorder based on LLM ranking
+        const idOrder = new Map(rerankedIds.map((id, i) => [id, i]));
+        topN.sort((a, b) => {
+          const aIdx = idOrder.get(a.entity.id) ?? 999;
+          const bIdx = idOrder.get(b.entity.id) ?? 999;
+          return aIdx - bIdx;
+        });
+        // Splice reranked items back in
+        scored.splice(0, topN.length, ...topN);
+      } catch {
+        // Re-ranking failed, continue with static ordering
+      }
+    }
+  }
+
   // Reserve dedicated procedure budget, allocate remaining to tiers
   const procedureBudget = Math.floor(tokenBudget * PROCEDURE_BUDGET_FRACTION);
   const remainingBudget = tokenBudget - procedureBudget;