claude-memory-layer 1.0.26 → 1.0.28

package/src/core/retrieval-benchmark.ts

@@ -0,0 +1,117 @@
+export interface ReplayMetricInput {
+  queryId: string;
+  expectedIds: string[];
+  retrievedIds: string[];
+  /** Optional graded qrels labels. Missing expected ids default to relevance 1. */
+  expectedRelevance?: Record<string, number>;
+}
+
+export interface PrecisionRecallAtK {
+  precision: number;
+  recall: number;
+  hits: number;
+  ndcg: number;
+}
+
+export interface ReplayQueryMetrics {
+  queryId: string;
+  at: Record<number, PrecisionRecallAtK>;
+}
+
+export interface ReplayMetricsSummary {
+  queryCount: number;
+  precisionAtK: Record<number, number>;
+  recallAtK: Record<number, number>;
+  ndcgAtK: Record<number, number>;
+}
+
+export function computePrecisionRecallAtK(
+  inputs: ReplayMetricInput[],
+  ks: number[]
+): ReplayQueryMetrics[] {
+  const normalizedKs = normalizeKs(ks);
+  return inputs.map((input) => {
+    const expected = new Set(input.expectedIds);
+    const relevance = normalizeRelevance(input.expectedIds, input.expectedRelevance);
+    const at: Record<number, PrecisionRecallAtK> = {};
+
+    for (const k of normalizedKs) {
+      const retrieved = input.retrievedIds.slice(0, k);
+      const hits = new Set(retrieved.filter((id) => expected.has(id))).size;
+      at[k] = {
+        precision: k === 0 ? 0 : hits / k,
+        recall: expected.size === 0 ? 0 : hits / expected.size,
+        hits,
+        ndcg: computeNdcgAtK(retrieved, relevance, k)
+      };
+    }
+
+    return { queryId: input.queryId, at };
+  });
+}
+
+export function summarizeReplayMetrics(
+  metrics: ReplayQueryMetrics[],
+  ks: number[]
+): ReplayMetricsSummary {
+  const normalizedKs = normalizeKs(ks);
+  const precisionAtK: Record<number, number> = {};
+  const recallAtK: Record<number, number> = {};
+  const ndcgAtK: Record<number, number> = {};
+
+  for (const k of normalizedKs) {
+    precisionAtK[k] = average(metrics.map((metric) => metric.at[k]?.precision ?? 0));
+    recallAtK[k] = average(metrics.map((metric) => metric.at[k]?.recall ?? 0));
+    ndcgAtK[k] = average(metrics.map((metric) => metric.at[k]?.ndcg ?? 0));
+  }
+
+  return { queryCount: metrics.length, precisionAtK, recallAtK, ndcgAtK };
+}
+
+function normalizeKs(ks: number[]): number[] {
+  const seen = new Set<number>();
+  const normalized: number[] = [];
+
+  for (const rawK of ks) {
+    const k = Math.max(0, Math.floor(rawK));
+    if (seen.has(k)) continue;
+    seen.add(k);
+    normalized.push(k);
+  }
+
+  return normalized.sort((a, b) => a - b);
+}
+
+function normalizeRelevance(expectedIds: string[], expectedRelevance?: Record<string, number>): Record<string, number> {
+  const relevance: Record<string, number> = {};
+  for (const id of expectedIds) {
+    const raw = expectedRelevance?.[id] ?? 1;
+    relevance[id] = Number.isFinite(raw) ? Math.max(0, raw) : 0;
+  }
+  return relevance;
+}
+
+function computeNdcgAtK(retrieved: string[], relevance: Record<string, number>, k: number): number {
+  if (k <= 0) return 0;
+
+  const seen = new Set<string>();
+  const dcg = retrieved.reduce((sum, id, index) => {
+    if (seen.has(id)) return sum;
+    seen.add(id);
+    return sum + discountedGain(relevance[id] ?? 0, index);
+  }, 0);
+
+  const idealRelevance = Object.values(relevance).sort((a, b) => b - a).slice(0, k);
+  const idealDcg = idealRelevance.reduce((sum, rel, index) => sum + discountedGain(rel, index), 0);
+  return idealDcg === 0 ? 0 : dcg / idealDcg;
+}
+
+function discountedGain(relevance: number, zeroBasedRank: number): number {
+  if (relevance <= 0) return 0;
+  return (2 ** relevance - 1) / Math.log2(zeroBasedRank + 2);
+}
+
+function average(values: number[]): number {
+  if (values.length === 0) return 0;
+  return values.reduce((sum, value) => sum + value, 0) / values.length;
+}

package/src/core/retrieval-quality.ts

@@ -0,0 +1,109 @@
+/**
+ * Retrieval quality guards.
+ *
+ * These are deliberately small deterministic heuristics used to avoid injecting
+ * obviously irrelevant memories. They are not a second source of truth; they
+ * only filter candidate retrieval results before context assembly.
+ */
+
+const COMMAND_ARTIFACT_PATTERNS = [
+  /<\/?(?:local-command-(?:stdout|stderr)|command-(?:name|message))\b/i,
+  /<command-name>[\s\S]*?<\/command-name>/i,
+  /<local-command-stdout>[\s\S]*?<\/local-command-stdout>/i,
+  /<local-command-stderr>[\s\S]*?<\/local-command-stderr>/i
+];
+
+const LOW_SIGNAL_CONTEXT_PATTERNS = [
+  /<environment_context\b[\s\S]*<\/environment_context>/i,
+  /<turn_aborted>/i,
+  /^#\s*AGENTS\.md\s+instructions\b[\s\S]*<INSTRUCTIONS>/i,
+  /^\s*(?:understood[,\s.]*)?(?:stopping|stopped|pausing|paused)\s+here\b[\s\S]{0,180}\blet\s+me\s+know\s+when\s+you(?:'d|\s+would)?\s+like\s+to\s+continue\b/i,
+  /^➜\s+\S+\s+git:\([^)]*\)\s+/i,
+  /^\$\s+\S+/i
+];
+
+const CONTINUATION_QUERY_PATTERNS = [
+  /^\s*(?:continue|resume|next|what(?:'s| is)? next|next\s+(?:step|task|action)|recommended\s+(?:next\s+)?(?:step|task|action)|what should (?:we|i) do next)\??\s*$/i,
+  /^\s*(?:이어서(?:\s*진행해줘)?|계속(?:\s*해줘)?|다음\s*(?:단계|작업|추천\s*작업|추천|할\s*일)?(?:은|는)?(?:\s*뭐야)?\??|추천\s*작업(?:은|는)?(?:\s*뭐야)?\??|진행해줘)\s*$/i
+];
+
+const GENERIC_TECHNICAL_TERMS = new Set([
+  'api',
+  'cli',
+  'ui',
+  'json',
+  'jsonl',
+  'html',
+  'http',
+  'https',
+  'url',
+  'uri',
+  'id',
+  'ids',
+  'uuid',
+  'db',
+  'sql'
+]);
+
+export function isCommandArtifactQuery(query: string): boolean {
+  const trimmed = query.trim();
+  if (!trimmed) return false;
+  const normalized = trimmed.toLowerCase();
+  if (normalized.includes('local-command-stdout') || normalized.includes('local-command-stderr')) return true;
+  if (normalized.includes('command-name') || normalized.includes('command-message')) return true;
+  return COMMAND_ARTIFACT_PATTERNS.some((pattern) => pattern.test(trimmed));
+}
+
+export function isCommandArtifactContent(content: string): boolean {
+  const trimmed = content.trim();
+  if (!trimmed) return false;
+  const normalized = trimmed.toLowerCase();
+  if (normalized.includes('local-command-stdout') || normalized.includes('local-command-stderr')) return true;
+  if (normalized.includes('command-name') || normalized.includes('command-message')) return true;
+  return COMMAND_ARTIFACT_PATTERNS.some((pattern) => pattern.test(trimmed));
+}
+
+export function isLowSignalContextContent(content: string): boolean {
+  const trimmed = content.trim();
+  if (!trimmed) return true;
+  if (isCommandArtifactContent(trimmed)) return true;
+  if (LOW_SIGNAL_CONTEXT_PATTERNS.some((pattern) => pattern.test(trimmed))) return true;
+  return false;
+}
+
+export function isGenericContinuationQuery(query: string): boolean {
+  const trimmed = query.trim();
+  if (!trimmed) return false;
+  if (!CONTINUATION_QUERY_PATTERNS.some((pattern) => pattern.test(trimmed))) return false;
+  if (extractTechnicalQueryTerms(trimmed).length > 0) return false;
+
+  const tokens = trimmed.match(/[A-Za-z0-9가-힣#._/-]+/g) ?? [];
+  if (tokens.length > 10) return false;
+
+  return !/[A-Za-z0-9_-]+\.[A-Za-z0-9]+/.test(trimmed) &&
+    !/(?:^|\s)(?:feat|fix|chore|refactor|docs)\/[A-Za-z0-9._-]+/.test(trimmed) &&
+    !/[A-Za-z]:?[\\/]|\/Users\/|\.\/|\.\.\//.test(trimmed);
+}
+
+export function extractTechnicalQueryTerms(query: string): string[] {
+  const matches = query.match(/[A-Za-z][A-Za-z0-9_.:-]{2,}/g) ?? [];
+  const terms = matches.filter((term) => {
+    const lower = term.toLowerCase();
+    if (GENERIC_TECHNICAL_TERMS.has(lower)) return false;
+    return /[._:-]/.test(term) || /[a-z][A-Z]/.test(term) || /[A-Z]{2,}/.test(term) || /\d/.test(term);
+  });
+
+  return Array.from(new Set(terms.map((term) => term.toLowerCase())));
+}
+
+export function hasTechnicalTermOverlap(query: string, content: string): boolean {
+  const terms = extractTechnicalQueryTerms(query);
+  if (terms.length === 0) return true;
+
+  const normalizedContent = content.toLowerCase();
+  return terms.some((term) => normalizedContent.includes(term));
+}
+
+export function shouldApplyTechnicalGuard(query: string): boolean {
+  return extractTechnicalQueryTerms(query).length > 0;
+}

package/src/core/retriever.ts

@@ -10,6 +10,11 @@ import { Matcher } from './matcher.js';
 import { SharedStore } from './shared-store.js';
 import { SharedVectorStore } from './shared-vector-store.js';
 import { GraduationPipeline } from './graduation.js';
+import {
+  hasTechnicalTermOverlap,
+  isCommandArtifactQuery,
+  shouldApplyTechnicalGuard
+} from './retrieval-quality.js';
 import type { MemoryEvent, MatchResult, SharedTroubleshootingEntry } from './types.js';
 
 export interface RetrievalScope {
@@ -23,6 +28,8 @@ export interface RetrievalScope {
 
 export type RetrievalStrategy = 'auto' | 'fast' | 'deep';
 export type ProjectScopeMode = 'strict' | 'prefer' | 'global';
+type DecayPolicy = NonNullable<RetrievalOptions['decayPolicy']>;
+type GraphHopOptions = NonNullable<RetrievalOptions['graphHop']>;
 
 export interface RetrievalOptions {
   topK: number;
@@ -166,6 +173,20 @@ export class Retriever {
     const sessionFilter = opts.scope?.sessionId ?? opts.sessionId;
     const fallbackTrace: string[] = [];
 
+    if (isCommandArtifactQuery(query)) {
+      fallbackTrace.push('guard:command-artifact-query');
+      const emptyMatch = this.matcher.matchSearchResults([], () => 0);
+      return {
+        memories: [],
+        matchResult: emptyMatch,
+        totalTokens: 0,
+        context: '',
+        fallbackTrace,
+        selectedDebug: [],
+        candidateDebug: []
+      };
+    }
+
     const fallbackEnabled = (opts.strategy ?? 'auto') === 'auto';
 
     // Stage 1: primary retrieval
@@ -311,28 +332,21 @@ export class Retriever {
       minScore: number;
       sessionId?: string;
       scope?: RetrievalScope;
-      rerankWithKeyword: boolean;
+      rerankWithKeyword?: boolean;
       rerankWeights?: {
         semantic?: number;
         lexical?: number;
         recency?: number;
       };
-      decayPolicy?: {
-        enabled?: boolean;
-        windowDays?: number;
-        maxPenalty?: number;
-      };
+      decayPolicy?: DecayPolicy;
       intentRewrite?: boolean;
-      graphHop?: {
-        enabled?: boolean;
-        maxHops?: number;
-        hopPenalty?: number;
-      };
+      graphHop?: GraphHopOptions;
       projectScopeMode?: ProjectScopeMode;
       projectHash?: string;
       allowedProjectHashes?: string[];
     }
-  ): Promise<{ results: SearchResult[]; matchResult: MatchResult }> {
+  ): Promise<{ results: SearchResult[]; candidateResults: SearchResult[]; matchResult: MatchResult }> {
+    let rerankQuery = query;
     let initialResults = await this.searchByStrategy(query, {
       strategy: input.strategy,
       topK: input.topK,
@@ -343,6 +357,7 @@ export class Retriever {
     if (input.intentRewrite && input.strategy === 'deep' && this.queryRewriter) {
       const rewritten = (await this.queryRewriter(query))?.trim();
       if (rewritten && rewritten !== query) {
+        rerankQuery = `${query} ${rewritten}`;
         const rewrittenResults = await this.searchByStrategy(rewritten, {
           strategy: 'deep',
           topK: input.topK,
@@ -362,7 +377,7 @@ export class Retriever {
         });
 
     const rerankedResults = input.rerankWithKeyword
-      ? this.rerankByKeywordOverlap(expandedResults, query, input.rerankWeights, input.decayPolicy)
+      ? this.rerankByKeywordOverlap(expandedResults, rerankQuery, input.rerankWeights, input.decayPolicy)
       : expandedResults;
 
     const filtered = await this.applyScopeFilters(rerankedResults, {
@@ -371,10 +386,33 @@ export class Retriever {
       projectHash: input.projectHash,
       allowedProjectHashes: input.allowedProjectHashes
     });
-    const top = filtered.slice(0, input.topK);
+    const qualityFiltered = this.applyQualityFilters(filtered, {
+      query,
+      minScore: input.minScore
+    });
+    const top = qualityFiltered.slice(0, input.topK);
     const matchResult = this.matcher.matchSearchResults(top, () => 0);
 
-    return { results: top, candidateResults: filtered, matchResult };
+    return { results: top, candidateResults: qualityFiltered, matchResult };
+  }
+
+  private applyQualityFilters(
+    results: Array<SearchResult & { semanticScore?: number; lexicalScore?: number; recencyScore?: number }>,
+    options: { query: string; minScore: number }
+  ): Array<SearchResult & { semanticScore?: number; lexicalScore?: number; recencyScore?: number }> {
+    let filtered = [...results];
+
+    if (shouldApplyTechnicalGuard(options.query)) {
+      filtered = filtered.filter((result) => hasTechnicalTermOverlap(options.query, result.content));
+    }
+
+    if (filtered.length <= 2) return filtered;
+
+    const topScore = filtered[0].score;
+    if (topScore < 0.8) return filtered;
+
+    const cliffThreshold = Math.max(options.minScore, topScore - 0.25);
+    return filtered.filter((result) => result.score >= cliffThreshold);
   }
 
   private mergeResults(primary: SearchResult[], secondary: SearchResult[], limit: number): SearchResult[] {