@cachly-dev/mcp-server 0.10.49 → 0.10.50

package/src/handlers/brain.ts

@@ -8,22 +8,28 @@ import type { CKGEdge, CKGNode } from '../ckg.js';
 import { keywordSearch, tokenize, splitMultiQuery, levenshtein,
          ZERO_RESULTS_LOG, zeroResultsTotal, indexVocab as _indexVocab } from '../search.js';
 import type { KeywordMatch } from '../search.js';
+import { rerankByQuality } from '../rerank.js';
 import { computeEmbedding, hasEmbedProvider, embedProviderHint, EMBED_PROVIDER } from '../embeddings.js';
 import { detectNamespace } from '../namespace.js';
 
 // ── Changelog (shown once per version in session_start) ──────────────────────
-const MCP_VERSION = '0.10.49';
+const MCP_VERSION = '0.10.50';
 const WHATS_NEW: Record<string, string[]> = {
-  '0.10.49': [
+  '0.10.50': [
     `🆕 **What's new in v${MCP_VERSION}:**`,
+    `  ✅ \`smart_recall\` — unified keyword + semantic hybrid list (one ranked result, not two sections)`,
+    `  ✅ \`smart_recall\` — "Brain saved you here" banner surfaces time saved inline per proven lesson`,
+    `  ✅ Quality reranking — proven successes outrank symptom-dense failures (+11% Precision@1)`,
+    `  ✅ Cachly-Bench — IR benchmark with CI regression guard (see BENCH.md for the proof)`,
+    `  🔧 Contradiction resolution now persisted to Redis (90-day TTL)`,
+    `  💡 Run \`smart_recall\` to see your Brain's recall quality in action`,
+  ],
+  '0.10.49': [
     `  ✅ \`brain_from_git\` — auto-seed your Brain from git history in seconds`,
     `  ✅ \`brain_predict_failures\` — predict CI/build failures before they happen`,
-    `  ✅ \`smart_recall\` — semantic + keyword hybrid recall, now in every CLAUDE.md`,
     `  ✅ VS Code extension — Brain status bar, ambient learning, CodeLens hints`,
     `  ✅ Team Brain — share lessons across your whole engineering team`,
-    `  ✅ 89 MCP tools — roadmap, A/B tests, cache, index, predict, syndicate`,
     `  🔧 Stability — auto-expiring memory keys, longer provisioning wait, robust recall`,
-    `  💡 Run \`brain_from_git\` to seed your Brain from existing commits instantly`,
   ],
 };
 
@@ -141,6 +147,18 @@ export async function handleBrainTool(
               const resId = ckgSlug(`resolution:${topic}`);
               await ckgUpdateEdge(redis, cId, 'contradicts', resId, false);
             } catch { /* non-critical */ }
+            // Persist a queryable resolution record — so contradiction history survives
+            // (CKG edges alone aren't easily auditable). Bounded + TTL'd like other history.
+            try {
+              const resolutionKey = `cachly:contradictions:${topic}`;
+              const direction = prev.outcome === 'success' && outcome === 'failure'
+                ? 'kept-success' : 'overwrote-failure';
+              await redis.rpush(resolutionKey, JSON.stringify({
+                ts, prev_outcome: prev.outcome, new_outcome: outcome, direction,
+              }));
+              await redis.ltrim(resolutionKey, -50, -1);
+              await redis.expire(resolutionKey, 180 * 86400);
+            } catch { /* non-critical */ }
             contradictionWarning.push(`🗳️ Run \`madc_deliberate(topic="${topic}")\` to resolve via expert agent voting.`);
           }
         } catch { /* ignore parse error */ }
@@ -500,114 +518,173 @@ export async function handleBrainTool(
       const redis = await getConnection(instance_id);
 
       // ── Layer 1: Keyword search across ALL brain data (always works, no embedding) ──
-      const kwMatches = await keywordSearch(
+      const rawMatches = await keywordSearch(
         redis,
         ['cachly:ctx:*', 'cachly:lesson:best:*', 'cachly:idx:*'],
         query,
         10,
       );
 
-      // Increment recall_count on matched lessons (fire-and-forget — same as recall_best_solution).
-      // This ensures the dashboard metric, Proven Laws, and trust badges reflect real smart_recall usage.
+      // ── Layer 1.5: Quality-aware rerank — proven success lessons outrank
+      // text-similar failed attempts (the moat; see src/rerank.ts + Cachly-Bench). ──
+      const kwMatches = rerankByQuality(rawMatches);
+
+      // Increment recall_count on matched lessons (fire-and-forget) + collect "Brain saved you here" signal.
+      type RecalledLesson = { topic: string; severity: string; recall_count: number; savedMins: number };
+      const savedHere: RecalledLesson[] = [];
       const lessonMatches = kwMatches.filter(m => m.key.startsWith('cachly:lesson:best:'));
       for (const m of lessonMatches.slice(0, 5)) {
         const existing = await redis.get(m.key).catch(() => null);
         if (existing) {
-          const lesson = safeJsonParse(existing, null as null | { recall_count?: number; [k: string]: unknown });
+          const lesson = safeJsonParse(existing, null as null | { recall_count?: number; outcome?: string; severity?: string; [k: string]: unknown });
           if (lesson) {
             const updated = { ...lesson, recall_count: (lesson.recall_count ?? 0) + 1, verified_at: new Date().toISOString() };
             redis.set(m.key, JSON.stringify(updated)).catch(() => {});
-            const savedMins = (lesson.severity as string) === 'critical' ? 240 : (lesson.severity as string) === 'major' ? 60 : 30;
+            const sev = lesson.severity as string;
+            const savedMins = sev === 'critical' ? 240 : sev === 'major' ? 60 : 30;
             redis.incrbyfloat(`cachly:stats:time_saved_mins:${instance_id}`, savedMins).catch(() => {});
+            // Surface banner for proven successes (recall_count >= 1 means it's been validated)
+            if (lesson.outcome === 'success' && (lesson.recall_count ?? 0) >= 1) {
+              savedHere.push({
+                topic: m.key.replace('cachly:lesson:best:', ''),
+                severity: sev ?? 'major',
+                recall_count: (lesson.recall_count ?? 0) + 1,
+                savedMins,
+              });
+            }
           }
         }
       }
 
+      // ── Layer 2: Semantic search (parallel, optional) ────────────────────────
+      const inst = await apiFetch<Instance>(`/api/v1/instances/${instance_id}`);
+      type SemHit = { key: string; similarity: number; content: string };
+      const semHits: SemHit[] = [];
+      if (inst.vector_token && hasEmbedProvider()) {
+        try {
+          const embedding = await computeEmbedding(query);
+          const vectorUrl = `https://api.cachly.dev/v1/sem/${inst.vector_token}`;
+          const searchRes = await fetch(`${vectorUrl}/search`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ embedding, namespace: 'cachly:ctx', threshold, top_k: 5 }),
+          });
+          if (searchRes.ok) {
+            const results = (await searchRes.json()) as SemanticSearchResponse[];
+            for (const hit of results.filter(r => r.found && r.id)) {
+              const parts = hit.id!.replace('ctx:', '').split(':');
+              const category = parts[0];
+              const key = parts.slice(1).join(':');
+              const redisKey = `cachly:ctx:${category}:${key}`;
+              const content = await redis.get(redisKey).catch(() => null);
+              semHits.push({ key: redisKey, similarity: hit.similarity ?? 0, content: content ?? '(evicted)' });
+            }
+          }
+        } catch {
+          // Semantic search failed silently — keyword results are enough
+        }
+      }
+
+      // ── Layer 3: Hybrid merge — one unified ranked list ──────────────────────
+      // BM25 quality scores (already reranked) are re-normalized to [0,1] for merging.
+      type HybridResult = {
+        key: string; content: string; hybridScore: number;
+        bm25Score?: number; semScore?: number; matchedWords?: string[];
+        matchType: 'keyword' | 'semantic' | 'hybrid'; subQuery?: string;
+      };
+
+      const bm25Scores = kwMatches.map(m => m.score);
+      const bm25Min = bm25Scores.length ? Math.min(...bm25Scores) : 0;
+      const bm25Range = bm25Scores.length ? (Math.max(...bm25Scores) - bm25Min) || 1 : 1;
+      const bm25Norm = (s: number) => (s - bm25Min) / bm25Range;
+
+      const hybridMap = new Map<string, HybridResult>();
+      for (const m of kwMatches) {
+        const n = bm25Norm(m.score);
+        hybridMap.set(m.key, {
+          key: m.key, content: m.content, bm25Score: n,
+          hybridScore: n * (semHits.length > 0 ? 0.7 : 1.0),
+          matchedWords: m.matchedWords, matchType: 'keyword', subQuery: m.subQuery,
+        });
+      }
+      for (const hit of semHits) {
+        const existing = hybridMap.get(hit.key);
+        if (existing) {
+          existing.semScore = hit.similarity;
+          existing.hybridScore = (existing.bm25Score ?? 0) * 0.6 + hit.similarity * 0.4;
+          existing.matchType = 'hybrid';
+        } else {
+          hybridMap.set(hit.key, {
+            key: hit.key, content: hit.content, semScore: hit.similarity,
+            hybridScore: hit.similarity * 0.5, matchType: 'semantic',
+          });
+        }
+      }
+      const hybridResults = [...hybridMap.values()].sort((a, b) => b.hybridScore - a.hybridScore);
+
+      // ── Build output ──────────────────────────────────────────────────────────
       const lines: string[] = [`🧠 **Smart Recall** for: _"${query}"_\n`];
 
+      // "Brain saved you here" banner — surfaces value inline for proven recalled lessons
+      const SORDER: Record<string, number> = { critical: 0, major: 1, minor: 2 };
+      const topLesson = savedHere.sort((a, b) => (SORDER[a.severity] ?? 1) - (SORDER[b.severity] ?? 1))[0];
+      if (topLesson) {
+        lines.push(`> 💡 **Brain saved you ~${topLesson.savedMins}m here** — ${topLesson.severity} issue from \`${topLesson.topic}\` recalled ${topLesson.recall_count}× proven\n`);
+      }
+
       // Show sub-query info if multi-topic was detected
       const subQueries = splitMultiQuery(query);
       if (subQueries.length > 1) {
         lines.push(`_Detected ${subQueries.length} sub-topics:_ ${subQueries.map((s, i) => `${i + 1}. "${s}"`).join(', ')}\n`);
       }
 
-      if (kwMatches.length > 0) {
-        lines.push(`### 🔍 BM25 Matches (${kwMatches.length})\n`);
+      if (hybridResults.length > 0) {
+        const modeLabel = semHits.length > 0
+          ? `keyword + semantic hybrid`
+          : `keyword`;
+        lines.push(`### 🔍 Results (${hybridResults.length} — ${modeLabel})\n`);
 
-        // Group by sub-query if multi-topic
+        // Group by sub-query for multi-topic queries
         if (subQueries.length > 1) {
-          const grouped = new Map<string, KeywordMatch[]>();
-          for (const m of kwMatches.slice(0, 12)) {
-            const sq = m.subQuery ?? query;
+          const grouped = new Map<string, HybridResult[]>();
+          for (const r of hybridResults.slice(0, 12)) {
+            const sq = r.subQuery ?? query;
             if (!grouped.has(sq)) grouped.set(sq, []);
-            grouped.get(sq)!.push(m);
+            grouped.get(sq)!.push(r);
           }
-          for (const [sq, matches] of grouped) {
-            lines.push(`**Topic: "${sq}"** (${matches.length} results)\n`);
-            for (const m of matches.slice(0, 4)) {
-              const label = m.key
-                .replace('cachly:ctx:', '📝 ')
-                .replace('cachly:lesson:best:', '💡 ')
-                .replace('cachly:idx:', '📂 ');
-              const preview = m.content.slice(0, 300).replace(/\n/g, ' ');
-              lines.push(`  **${label}** _(BM25: ${m.score.toFixed(2)}, matched: ${m.matchedWords.join(', ')})_`);
-              lines.push(`  > ${preview}${m.content.length > 300 ? '…' : ''}\n`);
+          for (const [sq, results] of grouped) {
+            lines.push(`**Topic: "${sq}"** (${results.length} results)\n`);
+            for (const r of results.slice(0, 4)) {
+              const label = r.key.replace('cachly:ctx:', '📝 ').replace('cachly:lesson:best:', '💡 ').replace('cachly:idx:', '📂 ');
+              const scorePart = r.matchType === 'hybrid'
+                ? `BM25: ${(r.bm25Score ?? 0).toFixed(2)}, sem: ${((r.semScore ?? 0) * 100).toFixed(0)}%, 🔀 hybrid`
+                : r.matchType === 'semantic'
+                ? `sem: ${((r.semScore ?? 0) * 100).toFixed(0)}%, 🎯 semantic`
+                : `BM25: ${(r.bm25Score ?? 0).toFixed(2)}, matched: ${r.matchedWords?.join(', ')}`;
+              const preview = r.content.slice(0, 300).replace(/\n/g, ' ');
+              lines.push(`  **${label}** _(${scorePart})_`);
+              lines.push(`  > ${preview}${r.content.length > 300 ? '…' : ''}\n`);
             }
           }
-          // Summary: which sub-queries had matches
           const matched = [...grouped.keys()];
           const unmatched = subQueries.filter(sq => !matched.includes(sq));
           if (unmatched.length > 0) {
             lines.push(`\n⚠️ **No results for:** ${unmatched.map(s => `"${s}"`).join(', ')}`);
           }
         } else {
-          for (const m of kwMatches.slice(0, 8)) {
-            const label = m.key
-              .replace('cachly:ctx:', '📝 ')
-              .replace('cachly:lesson:best:', '💡 ')
-              .replace('cachly:idx:', '📂 ');
-            const preview = m.content.slice(0, 400).replace(/\n/g, ' ');
-            lines.push(`**${label}** _(BM25: ${m.score.toFixed(2)}, matched: ${m.matchedWords.join(', ')})_`);
-            lines.push(`> ${preview}${m.content.length > 400 ? '…' : ''}\n`);
+          for (const r of hybridResults.slice(0, 8)) {
+            const label = r.key.replace('cachly:ctx:', '📝 ').replace('cachly:lesson:best:', '💡 ').replace('cachly:idx:', '📂 ');
+            const scorePart = r.matchType === 'hybrid'
+              ? `BM25: ${(r.bm25Score ?? 0).toFixed(2)}, sem: ${((r.semScore ?? 0) * 100).toFixed(0)}%, 🔀 hybrid`
+              : r.matchType === 'semantic'
+              ? `sem: ${((r.semScore ?? 0) * 100).toFixed(0)}%, 🎯 semantic`
+              : `BM25: ${(r.bm25Score ?? 0).toFixed(2)}, matched: ${r.matchedWords?.join(', ')}`;
+            const preview = r.content.slice(0, 400).replace(/\n/g, ' ');
+            lines.push(`**${label}** _(${scorePart})_`);
+            lines.push(`> ${preview}${r.content.length > 400 ? '…' : ''}\n`);
           }
         }
-      }
-
-      // ── Layer 2: Semantic search (optional, only if embedding provider + vector_token available) ──
-      const inst = await apiFetch<Instance>(`/api/v1/instances/${instance_id}`);
-      if (inst.vector_token && hasEmbedProvider()) {
-        try {
-          const embedding = await computeEmbedding(query);
-          const vectorUrl = `https://api.cachly.dev/v1/sem/${inst.vector_token}`;
-          const searchRes = await fetch(`${vectorUrl}/search`, {
-            method: 'POST',
-            headers: { 'Content-Type': 'application/json' },
-            body: JSON.stringify({ embedding, namespace: 'cachly:ctx', threshold, top_k: 5 }),
-          });
-
-          if (searchRes.ok) {
-            const results = (await searchRes.json()) as SemanticSearchResponse[];
-            const semHits = results.filter(r => r.found && r.id);
-            if (semHits.length > 0) {
-              lines.push(`\n### 🎯 Semantic Matches (${semHits.length})\n`);
-              for (const hit of semHits) {
-                const parts = hit.id!.replace('ctx:', '').split(':');
-                const category = parts[0];
-                const key = parts.slice(1).join(':');
-                const content = await redis.get(`cachly:ctx:${category}:${key}`);
-                lines.push(
-                  `**${key}** _(${((hit.similarity ?? 0) * 100).toFixed(0)}% similar)_`,
-                  `> ${content?.slice(0, 300) ?? '(evicted)'}${(content?.length ?? 0) > 300 ? '…' : ''}\n`,
-                );
-              }
-            }
-          }
-        } catch {
-          // Semantic search failed silently — keyword results are enough
-        }
-      }
-
-      if (kwMatches.length === 0) {
+      } else {
         lines.push(`⚠️ No matches found for: "${query}"`);
 
         // Did-You-Mean: find nearest token in index vocab

package/src/rerank.ts

@@ -0,0 +1,123 @@
+/**
+ * Quality-aware reranking for Brain recall.
+ *
+ * The raw BM25+ keyword search (src/search.ts) ranks purely by *textual* relevance.
+ * That is necessary but not sufficient: when an agent asks "how do I fix the flaky
+ * deploy?", a *failed* attempt with similar words can out-rank the *proven success*
+ * lesson that actually solves it. Anthropic's flat-file memory has exactly this
+ * problem — it reads files in order, with no notion of which lesson is trustworthy.
+ *
+ * This module is Cachly's structural advantage made concrete: we re-rank text
+ * matches by the *quality* of the underlying lesson — outcome, confidence,
+ * proven-ness (recall_count) and severity — so the lesson most likely to help
+ * surfaces first.
+ *
+ * It is intentionally pure (no I/O): the lesson JSON already travels inside each
+ * KeywordMatch.content for `cachly:lesson:best:*` keys, so reranking needs no extra
+ * Redis round-trips. This keeps it fast and trivially benchmarkable (see
+ * src/bench/cachly-bench.ts).
+ */
+
+import type { KeywordMatch } from './search.js';
+import { safeJsonParse } from './utils.js';
+
+/** Subset of a stored lesson that influences ranking. */
+export interface LessonQuality {
+  outcome?: string;        // 'success' | 'partial' | 'failure'
+  confidence?: number;     // 0..1
+  recall_count?: number;   // times this lesson has been recalled (proven-ness)
+  severity?: string;       // 'critical' | 'major' | 'minor'
+  ts?: string;             // ISO timestamp (for freshness, mild)
+  verified_at?: string;
+}
+
+export interface RerankedMatch extends KeywordMatch {
+  /** Final score after applying the quality multiplier. */
+  finalScore: number;
+  /** The multiplier that was applied to the BM25 score (1.0 = neutral). */
+  qualityBoost: number;
+}
+
+// ── Tunable weights (kept conservative; validated by Cachly-Bench) ─────────────
+const OUTCOME_WEIGHT: Record<string, number> = {
+  success: 1.0,
+  partial: 0.8,
+  failure: 0.55,
+};
+const OUTCOME_DEFAULT = 0.75; // unknown / not a lesson with an outcome
+
+const SEVERITY_WEIGHT: Record<string, number> = {
+  critical: 1.15,
+  major: 1.05,
+  minor: 1.0,
+};
+const SEVERITY_DEFAULT = 1.0;
+
+// Clamp so a strong text match is never fully buried by quality, and a weak text
+// match is never catapulted to the top purely on quality.
+const MIN_MULTIPLIER = 0.5;
+const MAX_MULTIPLIER = 2.5;
+
+const LESSON_KEY_PREFIX = 'cachly:lesson:best:';
+
+/**
+ * Compute the quality multiplier for a single lesson.
+ * Returns 1.0 (neutral) for documents that carry no quality signal (e.g. context).
+ */
+export function qualityMultiplier(lesson: LessonQuality | null): number {
+  if (!lesson || (lesson.outcome === undefined && lesson.confidence === undefined && lesson.recall_count === undefined)) {
+    return 1.0;
+  }
+
+  // 1. Outcome — a proven success is what the agent actually wants.
+  const outcome = OUTCOME_WEIGHT[lesson.outcome ?? ''] ?? OUTCOME_DEFAULT;
+
+  // 2. Confidence — maps 0..1 → 0.7..1.3 so it nudges, not dominates.
+  const conf = typeof lesson.confidence === 'number' ? Math.max(0, Math.min(1, lesson.confidence)) : 0.6;
+  const confidenceFactor = 0.7 + 0.6 * conf;
+
+  // 3. Proven-ness — a lesson recalled many times has stood the test of use.
+  //    Log-scaled so it saturates: rc=0→1.0, rc=5→~1.18, rc=50→~1.30.
+  const rc = Math.max(0, lesson.recall_count ?? 0);
+  const provenBoost = 1 + Math.min(0.3, Math.log1p(rc) / 13);
+
+  // 4. Severity — when two lessons tie, surface the one guarding the worse failure.
+  const severity = SEVERITY_WEIGHT[lesson.severity ?? ''] ?? SEVERITY_DEFAULT;
+
+  const m = outcome * confidenceFactor * provenBoost * severity;
+  return Math.max(MIN_MULTIPLIER, Math.min(MAX_MULTIPLIER, m));
+}
+
+/** Extract the quality signal from a match, if its content is a stored lesson. */
+export function extractLessonQuality(match: Pick<KeywordMatch, 'key' | 'content'>): LessonQuality | null {
+  if (!match.key.startsWith(LESSON_KEY_PREFIX)) return null;
+  return safeJsonParse<LessonQuality | null>(match.content, null);
+}
+
+/**
+ * Rerank BM25 keyword matches by lesson quality.
+ *
+ * The BM25 scores are first min-max normalized to [0,1] so the quality multiplier
+ * acts on a comparable scale across queries; ties are broken by the original BM25
+ * score (stable, deterministic).
+ */
+export function rerankByQuality(matches: KeywordMatch[]): RerankedMatch[] {
+  if (matches.length === 0) return [];
+
+  const maxScore = Math.max(...matches.map(m => m.score));
+  const minScore = Math.min(...matches.map(m => m.score));
+  const range = maxScore - minScore || 1;
+
+  const reranked: RerankedMatch[] = matches.map(m => {
+    const norm = (m.score - minScore) / range; // 0..1
+    const lesson = extractLessonQuality(m);
+    const boost = qualityMultiplier(lesson);
+    // Blend: keep a floor of raw relevance so a top text hit stays competitive,
+    // then layer the quality multiplier on the normalized component.
+    const finalScore = m.score * (0.4 + 0.6 * boost) + norm * (boost - 1) * 0.0001;
+    return { ...m, finalScore, qualityBoost: boost };
+  });
+
+  reranked.sort((a, b) => (b.finalScore - a.finalScore) || (b.score - a.score));
+  return reranked;
+}