@cachly-dev/mcp-server 0.2.0 → 0.5.2

package/src/index.ts

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 import { randomUUID } from 'node:crypto';
+import { createHash } from 'node:crypto';
 import { readdir, readFile, stat } from 'node:fs/promises';
 import { join, relative, extname } from 'node:path';
 /**
@@ -24,7 +25,7 @@ import { join, relative, extname } from 'node:path';
  *   • cache_keys            – list keys matching a glob pattern
  *   • cache_stats           – memory, hit rate, ops/sec, keyspace info
  *   • semantic_search       – find semantically similar cached entries
- *                             (requires OPENAI_API_KEY + CACHLY_VECTOR_URL)
+ *                             (needs OPENAI_API_KEY or other embed provider in .env)
  *
  * ── Auth & Status ────────────────────────────────────────────────────────────
  *   • get_api_status        – check API health + JWT auth info (Keycloak)
@@ -32,8 +33,8 @@ import { join, relative, extname } from 'node:path';
  * Configuration (env vars):
  *   CACHLY_API_URL      – default https://api.cachly.dev
  *   CACHLY_JWT          – your JWT (Keycloak access token)
- *   OPENAI_API_KEY      – for semantic_search (optional)
- *   CACHLY_EMBED_MODEL  – embedding model (default: text-embedding-3-small)
+ *   CACHLY_EMBED_PROVIDER – embedding backend: openai (default), gemini, mistral, cohere, ollama, cachly (server fallback)
+ *   CACHLY_EMBED_MODEL  – override embedding model (optional)
  */
 
 import { Server } from '@modelcontextprotocol/sdk/server/index.js';
@@ -45,33 +46,98 @@ import {
   McpError,
 } from '@modelcontextprotocol/sdk/types.js';
 import { Redis } from 'ioredis';
-import OpenAI from 'openai';
 
 // ── Config ────────────────────────────────────────────────────────────────────
 
 const API_URL = process.env.CACHLY_API_URL ?? 'https://api.cachly.dev';
 const JWT = process.env.CACHLY_JWT ?? '';
-const EMBED_MODEL = process.env.CACHLY_EMBED_MODEL ?? 'text-embedding-3-small';
-/** Embedding provider: openai (default) | mistral | cohere | ollama | gemini */
-const EMBED_PROVIDER = (process.env.CACHLY_EMBED_PROVIDER ?? 'openai').toLowerCase();
+const EMBED_MODEL = process.env.CACHLY_EMBED_MODEL ?? '';
+/**
+ * ┌──────────────────────────────────────────────────────────────────────┐
+ * │  EMBEDDING PROVIDER — pluggable, client-side first                  │
+ * │                                                                      │
+ * │  Auto-detects from env vars. To force a provider, set:              │
+ * │    CACHLY_EMBED_PROVIDER=openai   (+ OPENAI_API_KEY)                │
+ * │    CACHLY_EMBED_PROVIDER=gemini   (+ GEMINI_API_KEY)                │
+ * │    CACHLY_EMBED_PROVIDER=mistral  (+ MISTRAL_API_KEY)               │
+ * │    CACHLY_EMBED_PROVIDER=cohere   (+ COHERE_API_KEY)                │
+ * │    CACHLY_EMBED_PROVIDER=ollama   (+ OLLAMA_BASE_URL, local)        │
+ * │    CACHLY_EMBED_PROVIDER=cachly   (server-side fallback, no key)    │
+ * │                                                                      │
+ * │  Priority: openai > gemini > mistral > cohere > ollama > cachly     │
+ * │  Brain works WITHOUT embedding (keyword search + exact key lookup).│
+ * │  Embedding is an optional boost for semantic_search/index_project.  │
+ * └──────────────────────────────────────────────────────────────────────┘
+ */
+const EMBED_PROVIDER = (process.env.CACHLY_EMBED_PROVIDER ?? detectEmbedProvider()).toLowerCase();
+
+/** Fast content hash for index invalidation (not cryptographic, just change detection) */
+function simpleHash(text: string): string {
+  return createHash('md5').update(text).digest('hex').slice(0, 12);
+}
+
+function detectEmbedProvider(): string {
+  // Client-side keys first (direct, fast, no server roundtrip)
+  if (process.env.OPENAI_API_KEY) return 'openai';
+  if (process.env.GEMINI_API_KEY) return 'gemini';
+  if (process.env.MISTRAL_API_KEY) return 'mistral';
+  if (process.env.COHERE_API_KEY) return 'cohere';
+  if (process.env.OLLAMA_BASE_URL) return 'ollama';
+  // Server-side fallback — no API key needed, but adds latency
+  if (process.env.CACHLY_JWT) return 'cachly';
+  return 'none'; // no provider → embedding disabled, brain still works via exact keys
+}
 
 // ── Multi-provider embedding ─────────────────────────────────────────────────
 
 /**
  * Compute an embedding vector for `text` using the configured provider.
  *
- * Supported via CACHLY_EMBED_PROVIDER:
- *   openai  (default) – OPENAI_API_KEY  · model: text-embedding-3-small
- *   mistral           – MISTRAL_API_KEY · model: mistral-embed
- *   cohere            – COHERE_API_KEY  · model: embed-english-v3.0
- *   ollama            – OLLAMA_BASE_URL (default: http://localhost:11434) · model: nomic-embed-text
- *   gemini            – GEMINI_API_KEY  · model: text-embedding-004
+ * Client-side (recommended — set one API key in your .env):
+ *   openai   – OPENAI_API_KEY  · text-embedding-3-small
+ *   gemini   – GEMINI_API_KEY  · text-embedding-004
+ *   mistral  – MISTRAL_API_KEY · mistral-embed
+ *   cohere   – COHERE_API_KEY  · embed-english-v3.0
+ *   ollama   – OLLAMA_BASE_URL · nomic-embed-text (local, free)
+ *
+ * Server-side fallback (no key needed on client):
+ *   cachly   – POST /api/v1/embed (requires CACHLY_JWT)
  *
- * Note: Anthropic (Claude) does not provide embedding models.
- *       Use Cohere or OpenAI instead.
+ * Note: Brain works fully WITHOUT embedding (keyword search + exact keys).
+ *       Embedding is an OPTIONAL boost for semantic_search and index_project.
  */
 async function computeEmbedding(text: string): Promise<number[]> {
   switch (EMBED_PROVIDER) {
+    case 'cachly': {
+      // Server-side embedding — the Cachly API computes the embedding
+      // using whatever provider is configured on the server. No client-side API key needed.
+      if (!JWT) throw new Error(
+        'CACHLY_JWT not set.\n\n' +
+        'The "cachly" provider uses server-side embeddings via the Cachly API.\n' +
+        'Set CACHLY_JWT, or use another provider via CACHLY_EMBED_PROVIDER:\n' +
+        '  openai  → OPENAI_API_KEY\n' +
+        '  gemini  → GEMINI_API_KEY\n' +
+        '  ollama  → OLLAMA_BASE_URL (local, no key needed)'
+      );
+      const url = `${API_URL}/api/v1/embed`;
+      const body: Record<string, string> = { text };
+      if (EMBED_MODEL) body.model = EMBED_MODEL;
+      const res = await fetch(url, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Authorization': `Bearer ${JWT}`,
+        },
+        body: JSON.stringify(body),
+      });
+      if (!res.ok) {
+        const errBody = await res.text();
+        throw new Error(`Cachly embed API error ${res.status}: ${errBody}`);
+      }
+      const json = (await res.json()) as { embedding: number[]; dimensions: number };
+      return json.embedding;
+    }
+
     case 'mistral': {
       const key = process.env.MISTRAL_API_KEY;
       if (!key) throw new Error('MISTRAL_API_KEY not set');
@@ -130,20 +196,31 @@ async function computeEmbedding(text: string): Promise<number[]> {
       return json.embedding.values;
     }
 
-    case 'openai':
-    default: {
+    case 'openai': {
       const key = process.env.OPENAI_API_KEY;
       if (!key) throw new Error(
         'OPENAI_API_KEY not set.\n\n' +
-        'Other supported providers via CACHLY_EMBED_PROVIDER:\n' +
-        '  mistral  → MISTRAL_API_KEY\n' +
-        '  cohere   → COHERE_API_KEY\n' +
-        '  ollama   → OLLAMA_BASE_URL (local, no key needed)\n' +
-        '  gemini   → GEMINI_API_KEY'
+        'Set OPENAI_API_KEY in your .env, or switch provider:\n' +
+        '  CACHLY_EMBED_PROVIDER=gemini  (+ GEMINI_API_KEY)\n' +
+        '  CACHLY_EMBED_PROVIDER=ollama  (local, free)\n' +
+        '  CACHLY_EMBED_PROVIDER=cachly  (server-side, no key needed)'
+      );
+      const model = EMBED_MODEL || 'text-embedding-3-small';
+      const res = await fetch('https://api.openai.com/v1/embeddings', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },
+        body: JSON.stringify({ model, input: text }),
+      });
+      if (!res.ok) throw new Error(`OpenAI embedding error: ${res.statusText}`);
+      const json = (await res.json()) as { data: { embedding: number[] }[] };
+      return json.data[0].embedding;
+    }
+
+    default: {
+      throw new Error(
+        `Unknown CACHLY_EMBED_PROVIDER="${EMBED_PROVIDER}".\n` +
+        'Supported: cachly (default), openai, mistral, cohere, ollama, gemini'
       );
-      const openai = new OpenAI({ apiKey: key });
-      const embRes = await openai.embeddings.create({ model: EMBED_MODEL, input: text });
-      return embRes.data[0].embedding;
     }
   }
 }
@@ -151,28 +228,445 @@ async function computeEmbedding(text: string): Promise<number[]> {
 /** Returns true if the configured embedding provider has its required key/URL set. */
 function hasEmbedProvider(): boolean {
   switch (EMBED_PROVIDER) {
+    case 'cachly':  return !!JWT;
     case 'mistral': return !!process.env.MISTRAL_API_KEY;
     case 'cohere':  return !!process.env.COHERE_API_KEY;
     case 'ollama':  return true; // OLLAMA_BASE_URL is optional (defaults to localhost)
     case 'gemini':  return !!process.env.GEMINI_API_KEY;
-    case 'openai':
-    default:        return !!process.env.OPENAI_API_KEY;
+    case 'openai':  return !!process.env.OPENAI_API_KEY;
+    case 'none':    return false;
+    default:        return false;
   }
 }
 
 /** Returns a human-readable description of the current embed provider/model for error messages. */
 function embedProviderHint(): string {
   const providerKeys: Record<string, string> = {
+    cachly: 'CACHLY_JWT (server-side, no extra key needed)',
     openai: 'OPENAI_API_KEY',
     mistral: 'MISTRAL_API_KEY',
     cohere: 'COHERE_API_KEY',
     ollama: 'OLLAMA_BASE_URL (optional, default: http://localhost:11434)',
     gemini: 'GEMINI_API_KEY',
   };
-  const key = providerKeys[EMBED_PROVIDER] ?? 'OPENAI_API_KEY';
+  if (EMBED_PROVIDER === 'none') {
+    return 'No embedding provider configured. Set OPENAI_API_KEY, GEMINI_API_KEY, or another provider key in .env. Brain works without embedding.';
+  }
+  const key = providerKeys[EMBED_PROVIDER] ?? 'CACHLY_JWT';
   return `CACHLY_EMBED_PROVIDER=${EMBED_PROVIDER} → requires ${key}`;
 }
 
+// ── Search Engine (BM25+ with enhancements, works without embedding) ──────────
+//
+// Features beyond standard BM25:
+//   • BM25+ (delta=1) — fixes BM25's "long document penalty" bug
+//   • Bigram proximity boost — adjacent query terms in doc score 2× more
+//   • Recency boost — newer entries rank higher (exponential decay, 7-day half-life)
+//   • Multi-query splitting — numbered lists, semicolons, conjunctions
+//   • Fuzzy matching with Levenshtein distance ≤ 2
+//   • Multilingual stopwords (EN, DE, FR, ES, IT, PT)
+//   • Pipeline Redis reads for performance
+//
+
+/**
+ * Stopwords — filtered out during tokenization.
+ * Covers: English, German, French, Spanish, Italian, Portuguese.
+ * Keeps the index small and scores meaningful.
+ */
+const STOPWORDS = new Set([
+  // ── English ──
+  'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+  'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing',
+  'will', 'would', 'could', 'should', 'may', 'might', 'shall', 'can',
+  'to', 'of', 'in', 'for', 'on', 'with', 'at', 'by', 'from', 'as',
+  'into', 'through', 'during', 'before', 'after', 'above', 'below',
+  'between', 'about', 'against', 'among', 'around', 'without', 'within',
+  'along', 'across', 'behind', 'beyond', 'upon', 'toward', 'towards',
+  'and', 'but', 'or', 'not', 'no', 'nor', 'so', 'if', 'then', 'than',
+  'too', 'very', 'quite', 'rather', 'just', 'also', 'only', 'even',
+  'it', 'its', 'this', 'that', 'these', 'those', 'here', 'there',
+  'my', 'your', 'his', 'her', 'our', 'their', 'mine', 'yours', 'ours',
+  'what', 'which', 'who', 'whom', 'whose', 'how', 'when', 'where', 'why',
+  'all', 'each', 'every', 'both', 'few', 'more', 'most', 'much', 'many',
+  'other', 'some', 'such', 'own', 'same', 'any', 'either', 'neither',
+  'been', 'being', 'because', 'until', 'while', 'once', 'again', 'further',
+  'already', 'always', 'never', 'sometimes', 'often', 'still', 'yet',
+  // ── German ──
+  'der', 'die', 'das', 'den', 'dem', 'des', 'ein', 'eine', 'einen',
+  'einem', 'einer', 'eines', 'und', 'oder', 'aber', 'denn', 'weil',
+  'ist', 'sind', 'war', 'waren', 'sein', 'wird', 'werden', 'wurde',
+  'hat', 'haben', 'hatte', 'hatten', 'kann', 'können', 'konnte',
+  'soll', 'sollen', 'sollte', 'muss', 'müssen', 'musste', 'darf',
+  'mag', 'möchte', 'wollen', 'wollte', 'würde', 'könnte', 'sollte',
+  'mit', 'für', 'auf', 'von', 'aus', 'bei', 'nach', 'über', 'unter',
+  'vor', 'hinter', 'neben', 'zwischen', 'durch', 'gegen', 'ohne',
+  'um', 'bis', 'seit', 'während', 'wegen', 'trotz', 'statt',
+  'wie', 'was', 'wer', 'wen', 'wem', 'wessen', 'wo', 'wann', 'warum',
+  'nicht', 'noch', 'auch', 'schon', 'nur', 'sehr', 'mehr', 'viel',
+  'alle', 'jeder', 'jede', 'jedes', 'dieser', 'diese', 'dieses',
+  'jener', 'jene', 'jenes', 'mein', 'dein', 'sein', 'ihr', 'unser',
+  'euer', 'kein', 'keine', 'sich', 'mir', 'dir', 'ihm', 'uns', 'euch',
+  'ich', 'du', 'er', 'sie', 'es', 'wir', 'ihr', 'man',
+  'hier', 'dort', 'da', 'dann', 'also', 'doch', 'mal', 'eben', 'ganz',
+  // ── French ──
+  'le', 'la', 'les', 'un', 'une', 'des', 'du', 'de', 'au', 'aux',
+  'et', 'ou', 'mais', 'donc', 'car', 'ni', 'que', 'qui', 'quoi',
+  'est', 'sont', 'était', 'ont', 'avoir', 'être', 'fait', 'faire',
+  'pour', 'par', 'avec', 'dans', 'sur', 'sous', 'entre', 'vers',
+  'chez', 'sans', 'avant', 'après', 'pendant', 'depuis', 'contre',
+  'ce', 'cette', 'ces', 'mon', 'ton', 'son', 'notre', 'votre', 'leur',
+  'je', 'tu', 'il', 'elle', 'nous', 'vous', 'ils', 'elles', 'on',
+  'ne', 'pas', 'plus', 'très', 'bien', 'aussi', 'tout', 'tous', 'toute',
+  'même', 'autre', 'quel', 'quelle', 'comment', 'quand', 'où', 'pourquoi',
+  // ── Spanish ──
+  'el', 'la', 'los', 'las', 'un', 'una', 'unos', 'unas', 'del', 'al',
+  'lo', 'que', 'en', 'es', 'por', 'con', 'para', 'como', 'pero', 'más',
+  'fue', 'ser', 'hay', 'está', 'han', 'son', 'tiene', 'había', 'era',
+  'su', 'sus', 'este', 'esta', 'estos', 'estas', 'ese', 'esa', 'esos',
+  'mi', 'tu', 'yo', 'él', 'ella', 'nosotros', 'ellos', 'ellas', 'usted',
+  'no', 'ya', 'sí', 'sin', 'sobre', 'entre', 'hasta', 'desde', 'donde',
+  'muy', 'todo', 'toda', 'todos', 'cada', 'otro', 'otra', 'otros',
+  'cuando', 'porque', 'aunque', 'también', 'solo', 'después', 'antes',
+  // ── Italian ──
+  'il', 'lo', 'la', 'li', 'le', 'gli', 'uno', 'una', 'dei', 'del',
+  'che', 'di', 'da', 'per', 'con', 'tra', 'fra', 'sul', 'nel', 'al',
+  'è', 'sono', 'ha', 'hanno', 'era', 'essere', 'fare', 'fatto', 'stato',
+  'suo', 'sua', 'suoi', 'questo', 'questa', 'questi', 'quello', 'quella',
+  'io', 'tu', 'lui', 'lei', 'noi', 'voi', 'loro', 'ci', 'si',
+  'non', 'più', 'molto', 'anche', 'solo', 'tutto', 'tutti', 'ogni',
+  'come', 'dove', 'quando', 'perché', 'ancora', 'già', 'sempre', 'mai',
+  // ── Portuguese ──
+  'um', 'uma', 'uns', 'umas', 'do', 'da', 'dos', 'das', 'no', 'na',
+  'ao', 'aos', 'em', 'por', 'com', 'para', 'sem', 'sob', 'sobre',
+  'que', 'se', 'mas', 'ou', 'como', 'mais', 'entre', 'até', 'desde',
+  'é', 'são', 'foi', 'tem', 'ser', 'ter', 'estar', 'fazer', 'havia',
+  'seu', 'sua', 'seus', 'suas', 'este', 'esta', 'esse', 'essa', 'aquele',
+  'eu', 'tu', 'ele', 'ela', 'nós', 'eles', 'elas', 'você', 'vocês',
+  'não', 'já', 'sim', 'bem', 'muito', 'também', 'ainda', 'sempre',
+  'todo', 'toda', 'todos', 'cada', 'outro', 'outra', 'quando', 'porque',
+]);
+
+/** Tokenize text into meaningful keywords. Handles unicode (accented chars). */
+function tokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .replace(/[^a-záàâãäåæçéèêëíìîïñóòôõöúùûüýÿßœ0-9:_\-./]/g, ' ')
+    .split(/\s+/)
+    .filter(w => w.length > 1 && !STOPWORDS.has(w));
+}
+
+/**
+ * Split a multi-topic query into sub-queries.
+ * Detects numbered lists, semicolons, "and also", line breaks, etc.
+ *
+ * Example: "deploy API fix routing and also check auth" →
+ *   ["deploy API fix routing", "check auth"]
+ *
+ * Example: "1. deploy 2. fix routing 3. auth" →
+ *   ["deploy", "fix routing", "auth"]
+ */
+function splitMultiQuery(query: string): string[] {
+  // Numbered list: "1. foo 2. bar 3. baz"
+  const numberedParts = query.split(/\d+[.)]\s*/g).filter(s => s.trim().length > 2);
+  if (numberedParts.length >= 2) return numberedParts.map(s => s.trim());
+
+  // Semicolons or newlines
+  const semiParts = query.split(/[;\n]+/).filter(s => s.trim().length > 2);
+  if (semiParts.length >= 2) return semiParts.map(s => s.trim());
+
+  // Conjunctions (EN + DE + FR + ES)
+  const conjParts = query.split(
+    /\b(?:and also|also noch|außerdem|plus|additionally|furthermore|de plus|además|inoltre|além disso)\b/i
+  ).filter(s => s.trim().length > 2);
+  if (conjParts.length >= 2) return conjParts.map(s => s.trim());
+
+  // Comma-separated with 3+ parts (likely a list)
+  const commaParts = query.split(/,\s*/).filter(s => s.trim().length > 2);
+  if (commaParts.length >= 3) return commaParts.map(s => s.trim());
+
+  // Single query
+  return [query];
+}
+
+// ── BM25+ Scoring ─────────────────────────────────────────────────────────────
+//
+// BM25+ fixes the "long document under-scoring" bug in classic BM25.
+// Paper: Lv & Zhai (2011) "Lower-Bounding Term Frequency Normalization"
+//
+// Improvements over standard BM25:
+//   1. δ=1 additive term → guarantees TF>0 terms always contribute positively
+//   2. Bigram proximity boost → adjacent query terms in doc get 2× weight
+//   3. Recency boost → entries with timestamps get exp-decay bonus (7d half-life)
+//   4. Levenshtein fuzzy match → typo-tolerant (distance ≤ 2)
+//
+
+/** BM25+ parameters */
+const BM25_K1    = 1.2;   // term frequency saturation
+const BM25_B     = 0.75;  // length normalization
+const BM25_DELTA = 1.0;   // BM25+ lower-bound guarantee (0 = classic BM25)
+
+/** Recency boost: half-life in days. Entry from 7 days ago gets 0.5× boost. */
+const RECENCY_HALF_LIFE_DAYS = 7;
+
+interface DocEntry {
+  key: string;
+  content: string;
+  tokens: string[];
+  tokenFreq: Map<string, number>;  // term → count in this doc
+  bigrams: Set<string>;            // "term1|term2" adjacency pairs
+  timestamp?: number;              // epoch ms, extracted from content if present
+}
+
+interface KeywordMatch {
+  key: string;
+  content: string;
+  score: number;
+  matchedWords: string[];
+  subQuery?: string;   // which sub-query matched (for multi-topic)
+}
+
+/**
+ * Levenshtein distance — edit distance between two strings.
+ * Used for typo-tolerant fuzzy matching (distance ≤ 2 = match).
+ * O(n*m) but strings are short (tokens), so this is fast.
+ */
+function levenshtein(a: string, b: string): number {
+  if (a === b) return 0;
+  if (a.length === 0) return b.length;
+  if (b.length === 0) return a.length;
+  // Skip if length diff > 2 (can't be ≤ 2 edits)
+  if (Math.abs(a.length - b.length) > 2) return 3;
+
+  const prev = Array.from({ length: b.length + 1 }, (_, i) => i);
+  for (let i = 1; i <= a.length; i++) {
+    let corner = i - 1;
+    prev[0] = i;
+    for (let j = 1; j <= b.length; j++) {
+      const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+      const cur = Math.min(prev[j] + 1, prev[j - 1] + 1, corner + cost);
+      corner = prev[j];
+      prev[j] = cur;
+    }
+  }
+  return prev[b.length];
+}
+
+/** Extract a timestamp from JSON content (looks for "ts" or "created" fields). */
+function extractTimestamp(content: string): number | undefined {
+  // Fast regex for ISO dates in JSON: "ts":"2026-04-17T..." or "created":"..."
+  const match = content.match(/"(?:ts|created|created_at|timestamp)"\s*:\s*"([^"]+)"/);
+  if (match) {
+    const ms = Date.parse(match[1]);
+    if (!isNaN(ms)) return ms;
+  }
+  return undefined;
+}
+
+/** Recency multiplier: 1.0 for now, 0.5 after half-life days, exponential decay. */
+function recencyBoost(timestampMs: number | undefined): number {
+  if (!timestampMs) return 1.0; // no timestamp → neutral
+  const ageDays = (Date.now() - timestampMs) / (1000 * 60 * 60 * 24);
+  if (ageDays <= 0) return 1.5; // future/just-now → max boost
+  return Math.pow(0.5, ageDays / RECENCY_HALF_LIFE_DAYS) + 0.5; // range: [0.5, 1.5]
+}
+
+/**
+ * BM25+ with Bigram Proximity, Recency Boost, and Fuzzy Matching.
+ *
+ * Architecture:
+ * ┌─────────────────────────────────────────────────────────────────┐
+ * │  Query: "deploy API fix routing; check auth"                    │
+ * │    ↓ splitMultiQuery                                            │
+ * │  Sub-queries: ["deploy API fix routing", "check auth"]          │
+ * │    ↓ for each sub-query                                         │
+ * │  Tokenize → BM25+ score per doc → Bigram boost → Fuzzy match   │
+ * │    ↓ merge & deduplicate                                        │
+ * │  Recency boost → Sort → Top-K                                  │
+ * └─────────────────────────────────────────────────────────────────┘
+ */
+async function keywordSearch(
+  redis: Redis,
+  patterns: string[],
+  query: string,
+  topK = 10,
+): Promise<KeywordMatch[]> {
+  // ── Step 1: Collect and tokenize all documents ──
+  const allKeys: string[] = [];
+  for (const pattern of patterns) {
+    const stream = redis.scanStream({ match: pattern, count: 200 });
+    await new Promise<void>((resolve, reject) => {
+      stream.on('data', (batch: string[]) => {
+        allKeys.push(...batch.filter((k: string) => !k.endsWith(':meta')));
+      });
+      stream.on('end', resolve);
+      stream.on('error', reject);
+    });
+  }
+
+  if (allKeys.length === 0) return [];
+
+  // Pipeline GET for speed
+  const pipeline = redis.pipeline();
+  for (const key of allKeys) pipeline.get(key);
+  const results = await pipeline.exec();
+
+  const docs: DocEntry[] = [];
+  let totalTokens = 0;
+
+  for (let i = 0; i < allKeys.length; i++) {
+    const content = results?.[i]?.[1] as string | null;
+    if (!content) continue;
+
+    const tokens = tokenize(`${allKeys[i]} ${content}`);
+    if (tokens.length === 0) continue;
+
+    // Term frequency map
+    const tokenFreq = new Map<string, number>();
+    for (const t of tokens) {
+      tokenFreq.set(t, (tokenFreq.get(t) ?? 0) + 1);
+    }
+
+    // Bigrams — adjacent token pairs for proximity detection
+    const bigrams = new Set<string>();
+    for (let j = 0; j < tokens.length - 1; j++) {
+      bigrams.add(`${tokens[j]}|${tokens[j + 1]}`);
+    }
+
+    const timestamp = extractTimestamp(content);
+    docs.push({ key: allKeys[i], content, tokens, tokenFreq, bigrams, timestamp });
+    totalTokens += tokens.length;
+  }
+
+  if (docs.length === 0) return [];
+  const avgDL = totalTokens / docs.length;
+
+  // ── Step 2: IDF (inverse document frequency) ──
+  const docFreq = new Map<string, number>();
+  for (const doc of docs) {
+    const seen = new Set<string>();
+    for (const t of doc.tokens) {
+      if (!seen.has(t)) {
+        docFreq.set(t, (docFreq.get(t) ?? 0) + 1);
+        seen.add(t);
+      }
+    }
+  }
+  const N = docs.length;
+
+  function idf(term: string): number {
+    const df = docFreq.get(term) ?? 0;
+    return Math.log((N - df + 0.5) / (df + 0.5) + 1);
+  }
+
+  /**
+   * BM25+ score for a term in a document.
+   * Classic BM25: TF_norm = tf*(k1+1) / (tf + k1*(1 - b + b*dl/avgdl))
+   * BM25+ adds: + δ  (guarantees long docs with the term still score positively)
+   */
+  function bm25PlusTerm(term: string, doc: DocEntry): number {
+    const tf = doc.tokenFreq.get(term) ?? 0;
+    if (tf === 0) return 0;
+    const dl = doc.tokens.length;
+    const tfNorm = (tf * (BM25_K1 + 1)) / (tf + BM25_K1 * (1 - BM25_B + BM25_B * (dl / avgDL)));
+    return idf(term) * (tfNorm + BM25_DELTA);
+  }
+
+  /**
+   * Fuzzy match: tries exact → substring → Levenshtein ≤ 2.
+   * Returns [matchedTerm, weight] or null.
+   */
+  function fuzzyMatch(qt: string, docTermSet: Set<string>): [string, number] | null {
+    // Exact
+    if (docTermSet.has(qt)) return [qt, 1.0];
+    // Substring (partial)
+    for (const dt of docTermSet) {
+      if (dt.length > 3 && qt.length > 3 && (dt.includes(qt) || qt.includes(dt))) {
+        return [dt, 0.6];
+      }
+    }
+    // Levenshtein ≤ 2 (typo tolerance) — only for tokens ≥ 4 chars
+    if (qt.length >= 4) {
+      for (const dt of docTermSet) {
+        if (dt.length >= 4 && levenshtein(qt, dt) <= 2) {
+          return [dt, 0.4];
+        }
+      }
+    }
+    return null;
+  }
+
+  // ── Step 3: Score each sub-query independently ──
+  const subQueries = splitMultiQuery(query);
+  const allMatches = new Map<string, KeywordMatch>();
+
+  for (const sq of subQueries) {
+    const queryTokens = tokenize(sq);
+    if (queryTokens.length === 0) continue;
+
+    // Pre-compute query bigrams for proximity boost
+    const queryBigrams = new Set<string>();
+    for (let j = 0; j < queryTokens.length - 1; j++) {
+      queryBigrams.add(`${queryTokens[j]}|${queryTokens[j + 1]}`);
+    }
+
+    for (const doc of docs) {
+      let score = 0;
+      const matchedWords: string[] = [];
+      const docTermSet = new Set(doc.tokens);
+
+      for (const qt of queryTokens) {
+        // Try exact BM25+ first
+        const exactScore = bm25PlusTerm(qt, doc);
+        if (exactScore > 0) {
+          score += exactScore;
+          matchedWords.push(qt);
+          continue;
+        }
+        // Fuzzy match (substring or Levenshtein)
+        const fuzz = fuzzyMatch(qt, docTermSet);
+        if (fuzz) {
+          score += bm25PlusTerm(fuzz[0], doc) * fuzz[1];
+          matchedWords.push(`~${qt}`);
+        }
+      }
+
+      // Bigram proximity boost: +50% for each adjacent query term pair found in doc
+      if (queryBigrams.size > 0) {
+        let bigramHits = 0;
+        for (const bg of queryBigrams) {
+          if (doc.bigrams.has(bg)) bigramHits++;
+        }
+        if (bigramHits > 0) {
+          score *= 1 + 0.5 * (bigramHits / queryBigrams.size);
+        }
+      }
+
+      // Recency boost
+      score *= recencyBoost(doc.timestamp);
+
+      if (score > 0) {
+        const existing = allMatches.get(doc.key);
+        if (!existing || score > existing.score) {
+          allMatches.set(doc.key, {
+            key: doc.key,
+            content: doc.content,
+            score,
+            matchedWords: [...new Set(matchedWords)],
+            subQuery: subQueries.length > 1 ? sq : undefined,
+          });
+        }
+      }
+    }
+  }
+
+  // ── Step 4: Sort by score, return top-K ──
+  const sorted = [...allMatches.values()].sort((a, b) => b.score - a.score);
+  return sorted.slice(0, topK);
+}
+
+// ── Exported for testing ──────────────────────────────────────────────────────
+export { tokenize, splitMultiQuery, levenshtein, recencyBoost, extractTimestamp, STOPWORDS };
+
 // ── Types ─────────────────────────────────────────────────────────────────────
 
 interface Instance {
@@ -898,6 +1392,59 @@ const TOOLS = [
       required: ['instance_id', 'summary'],
     },
   },
+  // ── Session Handoff — cross-window continuity ─────────────────────────────
+  {
+    name: 'session_handoff',
+    description:
+      'Save a detailed handoff for the NEXT chat window / session. ' +
+      'Stores: current progress, TODO list (done + remaining), changed files with descriptions, ' +
+      'instructions for the next assistant, and any incomplete work. ' +
+      'The next session_start automatically includes this handoff so the new window knows EXACTLY what happened and what remains. ' +
+      'Call this BEFORE closing a chat window, especially if work is incomplete. ' +
+      'This prevents the "continue" problem where new windows lose context, skip tasks, or produce broken code.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        instance_id: { type: 'string', description: 'UUID of the cache instance' },
+        completed_tasks: {
+          type: 'array',
+          items: { type: 'string' },
+          description: 'Tasks that were fully completed (e.g. "Implemented brainSearch() in JS SDK")',
+        },
+        remaining_tasks: {
+          type: 'array',
+          items: { type: 'string' },
+          description: 'Tasks NOT yet done — the next window MUST pick these up',
+        },
+        files_changed: {
+          type: 'array',
+          items: {
+            type: 'object',
+            properties: {
+              path: { type: 'string', description: 'File path relative to project root' },
+              status: { type: 'string', enum: ['complete', 'partial', 'broken'], description: 'State of this file' },
+              description: { type: 'string', description: 'What was changed and what still needs work' },
+            },
+            required: ['path', 'status'],
+          },
+          description: 'Changed files with their current state — marks partial/broken files so next window knows to fix them',
+        },
+        instructions: {
+          type: 'string',
+          description: 'Free-form instructions for the next assistant. Be specific: what to do next, what to avoid, what broke.',
+        },
+        context_summary: {
+          type: 'string',
+          description: 'Brief summary of what happened this session (architecture decisions, key findings, blockers)',
+        },
+        blocked_on: {
+          type: 'string',
+          description: 'If work is blocked, describe what is needed to unblock (e.g. "waiting for API deploy", "needs user input on design")',
+        },
+      },
+      required: ['instance_id', 'completed_tasks', 'remaining_tasks'],
+    },
+  },
   // ── AI Brain — Extended features ─────────────────────────────────────────
   {
     name: 'auto_learn_session',
@@ -947,6 +1494,29 @@ const TOOLS = [
       required: ['instance_id', 'changed_files'],
     },
   },
+  {
+    name: 'team_learn',
+    description:
+      'Store a lesson in a shared team brain so all team members benefit. ' +
+      'Like learn_from_attempts, but REQUIRES an author name for attribution. ' +
+      'Shows up in team_recall with "by <author>" so the team knows who learned it.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        instance_id:  { type: 'string', description: 'UUID of the shared team brain instance' },
+        author:       { type: 'string', description: 'Your name or handle (required for team attribution)' },
+        topic:        { type: 'string', description: 'Topic in category:keyword format (e.g. "deploy:api")' },
+        outcome:      { type: 'string', enum: ['success', 'failure', 'partial'], description: 'What happened' },
+        what_worked:  { type: 'string', description: 'What worked (the solution)' },
+        what_failed:  { type: 'string', description: 'What did NOT work (avoid this)' },
+        severity:     { type: 'string', enum: ['critical', 'major', 'minor'], description: 'Impact level' },
+        file_paths:   { type: 'array', items: { type: 'string' }, description: 'Relevant file paths' },
+        commands:     { type: 'array', items: { type: 'string' }, description: 'Commands that worked' },
+        tags:         { type: 'array', items: { type: 'string' }, description: 'Tags for categorization' },
+      },
+      required: ['instance_id', 'author', 'topic', 'outcome', 'what_worked'],
+    },
+  },
   {
     name: 'team_recall',
     description:
@@ -1642,76 +2212,100 @@ async function handleTool(name: string, args: Record<string, unknown>): Promise<
       const vectorUrl =
         process.env.CACHLY_VECTOR_URL ??
         (inst.vector_token ? `https://api.cachly.dev/v1/sem/${inst.vector_token}` : null);
-
-      if (!vectorUrl) {
-        return (
-          `❌ index_project requires the Speed or Business tier (pgvector).\n\n` +
-          `Your instance "${inst.name}" (${inst.tier.toUpperCase()}) does not have a vector_token.\n` +
-          `Upgrade at https://cachly.dev/instances/${instance_id}`
-        );
-      }
+      const canEmbed = vectorUrl && hasEmbedProvider();
 
       let indexed = 0;
       let skipped = 0;
       let errors = 0;
+      let semanticIndexed = 0;
+      let unchanged = 0;
       const details: string[] = [];
       const expiresAt = new Date(Date.now() + ttl * 1000).toISOString();
+      const redis = await getConnection(instance_id);
 
       for (const filePath of files) {
         const relPath = relative(dir, filePath);
         let content: string;
+        let fileSize: number;
         try {
           const s = await stat(filePath);
           if (s.size > 200_000) { skipped++; continue; } // skip files >200 KB
+          fileSize = s.size;
           content = await readFile(filePath, 'utf-8');
         } catch {
           errors++;
           continue;
         }
 
-        // Prompt = relative path + first N chars of content (for semantic matching)
-        const excerpt = content.slice(0, summary_chars).replace(/\s+/g, ' ').trim();
-        const prompt = `File: ${relPath}\n${excerpt}`;
-        // Value = relative path so the AI knows which file to read_file next
-
-        try {
-          const embedding = await computeEmbedding(prompt);
-          const id = randomUUID();
+        // ── Smart invalidation: hash-based change detection ──
+        // Compute a simple hash of file content to skip unchanged files
+        const hashKey = `cachly:idx:hash:${relPath}`;
+        const contentHash = `${fileSize}:${content.length}:${simpleHash(content)}`;
+        const existingHash = await redis.get(hashKey);
+        if (existingHash === contentHash) {
+          // File unchanged — refresh TTL but skip re-indexing
+          const idxKey = `cachly:idx:${relPath}`;
+          if (ttl > 0) await redis.expire(idxKey, ttl);
+          if (ttl > 0) await redis.expire(hashKey, ttl);
+          unchanged++;
+          continue;
+        }
 
-          // Write value to vector index
-          await fetch(`${vectorUrl}/entries`, {
-            method: 'POST',
-            headers: { 'Content-Type': 'application/json' },
-            body: JSON.stringify({ id, prompt, namespace: nsArg, embedding, expires_at: expiresAt }),
-          });
+        const excerpt = content.slice(0, summary_chars).replace(/\s+/g, ' ').trim();
 
-          // Write value to Valkey for retrieval on semantic hit
-          const redis = await getConnection(instance_id);
-          await redis.set(`${nsArg}:val:${id}`, relPath, 'EX', ttl);
+        // ── Layer 1: Keyword index in Valkey (always works, no embedding needed) ──
+        const idxKey = `cachly:idx:${relPath}`;
+        const idxValue = `File: ${relPath}\n${excerpt}`;
+        if (ttl > 0) {
+          await redis.set(idxKey, idxValue, 'EX', ttl);
+        } else {
+          await redis.set(idxKey, idxValue);
+        }
+        // Store content hash for smart invalidation on next run
+        if (ttl > 0) {
+          await redis.set(hashKey, contentHash, 'EX', ttl);
+        } else {
+          await redis.set(hashKey, contentHash);
+        }
+        indexed++;
+        details.push(`  ✅ ${relPath}`);
 
-          indexed++;
-          details.push(`  ✅ ${relPath}`);
-        } catch {
-          errors++;
-          details.push(`  ❌ ${relPath} (embedding failed)`);
+        // ── Layer 2: Semantic vector index (optional, only if embedding available) ──
+        if (canEmbed) {
+          try {
+            const prompt = `File: ${relPath}\n${excerpt}`;
+            const embedding = await computeEmbedding(prompt);
+            const id = randomUUID();
+            await fetch(`${vectorUrl}/entries`, {
+              method: 'POST',
+              headers: { 'Content-Type': 'application/json' },
+              body: JSON.stringify({ id, prompt, namespace: nsArg, embedding, expires_at: expiresAt }),
+            });
+            await redis.set(`${nsArg}:val:${id}`, relPath, 'EX', ttl);
+            semanticIndexed++;
+          } catch {
+            // Semantic indexing failed — keyword index is enough
+          }
         }
       }
 
+      const mode = canEmbed ? '🔍 Keyword + 🎯 Semantic' : '🔍 Keyword only (no embedding provider)';
+
       return [
-        `📂 **index_project Complete**`,
+        `📂 **index_project Complete** — ${mode}`,
         ``,
-        `  📁 Dir:     ${dir}`,
-        `  ✅ Indexed: **${indexed}** files`,
-        `  ⏭️  Skipped: ${skipped} (too large or filtered)`,
-        `  ❌ Errors:  ${errors}`,
-        `  ⏱️  TTL:     ${ttl}s (${Math.round(ttl / 3600)}h)`,
-        `  🏷️  Namespace: \`${nsArg}\``,
+        `  📁 Dir:       ${dir}`,
+        `  ✅ Indexed:   **${indexed}** files (new/changed)`,
+        `  ♻️  Unchanged: ${unchanged} files (hash match — skipped)`,
+        ...(canEmbed ? [`  🎯 Semantic:  **${semanticIndexed}** files (vector-searchable)`] : []),
+        `  ⏭️  Skipped:   ${skipped} (too large or filtered)`,
+        `  ❌ Errors:    ${errors}`,
+        `  ⏱️  TTL:       ${ttl}s (${Math.round(ttl / 3600)}h)`,
         ``,
         `💡 **Next steps:**`,
-        `   1. Add to .github/copilot-instructions.md:`,
-        `      "Before reading source files, call semantic_search with the topic."`,
-        `   2. Re-run index_project after major refactors (or set up a cron TTL).`,
-        `   3. Use semantic_search("how does X work?") to find relevant files.`,
+        `   1. Use \`smart_recall("how does auth work")\` to find relevant files.`,
+        `   2. Re-run index_project after major refactors.`,
+        ...(canEmbed ? [] : [`   3. Set OPENAI_API_KEY (or similar) in .env to also enable semantic search.`]),
         ``,
         ...(details.length <= 20 ? details : [...details.slice(0, 20), `  … and ${details.length - 20} more`]),
       ].join('\n');
@@ -2067,72 +2661,106 @@ async function handleTool(name: string, args: Record<string, unknown>): Promise<
         threshold = 0.78,
       } = args as { instance_id: string; query: string; threshold?: number };
 
-      const inst = await apiFetch<Instance>(`/api/v1/instances/${instance_id}`);
-      if (!inst.vector_token) {
-        // Fallback to listing and keyword matching
-        const redis = await getConnection(instance_id);
-        const keys: string[] = [];
-        const stream = redis.scanStream({ match: 'cachly:ctx:*', count: 100 });
-        await new Promise<void>((resolve, reject) => {
-          stream.on('data', (batch: string[]) => {
-            keys.push(...batch.filter((k: string) => !k.endsWith(':meta')));
-          });
-          stream.on('end', resolve);
-          stream.on('error', reject);
-        });
+      const redis = await getConnection(instance_id);
 
-        // Simple keyword matching
-        const queryWords = query.toLowerCase().split(/\s+/);
-        const matches: string[] = [];
-        for (const k of keys) {
-          const keyLower = k.toLowerCase();
-          if (queryWords.some(w => keyLower.includes(w))) {
-            const content = await redis.get(k);
-            matches.push(`**${k.replace('cachly:ctx:', '')}**:\n${content?.slice(0, 300)}…\n`);
-          }
-        }
+      // ── Layer 1: Keyword search across ALL brain data (always works, no embedding) ──
+      const kwMatches = await keywordSearch(
+        redis,
+        ['cachly:ctx:*', 'cachly:lesson:best:*', 'cachly:idx:*'],
+        query,
+        10,
+      );
+
+      const lines: string[] = [`🧠 **Smart Recall** for: _"${query}"_\n`];
 
-        if (matches.length === 0) return `⚠️ No matching context found for: "${query}"\n\nTip: Upgrade to Speed tier for semantic search.`;
-        return `🔍 **Keyword-matched context:**\n\n${matches.join('\n')}`;
+      // Show sub-query info if multi-topic was detected
+      const subQueries = splitMultiQuery(query);
+      if (subQueries.length > 1) {
+        lines.push(`_Detected ${subQueries.length} sub-topics:_ ${subQueries.map((s, i) => `${i + 1}. "${s}"`).join(', ')}\n`);
       }
 
-      // Semantic search
-      const embedding = await computeEmbedding(query);
-      const vectorUrl = `https://api.cachly.dev/v1/sem/${inst.vector_token}`;
-      const searchRes = await fetch(`${vectorUrl}/search`, {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ embedding, namespace: 'cachly:ctx', threshold, top_k: 5 }),
-      });
+      if (kwMatches.length > 0) {
+        lines.push(`### 🔍 BM25 Matches (${kwMatches.length})\n`);
 
-      if (!searchRes.ok) {
-        return `❌ Semantic search failed: ${searchRes.statusText}`;
+        // Group by sub-query if multi-topic
+        if (subQueries.length > 1) {
+          const grouped = new Map<string, KeywordMatch[]>();
+          for (const m of kwMatches.slice(0, 12)) {
+            const sq = m.subQuery ?? query;
+            if (!grouped.has(sq)) grouped.set(sq, []);
+            grouped.get(sq)!.push(m);
+          }
+          for (const [sq, matches] of grouped) {
+            lines.push(`**Topic: "${sq}"** (${matches.length} results)\n`);
+            for (const m of matches.slice(0, 4)) {
+              const label = m.key
+                .replace('cachly:ctx:', '📝 ')
+                .replace('cachly:lesson:best:', '💡 ')
+                .replace('cachly:idx:', '📂 ');
+              const preview = m.content.slice(0, 300).replace(/\n/g, ' ');
+              lines.push(`  **${label}** _(BM25: ${m.score.toFixed(2)}, matched: ${m.matchedWords.join(', ')})_`);
+              lines.push(`  > ${preview}${m.content.length > 300 ? '…' : ''}\n`);
+            }
+          }
+          // Summary: which sub-queries had matches
+          const matched = [...grouped.keys()];
+          const unmatched = subQueries.filter(sq => !matched.includes(sq));
+          if (unmatched.length > 0) {
+            lines.push(`\n⚠️ **No results for:** ${unmatched.map(s => `"${s}"`).join(', ')}`);
+          }
+        } else {
+          for (const m of kwMatches.slice(0, 8)) {
+            const label = m.key
+              .replace('cachly:ctx:', '📝 ')
+              .replace('cachly:lesson:best:', '💡 ')
+              .replace('cachly:idx:', '📂 ');
+            const preview = m.content.slice(0, 400).replace(/\n/g, ' ');
+            lines.push(`**${label}** _(BM25: ${m.score.toFixed(2)}, matched: ${m.matchedWords.join(', ')})_`);
+            lines.push(`> ${preview}${m.content.length > 400 ? '…' : ''}\n`);
+          }
+        }
       }
 
-      const results = (await searchRes.json()) as SemanticSearchResponse[];
-      if (!results.length || !results[0].found) {
-        return `⚠️ No semantically similar context found for: "${query}"\n\nTry lowering the threshold (current: ${threshold}) or use \`list_remembered\` to see available context.`;
-      }
+      // ── Layer 2: Semantic search (optional, only if embedding provider + vector_token available) ──
+      const inst = await apiFetch<Instance>(`/api/v1/instances/${instance_id}`);
+      if (inst.vector_token && hasEmbedProvider()) {
+        try {
+          const embedding = await computeEmbedding(query);
+          const vectorUrl = `https://api.cachly.dev/v1/sem/${inst.vector_token}`;
+          const searchRes = await fetch(`${vectorUrl}/search`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ embedding, namespace: 'cachly:ctx', threshold, top_k: 5 }),
+          });
 
-      const redis = await getConnection(instance_id);
-      const lines: string[] = [`🧠 **Smart Recall Results** for: _"${query}"_\n`];
+          if (searchRes.ok) {
+            const results = (await searchRes.json()) as SemanticSearchResponse[];
+            const semHits = results.filter(r => r.found && r.id);
+            if (semHits.length > 0) {
+              lines.push(`\n### 🎯 Semantic Matches (${semHits.length})\n`);
+              for (const hit of semHits) {
+                const parts = hit.id!.replace('ctx:', '').split(':');
+                const category = parts[0];
+                const key = parts.slice(1).join(':');
+                const content = await redis.get(`cachly:ctx:${category}:${key}`);
+                lines.push(
+                  `**${key}** _(${((hit.similarity ?? 0) * 100).toFixed(0)}% similar)_`,
+                  `> ${content?.slice(0, 300) ?? '(evicted)'}${(content?.length ?? 0) > 300 ? '…' : ''}\n`,
+                );
+              }
+            }
+          }
+        } catch {
+          // Semantic search failed silently — keyword results are enough
+        }
+      }
 
-      for (const hit of results) {
-        if (!hit.found || !hit.id) continue;
-        // id format: ctx:category:key
-        const parts = hit.id.replace('ctx:', '').split(':');
-        const category = parts[0];
-        const key = parts.slice(1).join(':');
-        const content = await redis.get(`cachly:ctx:${category}:${key}`);
-        lines.push(
-          `### ${key} (${((hit.similarity ?? 0) * 100).toFixed(0)}% match)`,
-          `_Category: ${category}_`,
-          `\`\`\``,
-          content?.slice(0, 500) ?? '(content evicted)',
-          `${(content?.length ?? 0) > 500 ? '…' : ''}`,
-          `\`\`\``,
-          ``
-        );
+      if (kwMatches.length === 0) {
+        lines.push(`⚠️ No matches found for: "${query}"`);
+        lines.push(`\n💡 Tips:`);
+        lines.push(`  • Try different keywords`);
+        lines.push(`  • Use \`list_remembered\` to see available context`);
+        lines.push(`  • Use \`recall_best_solution("topic")\` for exact topic lookup`);
       }
 
       return lines.join('\n');
@@ -2268,6 +2896,37 @@ async function handleTool(name: string, args: Record<string, unknown>): Promise<
       // ── Build briefing ──────────────────────────────────────────────────────
       const lines: string[] = ['🧠 **Session Briefing**', ''];
 
+      // Handoff from previous window (if any)
+      const handoffRaw = await redis.get('cachly:session:handoff');
+      if (handoffRaw) {
+        try {
+          const handoff = JSON.parse(handoffRaw) as {
+            ts: string; completed_tasks: string[]; remaining_tasks: string[];
+            files_changed?: { path: string; status: string; description?: string }[];
+            instructions?: string; context_summary?: string; blocked_on?: string;
+          };
+          const ago = Math.round((Date.now() - new Date(handoff.ts).getTime()) / 60000);
+          const agoStr = ago < 60 ? `${ago}m ago` : ago < 1440 ? `${Math.round(ago / 60)}h ago` : `${Math.round(ago / 1440)}d ago`;
+
+          lines.push(`🤝 **Handoff from previous window** (${agoStr}):`);
+          if (handoff.context_summary) lines.push(`   ${handoff.context_summary}`);
+          if (handoff.remaining_tasks.length > 0) {
+            lines.push(`   ⏳ **Remaining tasks:**`);
+            for (const t of handoff.remaining_tasks) lines.push(`     - ${t}`);
+          }
+          if (handoff.completed_tasks.length > 0) {
+            lines.push(`   ✅ **Already done:** ${handoff.completed_tasks.join(', ')}`);
+          }
+          const brokenFiles = (handoff.files_changed ?? []).filter(f => f.status === 'broken' || f.status === 'partial');
+          if (brokenFiles.length > 0) {
+            lines.push(`   ⚠️ **Needs fix:** ${brokenFiles.map(f => `\`${f.path}\` (${f.status}${f.description ? ': ' + f.description : ''})`).join(', ')}`);
+          }
+          if (handoff.blocked_on) lines.push(`   🚫 **Blocked on:** ${handoff.blocked_on}`);
+          if (handoff.instructions) lines.push(`   📝 **Instructions:** ${handoff.instructions}`);
+          lines.push('');
+        } catch { /* ignore corrupt handoff */ }
+      }
+
       // Last session
       if (lastSession) {
         const ago = Math.round((Date.now() - new Date(lastSession.ts).getTime()) / 60000);
@@ -2367,6 +3026,80 @@ async function handleTool(name: string, args: Record<string, unknown>): Promise<
       // Clean up current session marker
       await redis.del('cachly:session:current');
 
+      // ── AUTO-LEARN from session summary (no manual call needed) ─────────────
+      // Parse the summary for actionable lessons and store them automatically.
+      const autoLearned: string[] = [];
+      try {
+        // Extract key sentences from the summary that contain action verbs
+        const actionVerbs = /\b(fixed|deployed|added|removed|refactored|migrated|updated|resolved|implemented|improved|optimized|configured|created|deleted|disabled|enabled|discovered|found|learned|debugged|patched|upgraded|installed|tested|built|rewrote|moved|renamed|split|merged|extracted)\b/i;
+        const sentences = summary
+          .split(/[.!\n]+/)
+          .map(s => s.trim())
+          .filter(s => s.length > 20 && actionVerbs.test(s));
+
+        for (const sentence of sentences.slice(0, 6)) {
+          // Build a topic slug from the first meaningful words
+          const words = sentence.toLowerCase()
+            .replace(/[^a-z0-9\s]/g, ' ')
+            .split(/\s+/)
+            .filter(w => w.length > 3 && !['that', 'this', 'with', 'from', 'have', 'been', 'were', 'they', 'then', 'when', 'also', 'into', 'will', 'would', 'could', 'should'].includes(w));
+          const slug = words.slice(0, 4).join('-');
+          if (!slug) continue;
+          const topic = `auto:${slug}`;
+          const key = `cachly:lesson:best:${topic}`;
+
+          // Don't overwrite existing successful lessons
+          const existing = await redis.get(key);
+          if (existing) {
+            try {
+              const ex = JSON.parse(existing) as { outcome: string };
+              if (ex.outcome === 'success') continue;
+            } catch { /* ignore */ }
+          }
+
+          const lesson = {
+            topic,
+            outcome: 'success',
+            what_worked: sentence,
+            context: `Auto-learned from session summary. Full summary: ${summary.slice(0, 300)}`,
+            severity: 'minor',
+            ts: now.toISOString(),
+            recall_count: 0,
+            auto_learned: true,
+            session_ts: now.toISOString(),
+            version: 2,
+          };
+          await redis.set(key, JSON.stringify(lesson));
+          // 90-day TTL for auto-learned lessons
+          await redis.expire(key, 90 * 86400);
+          autoLearned.push(topic);
+        }
+
+        // Also store a lesson per changed file area if files were changed
+        if (files_changed.length > 0) {
+          const areas = [...new Set(files_changed.map(f => f.split('/').slice(0, 2).join('/')))].slice(0, 3);
+          for (const area of areas) {
+            const slug = area.replace(/[^a-z0-9]/gi, '-').toLowerCase().replace(/-+/g, '-').slice(0, 30);
+            const topic = `auto:changed:${slug}`;
+            const key = `cachly:lesson:best:${topic}`;
+            const lesson = {
+              topic,
+              outcome: 'success',
+              what_worked: `Files changed in ${area}: ${files_changed.filter(f => f.startsWith(area.split('/')[0])).slice(0, 5).join(', ')}`,
+              context: summary.slice(0, 200),
+              severity: 'minor',
+              ts: now.toISOString(),
+              recall_count: 0,
+              auto_learned: true,
+              version: 2,
+            };
+            await redis.set(key, JSON.stringify(lesson));
+            await redis.expire(key, 90 * 86400);
+            autoLearned.push(topic);
+          }
+        }
+      } catch { /* auto-learn errors must never break session_end */ }
+
       const durationStr = durationMin !== undefined ? ` · ${durationMin} min` : '';
       return [
         `✅ **Session saved**${durationStr}`,
@@ -2374,11 +3107,69 @@ async function handleTool(name: string, args: Record<string, unknown>): Promise<
         `📋 **Summary:** ${summary}`,
         files_changed.length > 0 ? `📁 **Files changed:** ${files_changed.map(f => `\`${f}\``).join(', ')}` : '',
         lessons_learned !== undefined ? `🧠 **Lessons stored:** ${lessons_learned}` : '',
+        autoLearned.length > 0 ? `🤖 **Auto-learned:** ${autoLearned.length} lessons extracted from summary (${autoLearned.slice(0, 3).map(t => `\`${t}\``).join(', ')}${autoLearned.length > 3 ? '…' : ''})` : '',
         ``,
         `💡 Next session: \`session_start(focus="...")\` to see this summary.`,
       ].filter(l => l !== '').join('\n');
     }
 
+    // ── session_handoff — cross-window continuity ─────────────────────────────
+    case 'session_handoff': {
+      const {
+        instance_id,
+        completed_tasks = [],
+        remaining_tasks = [],
+        files_changed = [],
+        instructions = '',
+        context_summary = '',
+        blocked_on = '',
+      } = args as {
+        instance_id: string;
+        completed_tasks: string[];
+        remaining_tasks: string[];
+        files_changed?: { path: string; status: string; description?: string }[];
+        instructions?: string;
+        context_summary?: string;
+        blocked_on?: string;
+      };
+
+      const redis = await getConnection(instance_id);
+      const now = new Date();
+
+      const handoff = {
+        ts: now.toISOString(),
+        completed_tasks,
+        remaining_tasks,
+        files_changed,
+        instructions,
+        context_summary,
+        blocked_on,
+      };
+
+      // Store handoff — never expires until next handoff overwrites it
+      await redis.set('cachly:session:handoff', JSON.stringify(handoff));
+
+      // Also append to history
+      await redis.lpush('cachly:session:handoff:history', JSON.stringify(handoff));
+      await redis.ltrim('cachly:session:handoff:history', 0, 19);
+
+      const totalTasks = completed_tasks.length + remaining_tasks.length;
+      const pct = totalTasks > 0 ? Math.round((completed_tasks.length / totalTasks) * 100) : 0;
+      const brokenFiles = files_changed.filter(f => f.status === 'broken' || f.status === 'partial');
+
+      return [
+        `🤝 **Handoff saved** — ${completed_tasks.length}/${totalTasks} tasks done (${pct}%)`,
+        ``,
+        completed_tasks.length > 0 ? `✅ **Completed:**\n${completed_tasks.map(t => `  - ${t}`).join('\n')}` : '',
+        remaining_tasks.length > 0 ? `\n⏳ **Remaining for next window:**\n${remaining_tasks.map(t => `  - ${t}`).join('\n')}` : '',
+        brokenFiles.length > 0 ? `\n⚠️ **Needs attention:** ${brokenFiles.map(f => `\`${f.path}\` (${f.status})`).join(', ')}` : '',
+        blocked_on ? `\n🚫 **Blocked on:** ${blocked_on}` : '',
+        instructions ? `\n📝 **Instructions:** ${instructions}` : '',
+        ``,
+        `💡 The next \`session_start\` will include this handoff automatically.`,
+      ].filter(l => l !== '').join('\n');
+    }
+
     // ── auto_learn_session ────────────────────────────────────────────────────
     case 'auto_learn_session': {
       const { instance_id, observations } = args as {
@@ -2501,6 +3292,43 @@ async function handleTool(name: string, args: Record<string, unknown>): Promise<
       return lines.filter(Boolean).join('\n');
     }
 
+    // ── team_learn ────────────────────────────────────────────────────────────
+    case 'team_learn': {
+      const { instance_id, author, topic, outcome, what_worked, what_failed, severity, file_paths, commands, tags } = args as {
+        instance_id: string; author: string; topic: string; outcome: string;
+        what_worked: string; what_failed?: string; severity?: string;
+        file_paths?: string[]; commands?: string[]; tags?: string[];
+      };
+      if (!author || !topic || !outcome || !what_worked) {
+        return '❌ Required: author, topic, outcome, what_worked';
+      }
+      const iid = instance_id;
+      if (!iid) return '❌ instance_id required';
+
+      // Store with author attribution via the same learn_from_attempts Redis structure
+      const lesson = {
+        topic, outcome, what_worked,
+        what_failed: what_failed ?? '',
+        severity: severity ?? 'minor',
+        author,
+        file_paths: file_paths ?? [],
+        commands: commands ?? [],
+        tags: [...(tags ?? []), 'team'],
+        timestamp: new Date().toISOString(),
+        recall_count: 0,
+        version: 2,
+      };
+
+      const redis = await getConnection(iid);
+      const key = `cachly:lessons:${topic}`;
+      await redis.rpush(key, JSON.stringify(lesson));
+      if (outcome === 'success') {
+        await redis.set(`cachly:lesson:best:${topic}`, JSON.stringify(lesson));
+      }
+
+      return `✅ Team lesson stored by **${author}**: \`${topic}\` (${outcome})\n💡 ${what_worked.slice(0, 120)}`;
+    }
+
     // ── team_recall ───────────────────────────────────────────────────────────
     case 'team_recall': {
       const { instance_id, topic, author, limit = 10 } = args as {
@@ -2954,65 +3782,148 @@ async function handleTool(name: string, args: Record<string, unknown>): Promise<
       // Generate copilot-instructions.md content
       const tier = inst.tier.toUpperCase();
       const smartRecallNote = hasVector
-        ? '3. Call `smart_recall` with keywords for semantic context search'
-        : '3. Call `recall_context` with relevant key patterns like "arch:*" or "file:*"';
+        ? '- `smart_recall("natural language query")` — semantic search (finds by meaning)'
+        : '- `recall_context("arch:*")` — exact/glob key lookup (upgrade to Speed/Business for smart_recall)';
       const layerNote = hasVector
-        ? `Layer 3 (Semantic): smart_recall uses pgvector HNSW on your ${tier} instance for meaning-based search`
-        : `Layer 3 (Autopilot): This file — upgrade to Speed/Business tier to unlock smart_recall`;
+        ? `Layer 3 (Semantic): smart_recall uses pgvector HNSW on your ${tier} instance`
+        : `Layer 3 (Autopilot): this file — upgrade to Speed/Business tier to unlock smart_recall`;
 
-      const copilotInstructions = `# cachly AI Memory — ${project_description}
+      const copilotInstructions = `# cachly AI Brain — ${project_description}
 
-## 3-Layer AI Memory System
+> AI memory system powered by [cachly.dev](https://cachly.dev). Works with GitHub Copilot, Claude Code, Cursor, Windsurf, Continue.dev, and any MCP-compatible editor.
 
-- **Layer 1 — Storage:** cachly instance \`${inst.name}\` (${tier}) — persistent Valkey brain, survives restarts
-- **Layer 2 — Tools:** learn_from_attempts · recall_best_solution · smart_recall — the memory API
-- **Layer 3 — Autopilot (this file):** instructs the AI to run memory tools automatically
+---
 
-## BEFORE every task — check memory first
+## 🔴 MANDATORY RULES — Always enforced, no exceptions
 
-1. Call \`recall_best_solution\` with the task topic to get known solutions
-   - \`recall_best_solution("deploy:web")\` → returns working deploy command
-   - \`recall_best_solution("debug:redis-tls")\` → returns root cause + fix
-   - If a solution exists, use it — do NOT re-discover from scratch
+### Rule 1 — Session Start (FIRST thing in every window)
+Call \`session_start\` **before reading any source files or making any changes**:
 
-2. Call \`recall_context\` with the area you're working in
-   - \`recall_context("arch:*")\` → cached architecture decisions
-   - \`recall_context("file:src/auth*")\` → cached file summaries
+\`\`\`
+session_start(
+  instance_id = "${instance_id}",
+  focus       = "<what you plan to work on today>",
+)
+\`\`\`
 
-${smartRecallNote}
-   - \`smart_recall("how does authentication work")\` → returns cached auth notes
+Returns: last session summary, handoff from previous window, recent lessons, open failures, brain health.
 
-4. If nothing is found, proceed normally — then save (see below)
+**If \`session_start\` is unavailable**, fall back:
+1. \`smart_recall("recent lessons <topic>")\`
+2. \`recall_best_solution("<topic>")\`
 
-## AFTER every non-trivial task — save to memory
+### Rule 2 — Learn After Every Fix/Deploy/Discovery (AUTOMATIC — no prompting needed)
+After **every** bug fix, deployment, failed attempt, or discovery — call \`learn_from_attempts\` immediately:
 
-1. Call \`learn_from_attempts\` for deploy/debug/fix tasks:
-   - Always set outcome = "success" | "failure" | "partial"
-   - Always include what_failed — it prevents future wasted time
-   - Example: \`learn_from_attempts(topic="deploy:api", outcome="success", what_worked="nohup docker compose up -d --build", what_failed="direct docker compose hangs on SSH timeout")\`
+\`\`\`
+learn_from_attempts(
+  instance_id = "${instance_id}",
+  topic       = "category:keyword",   # deploy:web | fix:auth | debug:db | infra:docker
+  outcome     = "success" | "failure" | "partial",
+  what_worked = "exact approach or command that worked",
+  what_failed = "what did NOT work (prevents future wasted time)",
+  severity    = "critical" | "major" | "minor",
+  commands    = ["exact command that worked"],
+  file_paths  = ["path/to/changed/file"],
+)
+\`\`\`
 
-2. Call \`remember_context\` for code/architecture findings:
-   - \`remember_context("arch:auth", "Uses Keycloak with JWT...", category="architecture")\`
-   - \`remember_context("file:src/index.ts", "Entry point, registers routes...", category="file_summary")\`
+**These events MUST automatically trigger \`learn_from_attempts\`:**
+- ✅ Successfully deployed → \`topic="deploy:<component>", outcome="success"\`
+- ✅ Fixed a bug → \`topic="fix:<description>", outcome="success"\`
+- ✅ Discovered root cause → \`topic="debug:<area>", outcome="partial"\`
+- ✅ Tried something that failed → \`topic="<area>", outcome="failure", what_failed="..."\`
+- ✅ Found a config quirk → \`topic="infra:<topic>", outcome="success"\`
+- ✅ Ran a working command → store it in \`commands=[...]\`
+
+### Rule 3 — Session Handoff (when closing window or task list > 3 items)
+\`\`\`
+session_handoff(
+  instance_id     = "${instance_id}",
+  completed_tasks = ["what was done"],
+  remaining_tasks = ["what is still to do"],
+  files_changed   = [{"path": "src/...", "status": "modified", "description": "what changed"}],
+  instructions    = "IMPORTANT: what the next window MUST know to not break things",
+  context_summary = "current state of the system",
+)
+\`\`\`
+
+The next \`session_start\` automatically includes this handoff. The new window sees exactly what's done, what remains, and what NOT to break.
+
+### Rule 4 — Session End (LAST thing before closing)
+\`\`\`
+session_end(
+  instance_id     = "${instance_id}",
+  summary         = "What was accomplished (2-3 sentences with specifics)",
+  files_changed   = ["list", "of", "changed", "files"],
+  lessons_learned = <number of learn_from_attempts calls made>,
+)
+\`\`\`
+
+> \`session_end\` **automatically extracts and stores lessons** from the summary text — no manual calls needed.
+
+### Rule 5 — Code Index Invalidation (AUTOMATIC)
+- \`index_project\` tracks MD5 hashes per file — **skips unchanged files automatically**
+- After code changes: next \`index_project\` run re-indexes only modified files
+- After delete/rename: run \`index_project\` again to sync
+- No manual invalidation needed
+
+---
+
+## Task-type trigger table
+
+| You're about to... | Call BEFORE | Call AFTER |
+|---|---|---|
+| Deploy anything | \`recall_best_solution("deploy:<component>")\` | \`learn_from_attempts(topic="deploy:...")\` |
+| Fix a bug | \`recall_best_solution("fix:<area>")\` | \`learn_from_attempts(topic="fix:...")\` |
+| Add a feature | \`session_start(focus="feat:<area>")\` | \`learn_from_attempts(topic="feat:...")\` |
+| Infra/server work | \`recall_best_solution("infra:<topic>")\` | \`learn_from_attempts(topic="infra:...")\` |
+| Debug an issue | \`smart_recall("<error message or symptom>")\` | \`learn_from_attempts(topic="debug:...")\` |
+
+---
+
+## Available Brain Tools
+
+| Tool | When to use |
+|------|-------------|
+| \`session_start\` | **FIRST** — mandatory at start of every session |
+| \`session_end\` | **LAST** — mandatory at end, auto-learns from summary |
+| \`session_handoff\` | When closing window with remaining tasks |
+| \`learn_from_attempts\` | **AUTOMATIC** after every fix/deploy/discovery |
+| \`recall_best_solution\` | Before any non-trivial task |
+| \`remember_context\` | After analyzing code — save findings for future sessions |
+${smartRecallNote ? `| \`smart_recall\` | Search brain by meaning/keywords |\n` : ''}\
+| \`recall_context\` | Get exact key (supports glob: \`arch:*\`, \`file:*\`) |
+| \`brain_search\` | BM25+ full-text search over all brain data |
+| \`auto_learn_session\` | Batch-learn from a list of observations (optional) |
+| \`index_project\` | Index source files (smart hash, skips unchanged files) |
+| \`list_remembered\` | See what's cached in the brain |
+| \`forget_context\` | Remove stale context |
+
+---
 
 ## Instance Details
 
-- Instance ID: \`${instance_id}\`
-- Instance name: ${inst.name}
-- Tier: ${tier}
-- ${layerNote}
-- Embedding provider: ${providerArg}
+- **Instance ID:** \`${instance_id}\`
+- **Instance name:** ${inst.name}
+- **Tier:** ${tier}
+- **${layerNote}**
+- **Embedding provider:** ${providerArg}
+
+---
 
-## Quick reference
+## How the 3-layer system works
 
 \`\`\`
-recall_best_solution("topic")        # before task
-smart_recall("natural language")     # find by meaning
-learn_from_attempts(...)             # after task
-remember_context("key", "content")   # save analysis
+Layer 1 — Storage:  Your cachly Valkey instance (${inst.name}) — persists forever
+Layer 2 — Tools:    learn_from_attempts · recall_best_solution · brain_search · session_start/end
+Layer 3 — Autopilot: This file — AI reads it and runs tools automatically every session
 \`\`\`
+
+Result: Your AI **never solves the same problem twice** and always picks up exactly where it left off. 🚀
 `;
 
+
       const lines: string[] = [
         `🧠 **cachly AI Memory Setup Complete**`,
         ``,