clawmem 0.10.7 → 0.11.2

package/src/llm.ts

@@ -141,6 +141,88 @@ export type Queryable = {
   text: string;
 };
 
+/**
+ * Template-residue / leak patterns that indicate an expansion line is NOT a real
+ * query. The qmd-query-expansion finetune, when prompted out of distribution,
+ * echoed the old verbose prompt's format hints and leaked Qwen3 thinking tags;
+ * these never belong in a search query. Kept as a defensive guard even though the
+ * terse QMD-faithful prompt (expandQueryRemote, 2026-06-23) fixed generation.
+ */
+const EXPANSION_JUNK_PATTERNS: RegExp[] = [
+  /<\/?think>/i,
+  /keyword search terms \(/i,
+  /semantic search queries \(/i,
+  /hypothetical document passage that answers the query/i,
+];
+
+/** True if an expansion text is empty or matches a known template-residue pattern. */
+export function isJunkExpansion(text: string): boolean {
+  const t = text.trim();
+  if (!t) return true;
+  return EXPANSION_JUNK_PATTERNS.some(re => re.test(t));
+}
+
+/**
+ * Strip stray Qwen3 `/no_think` / `/think` control tokens the model sometimes
+ * echoes mid-line (generateRemote appends ` /no_think`, and the finetune can copy
+ * it into a hyde passage). Only matches the token when it stands alone (start/space
+ * bounded) so real paths like `src/think.ts` are never touched. Unlike the `<think>`
+ * tag (which signals reasoning leakage → whole line rejected), a stray control token
+ * just gets cleaned out so the otherwise-good line survives.
+ */
+function stripControlTokens(text: string): string {
+  return text.replace(/(?:^|\s)\/(?:no_)?think(?=\s|$)/gi, " ").replace(/\s+/g, " ").trim();
+}
+
+/**
+ * Shared guard for query-expansion output. Drops empty / template-residue /
+ * think-tag lines and de-duplicates by (type, text). Used by BOTH the llm.ts
+ * parsers and the store.ts wrapper (defense-in-depth across every provider).
+ * Does NOT inject a fallback — callers decide what to do with an empty result,
+ * and does NOT require the original query terms to appear (that would reject
+ * legitimate synonyms and keyword-only expansions, e.g. a future zegen lex leg).
+ */
+export function sanitizeExpandedQueries(items: Queryable[]): Queryable[] {
+  const seen = new Set<string>();
+  const out: Queryable[] = [];
+  for (const q of items) {
+    const text = stripControlTokens(q.text);
+    if (isJunkExpansion(text)) continue;
+    const key = `${q.type}:${text}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push({ type: q.type, text });
+  }
+  return out;
+}
+
+/**
+ * Typed fallback expansion set, used when generation fails or sanitization leaves
+ * nothing usable. lex+vec reuse the original query; hyde gets a minimal stub so the
+ * vector leg still has a hypothetical-document signal.
+ */
+export function expansionFallback(query: string, includeLexical: boolean = true): Queryable[] {
+  const out: Queryable[] = [
+    { type: 'vec', text: query },
+    { type: 'hyde', text: `Information about ${query}` },
+  ];
+  if (includeLexical) out.unshift({ type: 'lex', text: query });
+  return out;
+}
+
+/**
+ * True if `items` is exactly the typed fallback set for `query` — i.e. what every
+ * llm.expandQuery failure path returns. The store wrapper uses this to detect a
+ * leaked generation failure so it can return an expansions-only form WITHOUT
+ * caching it (a transient failure must not poison the cache). Compares against the
+ * default (lexical-included) fallback, which is what the store always requests.
+ */
+export function isFallbackExpansion(items: Queryable[], query: string): boolean {
+  const fb = expansionFallback(query);
+  return items.length === fb.length
+    && items.every((q, i) => q.type === fb[i]!.type && q.text === fb[i]!.text);
+}
+
 /**
  * Document to rerank
  */
@@ -290,6 +372,7 @@ function normalizeRemoteLlmReasoningEffort(value?: string): string | null {
 
 function buildRemoteChatCompletionsUrl(remoteLlmUrl: string): string {
   const baseUrl = remoteLlmUrl.replace(/\/+$/, "");
+  if (baseUrl.endsWith("/chat/completions")) return baseUrl;
   const endpoint = baseUrl.endsWith("/v1") ? "/chat/completions" : "/v1/chat/completions";
   return `${baseUrl}${endpoint}`;
 }
@@ -1073,24 +1156,22 @@ export class LlamaCpp implements LLM {
   }
 
   private async expandQueryRemote(query: string, includeLexical: boolean, context?: string, intent?: string): Promise<Queryable[]> {
-    const prompt = `Rewrite this search query for better retrieval. Output lines in format "type: text" where type is lex, vec, or hyde.
-- lex: keyword search terms (1-3 lines)
-- vec: semantic search queries (1-3 lines)
-- hyde: hypothetical document passage that answers the query (1 line)
-
-Query: ${query}${intent ? `\nQuery intent: ${intent}` : ""}${context ? `\nContext: ${context}` : ""}
-
-Output:`;
+    // QMD-faithful terse prompt. The qmd-query-expansion-1.7B finetune was trained
+    // on "/no_think Expand this search query: X" (cf. QMD src/llm.ts:1467). The prior
+    // verbose prose prompt was out-of-distribution: the model echoed the template
+    // ("lex: keyword search terms (") and leaked </think> (verified live 2026-06-23).
+    let prompt = intent
+      ? `/no_think Expand this search query: ${query}\nQuery intent: ${intent}`
+      : `/no_think Expand this search query: ${query}`;
+    if (context) prompt += `\nContext: ${context}`;
 
     const result = await this.generateRemote(prompt, 500, 0.7);
     if (!result?.text) {
-      const fallback: Queryable[] = [{ type: 'vec', text: query }];
-      if (includeLexical) fallback.unshift({ type: 'lex', text: query });
-      return fallback;
+      return expansionFallback(query, includeLexical);
     }
 
     const lines = result.text.trim().split("\n");
-    const queryables: Queryable[] = lines.map(line => {
+    const parsed: Queryable[] = lines.map(line => {
       const colonIdx = line.indexOf(":");
       if (colonIdx === -1) return null;
       const type = line.slice(0, colonIdx).trim();
@@ -1100,16 +1181,11 @@ Output:`;
       return { type: type as QueryType, text };
     }).filter((q): q is Queryable => q !== null);
 
-    if (queryables.length === 0) {
-      const fallback: Queryable[] = [{ type: 'vec', text: query }];
-      if (includeLexical) fallback.unshift({ type: 'lex', text: query });
-      return fallback;
-    }
-
-    if (!includeLexical) {
-      return queryables.filter(q => q.type !== 'lex');
-    }
-    return queryables;
+    // Drop template residue / <think> leaks / dups, then scope to requested types.
+    const cleaned = sanitizeExpandedQueries(parsed);
+    const scoped = includeLexical ? cleaned : cleaned.filter(q => q.type !== 'lex');
+    if (scoped.length === 0) return expansionFallback(query, includeLexical);
+    return scoped;
   }
 
   async modelExists(modelUri: string): Promise<ModelInfo> {
@@ -1147,10 +1223,8 @@ Output:`;
     // Remote is in cooldown (pre-existing or just set) — fall through to local
     if (this.remoteLlmUrl && this.isRemoteLlmDown()) {
       if (process.env.CLAWMEM_NO_LOCAL_MODELS === "true") {
-        // Can't fall back — return passthrough
-        const fallback: Queryable[] = [{ type: 'vec', text: query }];
-        if (includeLexical) fallback.unshift({ type: 'lex', text: query });
-        return fallback;
+        // Can't fall back to local inference — return the typed passthrough set
+        return expansionFallback(query, includeLexical);
       }
       this.noteRemoteFallback(
         "llm",
@@ -1229,7 +1303,7 @@ Final Output:`;
       });
 
       const lines = result.trim().split("\n");
-      const queryables: Queryable[] = lines.map(line => {
+      const parsed: Queryable[] = lines.map(line => {
         const colonIdx = line.indexOf(":");
         if (colonIdx === -1) return null;
         const type = line.slice(0, colonIdx).trim();
@@ -1238,17 +1312,14 @@ Final Output:`;
         return { type: type as QueryType, text };
       }).filter((q): q is Queryable => q !== null);
 
-      // Filter out lex entries if not requested
-      if (!includeLexical) {
-        return queryables.filter(q => q.type !== 'lex');
-      }
-      return queryables;
+      // Same guard as the remote path — drop residue/dups, scope to requested types.
+      const cleaned = sanitizeExpandedQueries(parsed);
+      const scoped = includeLexical ? cleaned : cleaned.filter(q => q.type !== 'lex');
+      if (scoped.length === 0) return expansionFallback(query, includeLexical);
+      return scoped;
     } catch (error) {
       console.error("Structured query expansion failed:", error);
-      // Fallback to original query
-      const fallback: Queryable[] = [{ type: 'vec', text: query }];
-      if (includeLexical) fallback.unshift({ type: 'lex', text: query });
-      return fallback;
+      return expansionFallback(query, includeLexical);
     } finally {
       await genContext.dispose();
     }

package/src/mcp.ts

@@ -629,19 +629,38 @@ This is the recommended entry point for ALL memory queries.`,
       const hasStrongSignal = !intent && initialFts.length > 0
         && topScore >= 0.85 && (topScore - secondScore) >= 0.15;
 
-      // Step 2: Query expansion (skipped if strong signal)
-      const queries = hasStrongSignal
-        ? [query]
+      // Step 2: Query expansion (skipped if strong signal). Typed routing —
+      // original → BOTH FTS + vector (2× RRF anchor), lex → FTS only, vec/hyde → vector only.
+      const expanded = hasStrongSignal
+        ? []
         : await store.expandQuery(query, DEFAULT_QUERY_MODEL, intent);
 
-      for (const q of queries) {
-        const ftsResults = q === query ? initialFts : store.searchFTS(q, 20, undefined, collections, dateRange);
-        if (ftsResults.length > 0) {
-          for (const r of ftsResults) docidMap.set(r.filepath, r.docid);
-          rankedLists.push(ftsResults.map(r => ({ file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score })));
+      // Original query — both backends, pushed FIRST so the positional 2× weight
+      // below lands on exactly the original's lists.
+      if (initialFts.length > 0) {
+        for (const r of initialFts) docidMap.set(r.filepath, r.docid);
+        rankedLists.push(initialFts.map(r => ({ file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score })));
+      }
+      if (hasVectors) {
+        const vecResults = await store.searchVec(query, DEFAULT_EMBED_MODEL, 20, undefined, collections, dateRange);
+        if (vecResults.length > 0) {
+          for (const r of vecResults) docidMap.set(r.filepath, r.docid);
+          rankedLists.push(vecResults.map(r => ({ file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score })));
         }
-        if (hasVectors) {
-          const vecResults = await store.searchVec(q, DEFAULT_EMBED_MODEL, 20, undefined, collections, dateRange);
+      }
+      // Lists contributed by the original query — these get the 2× RRF weight.
+      const numOriginalLists = rankedLists.length;
+
+      // Typed expansions — route by type: lex → FTS, vec/hyde → vector.
+      for (const eq of expanded) {
+        if (eq.type === 'lex') {
+          const ftsResults = store.searchFTS(eq.query, 20, undefined, collections, dateRange);
+          if (ftsResults.length > 0) {
+            for (const r of ftsResults) docidMap.set(r.filepath, r.docid);
+            rankedLists.push(ftsResults.map(r => ({ file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score })));
+          }
+        } else if (hasVectors) {
+          const vecResults = await store.searchVec(eq.query, DEFAULT_EMBED_MODEL, 20, undefined, collections, dateRange);
           if (vecResults.length > 0) {
             for (const r of vecResults) docidMap.set(r.filepath, r.docid);
             rankedLists.push(vecResults.map(r => ({ file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score })));
@@ -705,8 +724,9 @@ This is the recommended entry point for ALL memory queries.`,
         }
       }
 
-      // Weight: original query BM25+vec get 2x, expanded queries get 1x, temporal/entity legs get 1x
-      const numOriginalLists = hasVectors ? 2 : 1; // first BM25 + first vector from original query
+      // Weight: the original query's lists (pushed first) get 2×; expansion, temporal,
+      // and entity legs get 1×. numOriginalLists (computed above) is the actual count
+      // the original contributed — robust to an empty BM25 or vector leg.
       const weights = rankedLists.map((_, i) => i < numOriginalLists ? 2.0 : 1.0);
       const fused = reciprocalRankFusion(rankedLists, weights);
       const candidates = fused.slice(0, candLimit);