@comfanion/usethis_search 4.3.0-dev.4 → 4.4.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@comfanion/usethis_search",
-  "version": "4.3.0-dev.4",
-  "description": "OpenCode plugin: semantic search with context-efficient workspace state (v4.3: no injection, each tool returns full state inline, auto-prune history, auto-detect modes, line numbers, LSP memory leak fixed)",
+  "version": "4.4.0",
+  "description": "OpenCode plugin: semantic search with query decomposition, RRF merge, and context-efficient workspace (v4.4.0)",
   "type": "module",
   "main": "./index.ts",
   "exports": {
@@ -41,6 +41,7 @@
     "vectorizer/graph-db.ts",
     "vectorizer/chunk-store.ts",
     "vectorizer/usage-tracker.ts",
+    "vectorizer/query-decomposer.ts",
     "vectorizer/graph-builder.ts",
     "vectorizer/analyzers/regex-analyzer.ts",
     "vectorizer/analyzers/lsp-analyzer.ts",

package/tools/search.ts

@@ -13,9 +13,10 @@ import { tool } from "@opencode-ai/plugin"
 import path from "path"
 import fs from "fs/promises"
 
-import { CodebaseIndexer, getSearchConfig, getIndexer, releaseIndexer } from "../vectorizer/index.ts"
+import { CodebaseIndexer, getSearchConfig, getDecomposerConfig, getIndexer, releaseIndexer } from "../vectorizer/index.ts"
 import { workspaceCache } from "../cache/manager.ts"
 import { buildWorkspaceOutput } from "./workspace-state.ts"
+import { decomposeQuery } from "../vectorizer/query-decomposer.ts"
 
 // ── Context Expansion Helpers ─────────────────────────────────────────────
 
@@ -179,30 +180,55 @@ function parseFilter(filter: string): {
 }
 
 export default tool({
-  description: `Search codebase and automatically attach relevant context to workspace.
-
-Accepts any query - semantic search, file path, or chunk ID:
-- "authentication logic" → finds relevant code
-- "docs/architecture.md" → attaches file
-- "src/auth.ts:chunk-5" → attaches specific chunk
-
-Results are optimized for context - top chunks auto-attached with expanded context 
-(related code, imports, class methods). Returns full workspace state inline.
-Previous search outputs are automatically pruned from history.
-
-IMPORTANT: Workspace has limited token budget. Use workspace_forget() to remove 
-irrelevant files or old searches before adding new context.
-
-Choose index based on what you're looking for:
-- index: "code" → search source code
-- index: "docs" → search documentation  
-- searchAll: true → search everywhere
+  description: `Search the codebase semantically. Use this to find relevant code snippets, functions, or files based on meaning, not just text matching.
+
+Available indexes:
+- "code" (default) - Source code files (*.js, *.ts, *.py, *.go, etc.)
+- "docs" - Documentation files (*.md, *.txt, etc.)
+- "config" - Configuration files (*.yaml, *.json, etc.)
+- searchAll: true - Search across all indexes
+
+Auto-detects query type:
+- Semantic: "authentication logic" → vector search for relevant code
+- File path: "docs/architecture.md" → attaches entire file to workspace
+- Chunk ID: "src/auth.ts:chunk-5" → attaches specific chunk
+
+How workspace works:
+- Top results are AUTO-ATTACHED to workspace with expanded context (class methods, imports, related code via graph)
+- Workspace has a TOKEN BUDGET (~50K tokens, ~100 chunks). When full, oldest chunks are evicted
+- Each search call returns full <workspace_state> with all chunk contents inline
+- Only the LATEST search/workspace output is kept in chat history — older ones are auto-pruned
+- Workspace persists across searches — new results ADD to existing workspace
+
+IMPORTANT: Chunks contain DIRECT file content dumps (raw code/text from files).
+- You DO NOT need to verify chunk content with grep/read tools
+- Chunks are already the actual file content, not summaries or references
+- Trust the chunk content as the source of truth
+- Use Read tool only if you need content OUTSIDE the indexed chunks
+
+Context management (CRITICAL — follow these rules):
+- BEFORE searching a new topic, you MUST call workspace_forget() to remove irrelevant old context
+- Workspace has LIMITED token budget. If budget >60%, evict old chunks with workspace_forget({ what: "5" })
+- Use workspace_clear() when switching to a completely different task
+- After editing files, forget stale chunks: workspace_forget({ what: "edited-file.ts" })
+- The workspace is your working memory — KEEP IT FOCUSED. Stale context degrades search quality
+- Rule of thumb: forget BEFORE you search, not after
+
+Filter narrows results by path or language:
+- "internal/domain/" → only files under that path
+- "*.go" → only Go files
+- "internal/**/*.go" → path + language combined
+- "service" → files containing "service" in path
 
 Examples:
 - search({ query: "authentication logic" })
 - search({ query: "how to deploy", index: "docs" })
-- search({ query: "docs/prd.md" })  // attach file
-- search({ query: "internal/domain/", filter: "*.go" })`,
+- search({ query: "tenant management", filter: "internal/domain/" })
+- search({ query: "event handling", filter: "*.go" })
+- search({ query: "API routes", filter: "internal/**/*.go" })
+- search({ query: "metrics", searchAll: true })
+- search({ query: "docs/prd.md" })
+- search({ query: "src/auth.ts:chunk-5" })`,
 
   args: {
     query: tool.schema.string().describe("What to search: semantic query, file path, or chunk ID"),
@@ -657,10 +683,17 @@ Examples:
 
        const topScore = topChunks[0]?._finalScore ?? 0
       const hasBM25Only = allResults.some((r: any) => r._bm25Only)
+      const hasRRF = allResults.some((r: any) => r._rrfScore != null)
       const scope = args.searchAll ? "all indexes" : `index "${indexName}"`
       const filterLabel = args.filter ? ` filter:"${args.filter}"` : ""
       let output = `## Search: "${semanticQuery}" (${scope}${filterLabel})\n\n`
 
+      // Show decomposition info if query was decomposed
+      const decomposition = decomposeQuery(semanticQuery!, getDecomposerConfig())
+      if (decomposition.decomposed) {
+        output += `> **Query decomposed** (${decomposition.strategy}): ${decomposition.subQueries.map(q => `"${q}"`).join(", ")}\n\n`
+      }
+
       if (hasBM25Only) {
         output += `> **BM25-only mode** -- vector embeddings not yet available. Quality will improve after embedding completes.\n\n`
       }

package/tools/workspace.ts

@@ -20,7 +20,15 @@ import { buildWorkspaceOutput } from "./workspace-state.ts"
 // ── workspace.list ──────────────────────────────────────────────────────────
 
 export const workspace_list = tool({
-  description: `Show full workspace state with all chunk content. Returns file listing and inline content for every attached chunk.`,
+  description: `Show current workspace contents — all attached code chunks with full source code, line numbers, and metadata.
+
+Use this to:
+- Check what context is currently loaded after compaction or session restore
+- Verify workspace contents before starting implementation
+- See token budget usage (how much space is left for new searches)
+
+Returns <workspace_state> with every chunk's full content. This is the same state appended to every search() call.
+Only the LATEST workspace tool output is kept in chat — older outputs are auto-pruned.`,
 
   args: {},
 
@@ -37,6 +45,13 @@ export const workspace_forget = tool({
 IMPORTANT: Regularly clean up workspace by removing irrelevant files or old search results. 
 This keeps context focused and prevents token budget overflow.
 
+WHEN TO CLEAN UP:
+- BEFORE searching a new topic — forget the previous search results first:
+  workspace_forget({ what: "previous search query" }) → then search({ query: "new topic" })
+- AFTER finishing a subtask — forget files you no longer need
+- WHEN budget >60% — evict old chunks: workspace_forget({ what: "5" })
+- AFTER editing files — workspace chunks become stale, forget and re-search
+
 Auto-detects what to remove based on input:
 - Chunk ID: "src/auth.ts:chunk-5"
 - File path: "docs/architecture.md" (removes ALL chunks)
@@ -46,7 +61,8 @@ Auto-detects what to remove based on input:
 Examples:
 - workspace_forget({ what: "docs/prd.md" })
 - workspace_forget({ what: "5" })  // older than 5 min
-- workspace_forget({ what: "src/auth.ts:chunk-3" })`,
+- workspace_forget({ what: "src/auth.ts:chunk-3" })
+- workspace_forget({ what: "authentication logic" })  // forget previous search`,
 
   args: {
     what: tool.schema.string().describe("What to forget: chunk ID, file path, search query, or age in minutes"),
@@ -110,7 +126,15 @@ Examples:
 // ── workspace.clear ─────────────────────────────────────────────────────────
 
 export const workspace_clear = tool({
-  description: `Remove ALL chunks from workspace context. Use when switching tasks or starting fresh.`,
+  description: `Remove ALL chunks from workspace context. Use when switching tasks or starting fresh.
+
+Use when:
+- Switching to a completely different task or topic
+- Workspace is cluttered with irrelevant context from many searches
+- Starting a fresh investigation from scratch
+
+Prefer workspace_forget() for selective cleanup. Use workspace_clear() only for full reset.
+Returns empty workspace state.`,
 
   args: {},
 
@@ -126,7 +150,15 @@ export const workspace_clear = tool({
 // ── workspace.restore ───────────────────────────────────────────────────────
 
 export const workspace_restore = tool({
-  description: `Restore workspace from a saved session snapshot. Use after compaction or to switch context.`,
+  description: `Restore workspace from a previously saved session snapshot.
+
+Use when:
+- After compaction — restore the workspace context from before compaction
+- Resuming work on a previous task — switch back to that context
+- After workspace_clear() — if you need the old context back
+
+Call without sessionId to list available snapshots with their chunk counts and token sizes.
+Call with sessionId to restore a specific snapshot. Replaces current workspace entirely.`,
 
   args: {
     sessionId: tool.schema.string().optional().describe("Session ID to restore. If not provided, lists available snapshots."),

package/vectorizer/index.ts

@@ -19,6 +19,8 @@ import { GraphDB } from "./graph-db.ts";
 import { GraphBuilder, isStructuralPredicate } from "./graph-builder.ts";
 import { UsageTracker } from "./usage-tracker.ts";
 import { ChunkStore } from "./chunk-store.ts";
+import { decomposeQuery, rrfMerge, DEFAULT_DECOMPOSER_CONFIG } from "./query-decomposer.ts";
+import type { DecomposerConfig } from "./query-decomposer.ts";
 
 // Suppress transformers.js logs unless DEBUG is set
 const DEBUG = process.env.DEBUG?.includes("vectorizer") || process.env.DEBUG === "*";
@@ -86,6 +88,9 @@ let HYBRID_CONFIG = { ...DEFAULT_HYBRID_CONFIG };
 let METRICS_ENABLED = false;
 let CACHE_ENABLED = true;
 
+// ── Query decomposition config ───────────────────────────────────────────────
+let DECOMPOSER_CONFIG: DecomposerConfig = { ...DEFAULT_DECOMPOSER_CONFIG };
+
 // ── Search defaults (exposed to tool layer) ──────────────────────────────────
 const DEFAULT_SEARCH_CONFIG = {
   freshen: false,           // Don't freshen on every search — auto_index handles it
@@ -188,6 +193,13 @@ function defaultVectorizerYaml() {
     `    auto_prune_search: true      # Replace old search outputs with compact summaries\n` +
     `    substitute_tool_outputs: true # Replace tool outputs when files in workspace\n` +
     `\n` +
+    `  # Query decomposition (v4 — improves long query relevance)\n` +
+    `  decomposition:\n` +
+    `    enabled: true              # Split complex queries into focused sub-queries\n` +
+    `    min_words: 5               # Min significant words to trigger decomposition\n` +
+    `    max_sub_queries: 4         # Max sub-queries (including keyword core)\n` +
+    `    min_sub_query_words: 2     # Min words per sub-query\n` +
+    `\n` +
     `  # Quality monitoring\n` +
     `  quality:\n` +
     `    enable_metrics: false\n` +
@@ -370,6 +382,17 @@ async function loadConfig(projectRoot) {
       CACHE_ENABLED = parseBool(qs, "enable_cache", true);
     }
 
+    // ── Parse query decomposition config ────────────────────────────────────
+    const decomposerMatch = section.match(/^\s{2}decomposition:\s*\n([\s\S]*?)(?=^\s{2}[a-zA-Z_\-]+:|(?![\s\S]))/m);
+    if (decomposerMatch) {
+      const ds = decomposerMatch[1];
+      DECOMPOSER_CONFIG.enabled = parseBool(ds, "enabled", DEFAULT_DECOMPOSER_CONFIG.enabled);
+      DECOMPOSER_CONFIG.minWords = parseNumber(ds, "min_words", DEFAULT_DECOMPOSER_CONFIG.minWords);
+      DECOMPOSER_CONFIG.maxSubQueries = parseNumber(ds, "max_sub_queries", DEFAULT_DECOMPOSER_CONFIG.maxSubQueries);
+      DECOMPOSER_CONFIG.minSubQueryWords = parseNumber(ds, "min_sub_query_words", DEFAULT_DECOMPOSER_CONFIG.minSubQueryWords);
+      if (DEBUG) console.log("[vectorizer] Decomposer config:", DECOMPOSER_CONFIG);
+    }
+
     // ── Parse graph config (v3) ──────────────────────────────────────────────
     const graphMatch = section.match(/^\s{2}graph:\s*\n([\s\S]*?)(?=^\s{2}[a-zA-Z_\-]+:|(?![\s\S]))/m);
     if (graphMatch) {
@@ -1121,9 +1144,9 @@ class CodebaseIndexer {
     }
   }
 
-  // ── Search (v3: hybrid + BM25-only fallback + metadata filters + metrics) ──
+  // ── Single-query search (internal — used by search() for each sub-query) ──
 
-  async search(query, limit = 5, includeArchived = false, options = {}) {
+  async _searchSingle(query, limit = 5, includeArchived = false, options = {}) {
     const tableName = "chunks";
     const tables = await this.db.tableNames();
 
@@ -1178,14 +1201,9 @@ class CodebaseIndexer {
         }
       }
 
-      // Apply metadata filters then return (graph context added below)
+      // Apply metadata filters then return
       results = this._applyMetadataFilters(results, includeArchived, options);
-      const finalResults = results.slice(0, limit);
-
-      // Graph context expansion (same as vector path)
-      await this._expandGraphContext(finalResults, null, query);
-
-      return finalResults;
+      return results.slice(0, limit);
     }
 
     // ── Vector search (Phase 2 complete) ─────────────────────────────────────
@@ -1280,7 +1298,51 @@ class CodebaseIndexer {
 
     // ── Metadata filters ──────────────────────────────────────────────────
     results = this._applyMetadataFilters(results, includeArchived, options);
-    const finalResults = results.slice(0, limit);
+    return results.slice(0, limit);
+  }
+
+  // ── Search (v4: query decomposition + RRF merge + hybrid + metrics) ────────
+
+  async search(query, limit = 5, includeArchived = false, options = {}) {
+    // ── Query decomposition ──────────────────────────────────────────────────
+    const decomposition = decomposeQuery(query, DECOMPOSER_CONFIG);
+
+    let finalResults;
+
+    if (decomposition.decomposed && decomposition.subQueries.length > 1) {
+      if (DEBUG) {
+        console.log(`[vectorizer] Query decomposed (${decomposition.strategy}): ${decomposition.subQueries.length} sub-queries`);
+        for (const sq of decomposition.subQueries) {
+          console.log(`  → "${sq}"`);
+        }
+      }
+
+      // Run each sub-query independently, over-fetch to give RRF more signal
+      const perQueryLimit = Math.max(limit * 2, 20);
+      const resultSets = [];
+
+      for (const subQuery of decomposition.subQueries) {
+        const results = await this._searchSingle(subQuery, perQueryLimit, includeArchived, options);
+        if (results.length > 0) {
+          resultSets.push(results);
+        }
+      }
+
+      if (resultSets.length === 0) {
+        finalResults = [];
+      } else if (resultSets.length === 1) {
+        finalResults = resultSets[0].slice(0, limit);
+      } else {
+        // RRF merge across sub-query result sets
+        finalResults = rrfMerge(resultSets, 60, limit);
+        if (DEBUG) {
+          console.log(`[vectorizer] RRF merged ${resultSets.length} result sets → ${finalResults.length} results`);
+        }
+      }
+    } else {
+      // Short/simple query — single search (no decomposition overhead)
+      finalResults = await this._searchSingle(query, limit, includeArchived, options);
+    }
 
     // ── Metrics tracking ────────────────────────────────────────────────────
     if (METRICS_ENABLED) {
@@ -1304,6 +1366,8 @@ class CodebaseIndexer {
     }
 
     // ── Graph context expansion (v3) ───────────────────────────────────────
+    // Use original query for graph expansion (most complete context)
+    const queryEmbedding = finalResults.length > 0 ? await this.embedQuery(query).catch(() => null) : null;
     await this._expandGraphContext(finalResults, queryEmbedding, query);
 
     return finalResults;
@@ -1826,4 +1890,8 @@ async function destroyIndexer(projectRoot: string, indexName: string = "code") {
   }
 }
 
-export { CodebaseIndexer, INDEX_PRESETS, getEmbeddingModel, getSearchConfig, getWorkspaceConfig, getIndexer, releaseIndexer, destroyIndexer };
+function getDecomposerConfig() {
+  return DECOMPOSER_CONFIG;
+}
+
+export { CodebaseIndexer, INDEX_PRESETS, getEmbeddingModel, getSearchConfig, getWorkspaceConfig, getDecomposerConfig, getIndexer, releaseIndexer, destroyIndexer };

package/vectorizer/query-decomposer.ts

@@ -0,0 +1,397 @@
+/**
+ * Query Decomposer — splits complex queries into focused sub-queries.
+ *
+ * Problem: Long, multi-concept queries produce "diluted" embeddings
+ * because the embedding model (all-MiniLM-L6-v2, 384d) averages all
+ * token vectors into one. "JWT authentication middleware that validates
+ * permissions" → a blurry vector between auth, JWT, middleware, permissions.
+ *
+ * Solution: Decompose into focused sub-queries, search each independently,
+ * merge results via Reciprocal Rank Fusion (RRF).
+ *
+ * Strategy (no LLM — pure heuristics):
+ * 1. Short queries (≤4 significant words) → pass through unchanged
+ * 2. Medium queries (5-8 words) → extract keyword core + original
+ * 3. Long queries (9+ words) → split into 2-4 concept clusters + keyword core
+ *
+ * All decomposition is deterministic and fast (<1ms).
+ */
+
+// ── Types ───────────────────────────────────────────────────────────────────
+
+export interface DecompositionResult {
+  /** Original query (always included in sub-queries) */
+  original: string
+  /** Focused sub-queries (includes original if short enough) */
+  subQueries: string[]
+  /** Whether decomposition was applied */
+  decomposed: boolean
+  /** Strategy used */
+  strategy: "passthrough" | "keyword-core" | "concept-split"
+}
+
+export interface DecomposerConfig {
+  /** Enable/disable decomposition */
+  enabled: boolean
+  /** Min significant words to trigger decomposition */
+  minWords: number
+  /** Max sub-queries to generate (including original) */
+  maxSubQueries: number
+  /** Min words per sub-query */
+  minSubQueryWords: number
+}
+
+export const DEFAULT_DECOMPOSER_CONFIG: DecomposerConfig = {
+  enabled: true,
+  minWords: 5,
+  maxSubQueries: 4,
+  minSubQueryWords: 2,
+}
+
+// ── Stop words (shared with BM25 + extras for query context) ────────────────
+
+const STOP_WORDS = new Set([
+  "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
+  "have", "has", "had", "do", "does", "did", "will", "would", "could",
+  "should", "may", "might", "shall", "can", "need", "must",
+  "and", "or", "but", "not", "no", "nor",
+  "in", "on", "at", "to", "for", "of", "with", "by", "from", "as",
+  "into", "about", "between", "through", "during", "before", "after",
+  "this", "that", "these", "those", "it", "its",
+  "i", "you", "he", "she", "we", "they", "me", "him", "her", "us", "them",
+  "my", "your", "his", "our", "their",
+  "what", "which", "who", "whom", "where", "when", "how", "why",
+  "if", "then", "else", "so", "than", "too", "very",
+  // Query-specific stop words (common in agent queries)
+  "find", "search", "look", "show", "get", "give", "tell",
+  "using", "used", "uses", "use",
+  "like", "such", "also", "just", "only",
+  "all", "any", "each", "every", "some",
+  "code", "file", "files", "function", "class", "method",
+  "implement", "implementation", "implements", "implemented",
+  "related", "relevant", "similar",
+  "please", "help", "want", "need",
+])
+
+// ── Connectors that signal concept boundaries ───────────────────────────────
+
+const CONCEPT_CONNECTORS = new Set([
+  "and", "or", "that", "which", "where", "when", "while",
+  "with", "using", "through", "via", "for", "including",
+  "also", "both", "either", "neither",
+])
+
+// ── Domain compound terms (keep together) ───────────────────────────────────
+
+const COMPOUND_TERMS: Array<[string, string]> = [
+  ["error", "handling"],
+  ["event", "sourcing"],
+  ["dependency", "injection"],
+  ["access", "control"],
+  ["rate", "limiting"],
+  ["load", "balancing"],
+  ["unit", "test"],
+  ["integration", "test"],
+  ["api", "endpoint"],
+  ["api", "gateway"],
+  ["data", "model"],
+  ["data", "transfer"],
+  ["database", "connection"],
+  ["file", "system"],
+  ["message", "queue"],
+  ["state", "management"],
+  ["type", "checking"],
+  ["code", "review"],
+  ["pull", "request"],
+  ["design", "pattern"],
+  ["repository", "pattern"],
+  ["factory", "pattern"],
+  ["observer", "pattern"],
+  ["middleware", "chain"],
+  ["call", "hierarchy"],
+  ["graph", "traversal"],
+]
+
+// ── Tokenizer ───────────────────────────────────────────────────────────────
+
+/**
+ * Tokenize query into lowercase words, preserving compound terms.
+ */
+export function tokenizeQuery(query: string): string[] {
+  const raw = query
+    .toLowerCase()
+    .replace(/[^a-z0-9_\-]/g, " ")
+    .split(/\s+/)
+    .filter(t => t.length > 1)
+
+  // Merge compound terms
+  const merged: string[] = []
+  let i = 0
+  while (i < raw.length) {
+    let found = false
+    if (i < raw.length - 1) {
+      for (const [a, b] of COMPOUND_TERMS) {
+        if (raw[i] === a && raw[i + 1] === b) {
+          merged.push(`${a}_${b}`)
+          i += 2
+          found = true
+          break
+        }
+      }
+    }
+    if (!found) {
+      merged.push(raw[i])
+      i++
+    }
+  }
+
+  return merged
+}
+
+/**
+ * Extract significant (non-stop) words from token list.
+ */
+export function extractSignificant(tokens: string[]): string[] {
+  return tokens.filter(t => !STOP_WORDS.has(t) && t.length > 2)
+}
+
+// ── Concept Clustering ──────────────────────────────────────────────────────
+
+/**
+ * Split tokens into concept groups at connector boundaries.
+ *
+ * "JWT authentication middleware that validates user permissions for API endpoints"
+ * → ["JWT authentication middleware", "validates user permissions", "API endpoints"]
+ */
+export function splitByConcepts(tokens: string[]): string[][] {
+  const groups: string[][] = []
+  let current: string[] = []
+
+  for (const token of tokens) {
+    if (CONCEPT_CONNECTORS.has(token)) {
+      if (current.length > 0) {
+        groups.push(current)
+        current = []
+      }
+      // Skip the connector itself
+    } else {
+      current.push(token)
+    }
+  }
+
+  if (current.length > 0) {
+    groups.push(current)
+  }
+
+  return groups
+}
+
+/**
+ * Merge small concept groups with neighbors to meet minimum size.
+ */
+function mergeSmallGroups(groups: string[][], minSize: number): string[][] {
+  if (groups.length <= 1) return groups
+
+  const merged: string[][] = []
+  let buffer: string[] = []
+
+  for (const group of groups) {
+    buffer.push(...group)
+    // Extract significant words to check if buffer is "big enough"
+    const sig = extractSignificant(buffer)
+    if (sig.length >= minSize) {
+      merged.push([...buffer])
+      buffer = []
+    }
+  }
+
+  // Remaining buffer: merge with last group or push as-is
+  if (buffer.length > 0) {
+    if (merged.length > 0) {
+      merged[merged.length - 1].push(...buffer)
+    } else {
+      merged.push(buffer)
+    }
+  }
+
+  return merged
+}
+
+// ── Keyword Core Extraction ─────────────────────────────────────────────────
+
+/**
+ * Extract a "keyword core" — the most important 3-4 words from the query.
+ * Uses a simple heuristic: take significant words, prefer longer/rarer ones.
+ */
+export function extractKeywordCore(significant: string[], maxWords: number = 3): string {
+  // Score words: longer words and compound terms score higher
+  const scored = significant.map(w => ({
+    word: w,
+    score: w.length + (w.includes("_") ? 5 : 0),
+  }))
+
+  scored.sort((a, b) => b.score - a.score)
+  const top = scored.slice(0, maxWords).map(s => s.word)
+
+  // Restore original order
+  const ordered = significant.filter(w => top.includes(w))
+  return ordered.slice(0, maxWords).join(" ").replace(/_/g, " ")
+}
+
+// ── Main Decomposer ─────────────────────────────────────────────────────────
+
+/**
+ * Decompose a search query into focused sub-queries.
+ *
+ * @param query  The original search query
+ * @param config Decomposer configuration
+ * @returns DecompositionResult with sub-queries and metadata
+ */
+export function decomposeQuery(
+  query: string,
+  config: DecomposerConfig = DEFAULT_DECOMPOSER_CONFIG,
+): DecompositionResult {
+  if (!config.enabled) {
+    return {
+      original: query,
+      subQueries: [query],
+      decomposed: false,
+      strategy: "passthrough",
+    }
+  }
+
+  const tokens = tokenizeQuery(query)
+  const significant = extractSignificant(tokens)
+
+  // ── Strategy 1: Short query → passthrough ─────────────────────────────────
+  if (significant.length < config.minWords) {
+    return {
+      original: query,
+      subQueries: [query],
+      decomposed: false,
+      strategy: "passthrough",
+    }
+  }
+
+  // ── Strategy 2: Medium query (5-8 significant words) → keyword core ───────
+  if (significant.length <= 8) {
+    const core = extractKeywordCore(significant, 3)
+    const subQueries = [query]
+
+    // Only add core if it's meaningfully different from original
+    if (core !== query.toLowerCase().trim() && core.split(" ").length >= config.minSubQueryWords) {
+      subQueries.push(core)
+    }
+
+    return {
+      original: query,
+      subQueries: subQueries.slice(0, config.maxSubQueries),
+      decomposed: subQueries.length > 1,
+      strategy: subQueries.length > 1 ? "keyword-core" : "passthrough",
+    }
+  }
+
+  // ── Strategy 3: Long query (9+ significant words) → concept split ─────────
+  const conceptGroups = splitByConcepts(tokens)
+  const mergedGroups = mergeSmallGroups(conceptGroups, config.minSubQueryWords)
+
+  const subQueries: string[] = []
+
+  // Always include keyword core as first sub-query (highest signal)
+  const core = extractKeywordCore(significant, 4)
+  if (core.split(" ").length >= config.minSubQueryWords) {
+    subQueries.push(core)
+  }
+
+  // Add concept groups as sub-queries
+  for (const group of mergedGroups) {
+    const groupSig = extractSignificant(group)
+    if (groupSig.length >= config.minSubQueryWords) {
+      const subQuery = groupSig.join(" ").replace(/_/g, " ")
+      // Avoid duplicates
+      if (!subQueries.includes(subQuery)) {
+        subQueries.push(subQuery)
+      }
+    }
+  }
+
+  // If we still have room, add the original (truncated to first N significant words)
+  if (subQueries.length < config.maxSubQueries) {
+    const truncated = significant.slice(0, 6).join(" ").replace(/_/g, " ")
+    if (!subQueries.includes(truncated)) {
+      subQueries.push(truncated)
+    }
+  }
+
+  // Ensure we don't exceed max
+  const finalQueries = subQueries.slice(0, config.maxSubQueries)
+
+  return {
+    original: query,
+    subQueries: finalQueries.length > 0 ? finalQueries : [query],
+    decomposed: finalQueries.length > 1,
+    strategy: finalQueries.length > 1 ? "concept-split" : "passthrough",
+  }
+}
+
+// ── RRF Merge ───────────────────────────────────────────────────────────────
+
+/**
+ * Reciprocal Rank Fusion — merge ranked result lists from multiple sub-queries.
+ *
+ * RRF score = sum(1 / (k + rank_i)) for each sub-query where the result appears.
+ *
+ * @param resultSets  Array of result arrays, each sorted by relevance (best first)
+ * @param k           RRF constant (default: 60, standard value from the paper)
+ * @param limit       Max results to return
+ * @returns Merged results sorted by RRF score, with _rrfScore and _combinedScore set
+ */
+export function rrfMerge(
+  resultSets: Array<Array<Record<string, any>>>,
+  k: number = 60,
+  limit: number = 10,
+): Array<Record<string, any>> {
+  if (resultSets.length === 0) return []
+  if (resultSets.length === 1) return resultSets[0].slice(0, limit)
+
+  // Build RRF scores keyed by chunk identity (file:chunk_index)
+  const scoreMap = new Map<string, { row: Record<string, any>; rrfScore: number; bestOriginalScore: number }>()
+
+  for (const results of resultSets) {
+    for (let rank = 0; rank < results.length; rank++) {
+      const r = results[rank]
+      const key = `${r.file}:${r.chunk_index}`
+      const rrfContribution = 1 / (k + rank + 1) // rank is 0-based, RRF uses 1-based
+
+      const existing = scoreMap.get(key)
+      const originalScore = r._combinedScore ?? (r._distance != null ? Math.max(0, 1 - r._distance / 2) : 0)
+
+      if (existing) {
+        existing.rrfScore += rrfContribution
+        // Keep the row with the best original score (most metadata)
+        if (originalScore > existing.bestOriginalScore) {
+          existing.row = r
+          existing.bestOriginalScore = originalScore
+        }
+      } else {
+        scoreMap.set(key, {
+          row: r,
+          rrfScore: rrfContribution,
+          bestOriginalScore: originalScore,
+        })
+      }
+    }
+  }
+
+  // Sort by RRF score and return
+  const merged = Array.from(scoreMap.values())
+    .sort((a, b) => b.rrfScore - a.rrfScore)
+    .slice(0, limit)
+    .map(entry => ({
+      ...entry.row,
+      _rrfScore: entry.rrfScore,
+      _combinedScore: entry.bestOriginalScore, // preserve for downstream compatibility
+    }))
+
+  return merged
+}

package/vectorizer.yaml

@@ -68,6 +68,13 @@ vectorizer:
     auto_prune_search: true        # Replace old search outputs with compact summaries
     substitute_tool_outputs: true  # Replace read() outputs when chunks in workspace
 
+  # Query decomposition (v4 — improves long query relevance)
+  decomposition:
+    enabled: true              # Split complex queries into focused sub-queries
+    min_words: 5               # Min significant words to trigger decomposition
+    max_sub_queries: 4         # Max sub-queries (including keyword core)
+    min_sub_query_words: 2     # Min words per sub-query
+
   # Quality monitoring (v2)
   quality:
     enable_metrics: false   # Track search quality metrics