@gmickel/gno 0.3.0

package/src/pipeline/hybrid.ts

@@ -0,0 +1,535 @@
+/**
+ * Hybrid search orchestrator.
+ * Combines BM25, vector search, expansion, fusion, and reranking.
+ *
+ * @module src/pipeline/hybrid
+ */
+
+import type { Config } from '../config/types';
+import type { EmbeddingPort, GenerationPort, RerankPort } from '../llm/types';
+import type { StorePort } from '../store/types';
+import { err, ok } from '../store/types';
+import type { VectorIndexPort } from '../store/vector/types';
+import { createChunkLookup } from './chunk-lookup';
+import { expandQuery } from './expansion';
+import {
+  buildExplainResults,
+  explainBm25,
+  explainExpansion,
+  explainFusion,
+  explainRerank,
+  explainVector,
+} from './explain';
+import { type RankedInput, rrfFuse, toRankedInput } from './fusion';
+import { detectQueryLanguage } from './query-language';
+import { rerankCandidates } from './rerank';
+import type {
+  ExpansionResult,
+  ExplainLine,
+  HybridSearchOptions,
+  PipelineConfig,
+  SearchResult,
+  SearchResults,
+} from './types';
+import { DEFAULT_PIPELINE_CONFIG } from './types';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Dependencies
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface HybridSearchDeps {
+  store: StorePort;
+  config: Config;
+  vectorIndex: VectorIndexPort | null;
+  embedPort: EmbeddingPort | null;
+  genPort: GenerationPort | null;
+  rerankPort: RerankPort | null;
+  pipelineConfig?: PipelineConfig;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Score Normalization
+// ─────────────────────────────────────────────────────────────────────────────
+
+function _normalizeVectorScore(distance: number): number {
+  return Math.max(0, Math.min(1, 1 - distance / 2));
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// BM25 Strength Check
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Check if BM25 results are strong enough to skip expansion.
+ * Uses gap-based metric: how much better is #1 than #2?
+ * Returns 0-1 where 1 = #1 is clearly dominant, 0 = results are similar.
+ * Raw BM25: smaller (more negative) is better.
+ */
+async function checkBm25Strength(
+  store: StorePort,
+  query: string,
+  options?: { collection?: string; lang?: string }
+): Promise<number> {
+  const result = await store.searchFts(query, {
+    limit: 5,
+    collection: options?.collection,
+    language: options?.lang,
+  });
+  if (!result.ok || result.value.length === 0) {
+    return 0;
+  }
+
+  // Only one result = strong signal
+  if (result.value.length === 1) {
+    return 1;
+  }
+
+  // Get top 2 scores (smaller is better)
+  const scores = result.value.map((r) => r.score).sort((a, b) => a - b);
+  const best = scores[0] ?? 0;
+  const second = scores[1] ?? best;
+  const worst = scores.at(-1) ?? best;
+
+  // Compute gap-based strength
+  // If best and second are equal, gap = 0
+  // If second is much worse (larger), gap approaches 1
+  const range = worst - best;
+  if (range === 0) {
+    return 0; // All scores equal, no clear winner
+  }
+
+  // Gap = how much worse is #2 relative to the range (clamped for safety)
+  const gap = (second - best) / range;
+  return Math.max(0, Math.min(1, gap));
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// FTS Retrieval (returns ChunkIds)
+// ─────────────────────────────────────────────────────────────────────────────
+
+interface ChunkId {
+  mirrorHash: string;
+  seq: number;
+}
+
+type FtsChunksResult =
+  | { ok: true; chunks: ChunkId[] }
+  | { ok: false; code: 'INVALID_INPUT' | 'OTHER'; message: string };
+
+async function searchFtsChunks(
+  store: StorePort,
+  query: string,
+  options: { limit: number; collection?: string; lang?: string }
+): Promise<FtsChunksResult> {
+  const result = await store.searchFts(query, {
+    limit: options.limit,
+    collection: options.collection,
+    language: options.lang,
+  });
+  if (!result.ok) {
+    // Propagate INVALID_INPUT for FTS syntax errors
+    const code =
+      result.error.code === 'INVALID_INPUT' ? 'INVALID_INPUT' : 'OTHER';
+    return { ok: false, code, message: result.error.message };
+  }
+  return {
+    ok: true,
+    chunks: result.value.map((r) => ({
+      mirrorHash: r.mirrorHash,
+      seq: r.seq,
+    })),
+  };
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Vector Retrieval (returns ChunkIds)
+// ─────────────────────────────────────────────────────────────────────────────
+
+async function searchVectorChunks(
+  vectorIndex: VectorIndexPort,
+  embedPort: EmbeddingPort,
+  query: string,
+  options: { limit: number; minScore?: number }
+): Promise<ChunkId[]> {
+  if (!vectorIndex.searchAvailable) {
+    return [];
+  }
+
+  const embedResult = await embedPort.embed(query);
+  if (!embedResult.ok) {
+    return [];
+  }
+
+  const queryEmbedding = new Float32Array(embedResult.value);
+  const searchResult = await vectorIndex.searchNearest(
+    queryEmbedding,
+    options.limit,
+    { minScore: options.minScore }
+  );
+
+  if (!searchResult.ok) {
+    return [];
+  }
+
+  return searchResult.value.map((r) => ({
+    mirrorHash: r.mirrorHash,
+    seq: r.seq,
+  }));
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Hybrid Search
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Execute hybrid search with full pipeline.
+ */
+// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: search orchestration with BM25, vector, expansion, fusion, and reranking
+export async function searchHybrid(
+  deps: HybridSearchDeps,
+  query: string,
+  options: HybridSearchOptions = {}
+): Promise<
+  ReturnType<typeof ok<SearchResults>> | ReturnType<typeof err<SearchResults>>
+> {
+  const { store, vectorIndex, embedPort, genPort, rerankPort } = deps;
+  const pipelineConfig = deps.pipelineConfig ?? DEFAULT_PIPELINE_CONFIG;
+
+  const limit = options.limit ?? 20;
+  const explainLines: ExplainLine[] = [];
+  let expansion: ExpansionResult | null = null;
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // 0. Detect query language for PROMPT SELECTION only
+  //    CRITICAL: Detection does NOT change retrieval filters - options.lang does
+  //    Priority: queryLanguageHint (MCP) > lang (CLI) > detection
+  // ─────────────────────────────────────────────────────────────────────────
+  const detection = detectQueryLanguage(query);
+  // Use explicit hint > lang filter > detected language
+  const queryLanguage =
+    options.queryLanguageHint ?? options.lang ?? detection.bcp47;
+
+  // Build explain message for language detection
+  let langMessage: string;
+  if (options.queryLanguageHint) {
+    langMessage = `queryLanguage=${queryLanguage} (hint)`;
+  } else if (options.lang) {
+    langMessage = `queryLanguage=${queryLanguage} (explicit)`;
+  } else {
+    const confidence = detection.confident ? '' : ', low confidence';
+    langMessage = `queryLanguage=${queryLanguage} (detected${confidence})`;
+  }
+  explainLines.push({ stage: 'lang', message: langMessage });
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // 1. Check if expansion needed
+  // ─────────────────────────────────────────────────────────────────────────
+  const shouldExpand = !options.noExpand && genPort !== null;
+  let skipExpansionDueToStrength = false;
+
+  if (shouldExpand) {
+    const bm25Strength = await checkBm25Strength(store, query, {
+      collection: options.collection,
+      lang: options.lang,
+    });
+    skipExpansionDueToStrength =
+      bm25Strength >= pipelineConfig.strongBm25Threshold;
+
+    if (!skipExpansionDueToStrength) {
+      const expandResult = await expandQuery(genPort, query, {
+        // Use queryLanguage for prompt selection, NOT options.lang (retrieval filter)
+        lang: queryLanguage,
+        timeout: pipelineConfig.expansionTimeout,
+      });
+      if (expandResult.ok) {
+        expansion = expandResult.value;
+      }
+    }
+  }
+
+  explainLines.push(
+    explainExpansion(shouldExpand && !skipExpansionDueToStrength, expansion)
+  );
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // 2. Parallel retrieval using raw store/vector APIs for correct seq tracking
+  // ─────────────────────────────────────────────────────────────────────────
+  const rankedInputs: RankedInput[] = [];
+
+  // BM25: original query
+  const bm25Result = await searchFtsChunks(store, query, {
+    limit: limit * 2,
+    collection: options.collection,
+    lang: options.lang,
+  });
+
+  // Propagate FTS syntax errors as INVALID_INPUT
+  if (!bm25Result.ok && bm25Result.code === 'INVALID_INPUT') {
+    return err('INVALID_INPUT', `Invalid search query: ${bm25Result.message}`);
+  }
+  // Other errors: continue with empty BM25 results
+
+  const bm25Chunks = bm25Result.ok ? bm25Result.chunks : [];
+  const bm25Count = bm25Chunks.length;
+  if (bm25Count > 0) {
+    rankedInputs.push(toRankedInput('bm25', bm25Chunks));
+  }
+
+  // BM25: lexical variants (syntax errors here are ignored - variants are optional)
+  if (expansion?.lexicalQueries) {
+    for (const variant of expansion.lexicalQueries) {
+      const variantResult = await searchFtsChunks(store, variant, {
+        limit,
+        collection: options.collection,
+        lang: options.lang,
+      });
+      if (variantResult.ok && variantResult.chunks.length > 0) {
+        rankedInputs.push(toRankedInput('bm25_variant', variantResult.chunks));
+      }
+    }
+  }
+
+  explainLines.push(explainBm25(bm25Count));
+
+  // Vector search
+  let vecCount = 0;
+  const vectorAvailable = vectorIndex?.searchAvailable && embedPort !== null;
+
+  if (vectorAvailable && vectorIndex && embedPort) {
+    // Original query
+    const vecChunks = await searchVectorChunks(vectorIndex, embedPort, query, {
+      limit: limit * 2,
+    });
+
+    vecCount = vecChunks.length;
+    if (vecCount > 0) {
+      rankedInputs.push(toRankedInput('vector', vecChunks));
+    }
+
+    // Semantic variants
+    if (expansion?.vectorQueries) {
+      for (const variant of expansion.vectorQueries) {
+        const variantChunks = await searchVectorChunks(
+          vectorIndex,
+          embedPort,
+          variant,
+          { limit }
+        );
+        if (variantChunks.length > 0) {
+          rankedInputs.push(toRankedInput('vector_variant', variantChunks));
+        }
+      }
+    }
+
+    // HyDE
+    if (expansion?.hyde) {
+      const hydeChunks = await searchVectorChunks(
+        vectorIndex,
+        embedPort,
+        expansion.hyde,
+        { limit }
+      );
+      if (hydeChunks.length > 0) {
+        rankedInputs.push(toRankedInput('hyde', hydeChunks));
+      }
+    }
+  }
+
+  explainLines.push(explainVector(vecCount, vectorAvailable ?? false));
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // 3. RRF Fusion
+  // ─────────────────────────────────────────────────────────────────────────
+  const fusedCandidates = rrfFuse(rankedInputs, pipelineConfig.rrf);
+  explainLines.push(
+    explainFusion(pipelineConfig.rrf.k, fusedCandidates.length)
+  );
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // 4. Reranking
+  // ─────────────────────────────────────────────────────────────────────────
+  const rerankResult = await rerankCandidates(
+    { rerankPort: options.noRerank ? null : rerankPort, store },
+    query,
+    fusedCandidates,
+    { maxCandidates: pipelineConfig.rerankCandidates }
+  );
+
+  explainLines.push(
+    explainRerank(
+      !options.noRerank && rerankPort !== null,
+      pipelineConfig.rerankCandidates
+    )
+  );
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // 4b. Apply minScore filter (blendedScore is now normalized to [0,1])
+  // ─────────────────────────────────────────────────────────────────────────
+  const minScore = options.minScore ?? 0;
+  const filteredCandidates =
+    minScore > 0
+      ? rerankResult.candidates.filter((c) => c.blendedScore >= minScore)
+      : rerankResult.candidates;
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // 5. Build final results (optimized: batch lookups, no per-candidate queries)
+  // ─────────────────────────────────────────────────────────────────────────
+
+  // Collect unique mirrorHashes needed from candidates
+  // TODO: For large corpora (100k+ docs), add store.getDocumentsByMirrorHashes
+  // batch lookup to avoid loading all documents into memory.
+  const neededHashes = new Set(filteredCandidates.map((c) => c.mirrorHash));
+
+  // Fetch documents and collections
+  const docsResult = await store.listDocuments(options.collection);
+  const collectionsResult = await store.getCollections();
+
+  if (!docsResult.ok) {
+    return err('QUERY_FAILED', docsResult.error.message);
+  }
+
+  // Build lookup maps - only include docs needed by candidates
+  const docByMirrorHash = new Map<string, (typeof docsResult.value)[number]>();
+  for (const doc of docsResult.value) {
+    if (doc.active && doc.mirrorHash && neededHashes.has(doc.mirrorHash)) {
+      docByMirrorHash.set(doc.mirrorHash, doc);
+    }
+  }
+
+  const collectionPaths = new Map<string, string>();
+  if (collectionsResult.ok) {
+    for (const c of collectionsResult.value) {
+      collectionPaths.set(c.name, c.path);
+    }
+  }
+
+  // Pre-fetch all chunks in one batch query (eliminates N+1)
+  const chunksMapResult = await store.getChunksBatch([...neededHashes]);
+  if (!chunksMapResult.ok) {
+    return err('QUERY_FAILED', chunksMapResult.error.message);
+  }
+  const chunksMap = chunksMapResult.value;
+  const getChunk = createChunkLookup(chunksMap);
+
+  // Cache full content by mirrorHash for --full mode
+  const contentCache = new Map<
+    string,
+    Awaited<ReturnType<typeof store.getContent>>
+  >();
+
+  const results: SearchResult[] = [];
+  const docidMap = new Map<string, string>();
+  // Track seen docids for --full de-duplication
+  const seenDocids = new Set<string>();
+
+  // Iterate until we have enough results (don't slice early - deduping may skip candidates)
+  for (const candidate of filteredCandidates) {
+    // Stop when we have enough results
+    if (results.length >= limit) {
+      break;
+    }
+
+    // Find document from pre-fetched map
+    const doc = docByMirrorHash.get(candidate.mirrorHash);
+    if (!doc) {
+      continue;
+    }
+
+    // For --full mode, de-dupe by docid (keep best scoring candidate per doc)
+    if (options.full && seenDocids.has(doc.docid)) {
+      continue;
+    }
+
+    // Get chunk via O(1) lookup
+    const chunk = getChunk(candidate.mirrorHash, candidate.seq);
+    if (!chunk) {
+      continue;
+    }
+
+    docidMap.set(`${candidate.mirrorHash}:${candidate.seq}`, doc.docid);
+
+    const collectionPath = collectionPaths.get(doc.collection);
+
+    // For --full mode, fetch full mirror content
+    let snippet = chunk.text;
+    let snippetRange: { startLine: number; endLine: number } | undefined = {
+      startLine: chunk.startLine,
+      endLine: chunk.endLine,
+    };
+
+    if (options.full) {
+      // Get or fetch full content for this mirrorHash
+      let contentResult = contentCache.get(candidate.mirrorHash);
+      if (!contentResult) {
+        contentResult = await store.getContent(candidate.mirrorHash);
+        contentCache.set(candidate.mirrorHash, contentResult);
+      }
+
+      if (contentResult.ok && contentResult.value) {
+        snippet = contentResult.value;
+        snippetRange = undefined; // Full content has no range
+      }
+      // Fallback to chunk text if content unavailable
+    }
+
+    seenDocids.add(doc.docid);
+
+    results.push({
+      docid: doc.docid,
+      score: candidate.blendedScore,
+      uri: doc.uri,
+      title: doc.title ?? undefined,
+      snippet,
+      snippetLanguage: chunk.language ?? undefined,
+      snippetRange,
+      source: {
+        relPath: doc.relPath,
+        absPath: collectionPath
+          ? `${collectionPath}/${doc.relPath}`
+          : undefined,
+        mime: doc.sourceMime,
+        ext: doc.sourceExt,
+        modifiedAt: doc.sourceMtime,
+        sizeBytes: doc.sourceSize,
+        sourceHash: doc.sourceHash,
+      },
+      conversion: {
+        mirrorHash: candidate.mirrorHash,
+        converterId: doc.converterId ?? undefined,
+        converterVersion: doc.converterVersion ?? undefined,
+      },
+    });
+  }
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // 6. Build explain data (if requested)
+  // ─────────────────────────────────────────────────────────────────────────
+  const explainData = options.explain
+    ? {
+        lines: explainLines,
+        results: buildExplainResults(
+          filteredCandidates.slice(0, limit),
+          docidMap
+        ),
+      }
+    : undefined;
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // 7. Return results
+  // ─────────────────────────────────────────────────────────────────────────
+  return ok({
+    results,
+    meta: {
+      query,
+      mode: vectorAvailable ? 'hybrid' : 'bm25_only',
+      expanded: expansion !== null,
+      reranked: rerankResult.reranked,
+      vectorsUsed: vectorAvailable ?? false,
+      totalResults: results.length,
+      collection: options.collection,
+      lang: options.lang,
+      queryLanguage,
+      explain: explainData,
+    },
+  });
+}

package/src/pipeline/index.ts

@@ -0,0 +1,64 @@
+/**
+ * Search pipeline public API.
+ *
+ * @module src/pipeline
+ */
+
+// Expansion
+export { expandQuery, generateCacheKey } from './expansion';
+// Explain
+export {
+  buildExplainResults,
+  explainBm25,
+  explainExpansion,
+  explainFusion,
+  explainRerank,
+  explainVector,
+  formatExplain,
+  formatResultExplain,
+} from './explain';
+// Fusion
+export { type RankedInput, rrfFuse, toRankedInput } from './fusion';
+// Hybrid search
+export { type HybridSearchDeps, searchHybrid } from './hybrid';
+// Rerank
+export { rerankCandidates } from './rerank';
+// BM25 search
+export { searchBm25 } from './search';
+// Types
+export type {
+  AskMeta,
+  AskOptions,
+  AskPort,
+  AskResult,
+  BlendingTier,
+  Bm25SearchPort,
+  Citation,
+  ExpansionPort,
+  ExpansionResult,
+  ExplainLine,
+  ExplainResult,
+  FusionCandidate,
+  FusionSource,
+  HybridSearchOptions,
+  HybridSearchPort,
+  PipelineConfig,
+  RerankedCandidate,
+  RrfConfig,
+  SearchMeta,
+  SearchMode,
+  SearchOptions,
+  SearchResult,
+  SearchResultConversion,
+  SearchResultSource,
+  SearchResults,
+  SnippetRange,
+  VectorSearchPort,
+} from './types';
+export {
+  DEFAULT_BLENDING_SCHEDULE,
+  DEFAULT_PIPELINE_CONFIG,
+  DEFAULT_RRF_CONFIG,
+} from './types';
+// Vector search
+export { searchVector, type VectorSearchDeps } from './vsearch';