@gmickel/gno 0.4.0 → 0.5.1

package/src/pipeline/hybrid.ts

@@ -11,9 +11,11 @@ import type { StorePort } from '../store/types';
 import { err, ok } from '../store/types';
 import type { VectorIndexPort } from '../store/vector/types';
 import { createChunkLookup } from './chunk-lookup';
+import { formatQueryForEmbedding } from './contextual';
 import { expandQuery } from './expansion';
 import {
   buildExplainResults,
+  type ExpansionStatus,
   explainBm25,
   explainExpansion,
   explainFusion,
@@ -51,56 +53,64 @@ export interface HybridSearchDeps {
 // Score Normalization
 // ─────────────────────────────────────────────────────────────────────────────
 
-function _normalizeVectorScore(distance: number): number {
-  return Math.max(0, Math.min(1, 1 - distance / 2));
+// Removed: _normalizeVectorScore was dead code (vector distances normalized in vector index)
+
+// ─────────────────────────────────────────────────────────────────────────────
+// BM25 Score Normalization
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Normalize raw BM25 score to 0-1 range using sigmoid.
+ * BM25 scores are negative in SQLite FTS5 (more negative = better match).
+ * Typical range: -15 (excellent) to -2 (weak match).
+ * Maps to 0-1 where higher is better.
+ */
+function normalizeBm25Score(rawScore: number): number {
+  const absScore = Math.abs(rawScore);
+  // Sigmoid with center=4.5, scale=2.8
+  // Maps: -15 → ~0.99, -5 → ~0.55, -2 → ~0.29
+  return 1 / (1 + Math.exp(-(absScore - 4.5) / 2.8));
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
 // BM25 Strength Check
 // ─────────────────────────────────────────────────────────────────────────────
 
+// Thresholds for strong signal detection (conservative - prefer expansion over speed)
+const STRONG_TOP_SCORE = 0.84; // ~84th percentile confidence
+const STRONG_GAP = 0.14; // Clear separation from #2
+
 /**
  * Check if BM25 results are strong enough to skip expansion.
- * Uses gap-based metric: how much better is #1 than #2?
- * Returns 0-1 where 1 = #1 is clearly dominant, 0 = results are similar.
- * Raw BM25: smaller (more negative) is better.
+ * Returns true if top result is both confident AND clearly separated.
+ * This prevents skipping on weak-but-separated results.
  */
 async function checkBm25Strength(
   store: StorePort,
   query: string,
   options?: { collection?: string; lang?: string }
-): Promise<number> {
+): Promise<boolean> {
   const result = await store.searchFts(query, {
     limit: 5,
     collection: options?.collection,
     language: options?.lang,
   });
+
   if (!result.ok || result.value.length === 0) {
-    return 0;
+    return false;
   }
 
-  // Only one result = strong signal
-  if (result.value.length === 1) {
-    return 1;
-  }
+  // Normalize scores (higher = better)
+  const scores = result.value
+    .map((r) => normalizeBm25Score(r.score))
+    .sort((a, b) => b - a); // Descending
 
-  // Get top 2 scores (smaller is better)
-  const scores = result.value.map((r) => r.score).sort((a, b) => a - b);
-  const best = scores[0] ?? 0;
-  const second = scores[1] ?? best;
-  const worst = scores.at(-1) ?? best;
-
-  // Compute gap-based strength
-  // If best and second are equal, gap = 0
-  // If second is much worse (larger), gap approaches 1
-  const range = worst - best;
-  if (range === 0) {
-    return 0; // All scores equal, no clear winner
-  }
+  const topScore = scores[0] ?? 0;
+  const secondScore = scores[1] ?? 0;
+  const gap = topScore - secondScore;
 
-  // Gap = how much worse is #2 relative to the range (clamped for safety)
-  const gap = (second - best) / range;
-  return Math.max(0, Math.min(1, gap));
+  // Strong signal requires BOTH: high confidence AND clear separation
+  return topScore >= STRONG_TOP_SCORE && gap >= STRONG_GAP;
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -155,7 +165,8 @@ async function searchVectorChunks(
     return [];
   }
 
-  const embedResult = await embedPort.embed(query);
+  // Embed query with contextual formatting
+  const embedResult = await embedPort.embed(formatQueryForEmbedding(query));
   if (!embedResult.ok) {
     return [];
   }
@@ -225,17 +236,18 @@ export async function searchHybrid(
   // 1. Check if expansion needed
   // ─────────────────────────────────────────────────────────────────────────
   const shouldExpand = !options.noExpand && genPort !== null;
-  let skipExpansionDueToStrength = false;
+  let expansionStatus: ExpansionStatus = 'disabled';
 
   if (shouldExpand) {
-    const bm25Strength = await checkBm25Strength(store, query, {
+    const hasStrongSignal = await checkBm25Strength(store, query, {
       collection: options.collection,
       lang: options.lang,
     });
-    skipExpansionDueToStrength =
-      bm25Strength >= pipelineConfig.strongBm25Threshold;
 
-    if (!skipExpansionDueToStrength) {
+    if (hasStrongSignal) {
+      expansionStatus = 'skipped_strong';
+    } else {
+      expansionStatus = 'attempted';
       const expandResult = await expandQuery(genPort, query, {
         // Use queryLanguage for prompt selection, NOT options.lang (retrieval filter)
         lang: queryLanguage,
@@ -247,9 +259,7 @@ export async function searchHybrid(
     }
   }
 
-  explainLines.push(
-    explainExpansion(shouldExpand && !skipExpansionDueToStrength, expansion)
-  );
+  explainLines.push(explainExpansion(expansionStatus, expansion));
 
   // ─────────────────────────────────────────────────────────────────────────
   // 2. Parallel retrieval using raw store/vector APIs for correct seq tracking
@@ -293,7 +303,8 @@ export async function searchHybrid(
 
   // Vector search
   let vecCount = 0;
-  const vectorAvailable = vectorIndex?.searchAvailable && embedPort !== null;
+  const vectorAvailable =
+    (vectorIndex?.searchAvailable && embedPort !== null) ?? false;
 
   if (vectorAvailable && vectorIndex && embedPort) {
     // Original query
@@ -335,7 +346,7 @@ export async function searchHybrid(
     }
   }
 
-  explainLines.push(explainVector(vecCount, vectorAvailable ?? false));
+  explainLines.push(explainVector(vecCount, vectorAvailable));
 
   // ─────────────────────────────────────────────────────────────────────────
   // 3. RRF Fusion
@@ -441,7 +452,13 @@ export async function searchHybrid(
     }
 
     // Get chunk via O(1) lookup
-    const chunk = getChunk(candidate.mirrorHash, candidate.seq);
+    // For doc-level FTS (seq=0), fall back to first available chunk if exact lookup fails
+    let chunk = getChunk(candidate.mirrorHash, candidate.seq);
+    if (!chunk && candidate.seq === 0) {
+      // Doc-level FTS uses seq=0 as placeholder - try first chunk
+      const docChunks = chunksMap.get(candidate.mirrorHash);
+      chunk = docChunks?.[0];
+    }
     if (!chunk) {
       continue;
     }
@@ -524,7 +541,7 @@ export async function searchHybrid(
       mode: vectorAvailable ? 'hybrid' : 'bm25_only',
       expanded: expansion !== null,
       reranked: rerankResult.reranked,
-      vectorsUsed: vectorAvailable ?? false,
+      vectorsUsed: vectorAvailable,
       totalResults: results.length,
       collection: options.collection,
       lang: options.lang,

package/src/pipeline/index.ts

@@ -4,11 +4,18 @@
  * @module src/pipeline
  */
 
+// Contextual embedding
+export {
+  extractTitle,
+  formatDocForEmbedding,
+  formatQueryForEmbedding,
+} from './contextual';
 // Expansion
 export { expandQuery, generateCacheKey } from './expansion';
 // Explain
 export {
   buildExplainResults,
+  type ExpansionStatus,
   explainBm25,
   explainExpansion,
   explainFusion,

package/src/pipeline/rerank.ts

@@ -7,7 +7,6 @@
 
 import type { RerankPort } from '../llm/types';
 import type { StorePort } from '../store/types';
-import { createChunkLookup } from './chunk-lookup';
 import type { BlendingTier, FusionCandidate, RerankedCandidate } from './types';
 import { DEFAULT_BLENDING_SCHEDULE } from './types';
 
@@ -121,32 +120,44 @@ export async function rerankCandidates(
   const toRerank = candidates.slice(0, maxCandidates);
   const remaining = candidates.slice(maxCandidates);
 
-  // Pre-fetch all chunks in one batch query (eliminates N+1)
+  // Dedupe by document - multiple chunks from same doc use single full-doc rerank
   const uniqueHashes = [...new Set(toRerank.map((c) => c.mirrorHash))];
-  const chunksMapResult = await store.getChunksBatch(uniqueHashes);
 
-  // If chunk fetch fails, degrade gracefully (fusion-only)
-  // Don't rerank on empty/missing texts - produces non-deterministic results
-  if (!chunksMapResult.ok) {
-    return {
-      candidates: candidates.map((c) => ({
-        ...c,
-        rerankScore: null,
-        blendedScore: normalizeFusionScore(c.fusionScore),
-      })),
-      reranked: false,
-    };
+  // Fetch full document content for each unique document (parallel)
+  // Max 128K chars per doc to fit in reranker context
+  const MAX_DOC_CHARS = 128_000;
+  const contentResults = await Promise.all(
+    uniqueHashes.map((hash) => store.getContent(hash))
+  );
+  const docContents = new Map<string, string>();
+  for (let i = 0; i < uniqueHashes.length; i++) {
+    const hash = uniqueHashes[i] as string;
+    const result = contentResults[i] as Awaited<
+      ReturnType<typeof store.getContent>
+    >;
+    if (result.ok && result.value) {
+      const content = result.value;
+      docContents.set(
+        hash,
+        content.length > MAX_DOC_CHARS
+          ? `${content.slice(0, MAX_DOC_CHARS)}...`
+          : content
+      );
+    } else {
+      // Fallback to empty string if content not available
+      docContents.set(hash, '');
+    }
   }
-  const chunksMap = chunksMapResult.value;
-  const getChunk = createChunkLookup(chunksMap);
 
-  // Build texts array for reranking (O(1) lookup per candidate)
-  const texts: string[] = toRerank.map((c) => {
-    const chunk = getChunk(c.mirrorHash, c.seq);
-    return chunk?.text ?? '';
-  });
+  // Build texts array for reranking (one per unique document)
+  const hashToIndex = new Map<string, number>();
+  const texts: string[] = [];
+  for (const hash of uniqueHashes) {
+    hashToIndex.set(hash, texts.length);
+    texts.push(docContents.get(hash) ?? '');
+  }
 
-  // Run reranking
+  // Run reranking on full documents
   const rerankResult = await rerankPort.rerank(query, texts);
 
   if (!rerankResult.ok) {
@@ -163,16 +174,33 @@ export async function rerankCandidates(
 
   // Map rerank scores to candidates
   // Note: We use normalizeFusionScore defined above (across ALL candidates)
-  // Build index->score map for O(1) lookup instead of O(n) find per candidate
-  const scoreByIndex = new Map(
+  // Build doc index->score map for O(1) lookup
+  // All chunks from same document share the same rerank score
+  const scoreByDocIndex = new Map(
     rerankResult.value.map((s) => [s.index, s.score])
   );
+
+  // Normalize rerank scores using min-max (models return varying scales)
+  const rerankScores = rerankResult.value.map((s) => s.score);
+  const minRerank = Math.min(...rerankScores);
+  const maxRerank = Math.max(...rerankScores);
+  const rerankRange = maxRerank - minRerank;
+
+  function normalizeRerankScore(score: number): number {
+    if (rerankRange < 1e-9) {
+      return 1; // All tied for best
+    }
+    return (score - minRerank) / rerankRange;
+  }
+
   const rerankedCandidates: RerankedCandidate[] = toRerank.map((c, i) => {
-    const rerankScore = scoreByIndex.get(i) ?? null;
+    // Get document-level rerank score (shared by all chunks from same doc)
+    const docIndex = hashToIndex.get(c.mirrorHash) ?? -1;
+    const rerankScore = scoreByDocIndex.get(docIndex) ?? null;
 
-    // Normalize rerank score to 0-1 range (models may return different scales)
+    // Normalize rerank score to 0-1 range using min-max
     const normalizedRerankScore =
-      rerankScore !== null ? Math.max(0, Math.min(1, rerankScore)) : null;
+      rerankScore !== null ? normalizeRerankScore(rerankScore) : null;
 
     // Calculate blended score using normalized fusion score
     const position = i + 1;

package/src/pipeline/types.ts

@@ -213,8 +213,6 @@ export type RerankedCandidate = FusionCandidate & {
 
 /** Search pipeline configuration */
 export interface PipelineConfig {
-  /** Strong BM25 threshold to skip expansion */
-  strongBm25Threshold: number;
   /** Expansion timeout in ms */
   expansionTimeout: number;
   /** Max candidates to rerank */
@@ -227,7 +225,6 @@ export interface PipelineConfig {
 
 /** Default pipeline configuration */
 export const DEFAULT_PIPELINE_CONFIG: PipelineConfig = {
-  strongBm25Threshold: 0.7,
   expansionTimeout: 5000,
   rerankCandidates: 20,
   rrf: DEFAULT_RRF_CONFIG,

package/src/pipeline/vsearch.ts

@@ -11,6 +11,7 @@ import type { StorePort } from '../store/types';
 import { err, ok } from '../store/types';
 import type { VectorIndexPort } from '../store/vector/types';
 import { createChunkLookup } from './chunk-lookup';
+import { formatQueryForEmbedding } from './contextual';
 import { detectQueryLanguage } from './query-language';
 import type { SearchOptions, SearchResult, SearchResults } from './types';
 
@@ -268,8 +269,8 @@ export async function searchVector(
     );
   }
 
-  // Embed query
-  const embedResult = await embedPort.embed(query);
+  // Embed query with contextual formatting
+  const embedResult = await embedPort.embed(formatQueryForEmbedding(query));
   if (!embedResult.ok) {
     return err(
       'QUERY_FAILED',

package/src/serve/routes/api.ts

@@ -452,7 +452,7 @@ export async function handleAsk(
   if (ctx.genPort) {
     const maxTokens = body.maxAnswerTokens ?? 512;
     const rawResult = await generateGroundedAnswer(
-      ctx.genPort,
+      { genPort: ctx.genPort, store: ctx.store },
       query,
       results,
       maxTokens

package/src/store/migrations/002-documents-fts.ts

@@ -0,0 +1,40 @@
+/**
+ * Migration: Document-level FTS with Snowball stemmer.
+ *
+ * Replaces chunk-level content_fts with document-level documents_fts.
+ * Uses snowball tokenizer for multilingual stemming support.
+ *
+ * @module src/store/migrations/002-documents-fts
+ */
+
+import type { Database } from 'bun:sqlite';
+import type { FtsTokenizer } from '../../config/types';
+import type { Migration } from './runner';
+
+export const migration: Migration = {
+  version: 2,
+  name: 'documents_fts',
+
+  up(db: Database, ftsTokenizer: FtsTokenizer): void {
+    // Drop old chunk-level FTS (no backwards compat needed per epic)
+    db.exec('DROP TABLE IF EXISTS content_fts');
+
+    // Create document-level FTS with snowball stemmer
+    // Indexes: filepath (for path searches), title, body (full content)
+    // Note: NOT using content='' because contentless tables don't support DELETE
+    // The storage overhead is acceptable for simpler update semantics
+    db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(
+        filepath,
+        title,
+        body,
+        tokenize='${ftsTokenizer}'
+      )
+    `);
+  },
+
+  down(db: Database): void {
+    db.exec('DROP TABLE IF EXISTS documents_fts');
+    // Note: Cannot restore content_fts - would need full reindex
+  },
+};

package/src/store/migrations/index.ts

@@ -15,6 +15,7 @@ export {
 
 // Import all migrations
 import { migration as m001 } from './001-initial';
+import { migration as m002 } from './002-documents-fts';
 
 /** All migrations in order */
-export const migrations = [m001];
+export const migrations = [m001, m002];