@gmickel/gno 0.3.0

package/src/pipeline/query-language.ts

@@ -0,0 +1,118 @@
+/**
+ * Query language detection for prompt selection.
+ *
+ * IMPORTANT: This affects prompt selection and metadata ONLY.
+ * It does NOT affect retrieval filtering - that's controlled by CLI --lang flag.
+ */
+import { franc } from 'franc';
+
+const MIN_RELIABLE_LENGTH = 15;
+
+/**
+ * Supported languages for detection.
+ * Maps ISO 639-3 codes to BCP-47 (ISO 639-1) codes.
+ *
+ * Selection criteria:
+ * - Major world languages by speaker count
+ * - Significant tech/documentation communities
+ * - Linguistically distinct (to minimize false positives)
+ */
+const LANG_MAP = {
+  // Western European (Germanic)
+  eng: 'en', // English
+  deu: 'de', // German
+  nld: 'nl', // Dutch
+
+  // Western European (Romance)
+  fra: 'fr', // French
+  ita: 'it', // Italian
+  spa: 'es', // Spanish
+  por: 'pt', // Portuguese
+  cat: 'ca', // Catalan
+  ron: 'ro', // Romanian
+
+  // Scandinavian
+  swe: 'sv', // Swedish
+  dan: 'da', // Danish
+  nob: 'nb', // Norwegian Bokmål
+  nno: 'nn', // Norwegian Nynorsk
+  fin: 'fi', // Finnish
+
+  // Eastern European
+  pol: 'pl', // Polish
+  ces: 'cs', // Czech
+  slk: 'sk', // Slovak
+  rus: 'ru', // Russian
+  ukr: 'uk', // Ukrainian
+  bul: 'bg', // Bulgarian
+  hrv: 'hr', // Croatian
+  ell: 'el', // Greek
+  hun: 'hu', // Hungarian
+
+  // Middle Eastern
+  tur: 'tr', // Turkish
+  ara: 'ar', // Arabic
+  heb: 'he', // Hebrew
+  fas: 'fa', // Persian/Farsi
+
+  // South Asian
+  hin: 'hi', // Hindi
+
+  // Southeast Asian
+  vie: 'vi', // Vietnamese
+  tha: 'th', // Thai
+  ind: 'id', // Indonesian
+
+  // East Asian
+  cmn: 'zh', // Mandarin Chinese
+  jpn: 'ja', // Japanese
+  kor: 'ko', // Korean
+} as const;
+
+/** ISO 639-3 codes for franc's only filter */
+const SUPPORTED_LANGUAGES = Object.keys(LANG_MAP);
+
+export interface LanguageDetection {
+  /** BCP-47 code: 'en', 'de', 'fr', etc. 'und' if undetermined */
+  bcp47: string;
+  /** ISO 639-3 code: 'eng', 'deu', 'fra', etc. 'und' if undetermined */
+  iso639_3: string;
+  /** false if text too short or language undetermined */
+  confident: boolean;
+}
+
+/**
+ * Detect the language of query text for prompt selection.
+ *
+ * @param text - Query text to analyze
+ * @returns Language detection result with BCP-47 code and confidence
+ *
+ * @example
+ * detectQueryLanguage("wie konfiguriere ich kubernetes")
+ * // { bcp47: 'de', iso639_3: 'deu', confident: true }
+ *
+ * detectQueryLanguage("hello")
+ * // { bcp47: 'und', iso639_3: 'und', confident: false } // too short
+ */
+export function detectQueryLanguage(text: string): LanguageDetection {
+  const trimmed = text.trim();
+
+  if (trimmed.length < MIN_RELIABLE_LENGTH) {
+    return { bcp47: 'und', iso639_3: 'und', confident: false };
+  }
+
+  const detected = franc(trimmed, {
+    minLength: MIN_RELIABLE_LENGTH,
+    only: SUPPORTED_LANGUAGES,
+  });
+
+  if (detected === 'und') {
+    return { bcp47: 'und', iso639_3: 'und', confident: false };
+  }
+
+  const bcp47 = LANG_MAP[detected as keyof typeof LANG_MAP];
+  if (!bcp47) {
+    return { bcp47: 'und', iso639_3: 'und', confident: false };
+  }
+  return { bcp47, iso639_3: detected, confident: true };
+}

package/src/pipeline/rerank.ts

@@ -0,0 +1,223 @@
+/**
+ * Reranking and position-aware blending.
+ * Uses RerankPort to reorder candidates.
+ *
+ * @module src/pipeline/rerank
+ */
+
+import type { RerankPort } from '../llm/types';
+import type { StorePort } from '../store/types';
+import { createChunkLookup } from './chunk-lookup';
+import type { BlendingTier, FusionCandidate, RerankedCandidate } from './types';
+import { DEFAULT_BLENDING_SCHEDULE } from './types';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Types
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface RerankOptions {
+  /** Max candidates to rerank */
+  maxCandidates?: number;
+  /** Blending schedule */
+  blendingSchedule?: BlendingTier[];
+}
+
+export interface RerankResult {
+  candidates: RerankedCandidate[];
+  reranked: boolean;
+}
+
+export interface RerankDeps {
+  rerankPort: RerankPort | null;
+  store: StorePort;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Blending
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Get blending weights for a position.
+ */
+function getBlendingWeights(
+  position: number,
+  schedule: BlendingTier[]
+): { fusionWeight: number; rerankWeight: number } {
+  const tier = schedule.find((t) => position <= t.maxRank);
+  if (tier) {
+    return { fusionWeight: tier.fusionWeight, rerankWeight: tier.rerankWeight };
+  }
+  // Fallback to last tier
+  const last = schedule.at(-1);
+  return last
+    ? { fusionWeight: last.fusionWeight, rerankWeight: last.rerankWeight }
+    : { fusionWeight: 0.5, rerankWeight: 0.5 };
+}
+
+/**
+ * Blend fusion and rerank scores.
+ */
+function blend(
+  fusionScore: number,
+  rerankScore: number,
+  position: number,
+  schedule: BlendingTier[]
+): number {
+  const { fusionWeight, rerankWeight } = getBlendingWeights(position, schedule);
+  return fusionWeight * fusionScore + rerankWeight * rerankScore;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Rerank Implementation
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Rerank candidates using cross-encoder.
+ * Falls back to fusion-only if reranking fails.
+ */
+export async function rerankCandidates(
+  deps: RerankDeps,
+  query: string,
+  candidates: FusionCandidate[],
+  options: RerankOptions = {}
+): Promise<RerankResult> {
+  // Early return for empty candidates
+  if (candidates.length === 0) {
+    return { candidates: [], reranked: false };
+  }
+
+  const { rerankPort, store } = deps;
+  const maxCandidates = options.maxCandidates ?? 20;
+  const schedule = options.blendingSchedule ?? DEFAULT_BLENDING_SCHEDULE;
+
+  // Normalize fusion scores to 0-1 range across ALL candidates for stability.
+  // This ensures blendedScore is always in [0,1] regardless of reranker availability.
+  const fusionScoresAll = candidates.map((c) => c.fusionScore);
+  const minFusionAll = Math.min(...fusionScoresAll);
+  const maxFusionAll = Math.max(...fusionScoresAll);
+  const fusionRangeAll = maxFusionAll - minFusionAll;
+
+  function normalizeFusionScore(score: number): number {
+    if (fusionRangeAll < 1e-9) {
+      return 1; // tie for best
+    }
+    const v = (score - minFusionAll) / fusionRangeAll;
+    return Math.max(0, Math.min(1, v));
+  }
+
+  // If no reranker, return candidates with normalized fusion scores
+  if (!rerankPort) {
+    return {
+      candidates: candidates.map((c) => ({
+        ...c,
+        rerankScore: null,
+        blendedScore: normalizeFusionScore(c.fusionScore),
+      })),
+      reranked: false,
+    };
+  }
+
+  // Limit candidates for reranking
+  const toRerank = candidates.slice(0, maxCandidates);
+  const remaining = candidates.slice(maxCandidates);
+
+  // Pre-fetch all chunks in one batch query (eliminates N+1)
+  const uniqueHashes = [...new Set(toRerank.map((c) => c.mirrorHash))];
+  const chunksMapResult = await store.getChunksBatch(uniqueHashes);
+
+  // If chunk fetch fails, degrade gracefully (fusion-only)
+  // Don't rerank on empty/missing texts - produces non-deterministic results
+  if (!chunksMapResult.ok) {
+    return {
+      candidates: candidates.map((c) => ({
+        ...c,
+        rerankScore: null,
+        blendedScore: normalizeFusionScore(c.fusionScore),
+      })),
+      reranked: false,
+    };
+  }
+  const chunksMap = chunksMapResult.value;
+  const getChunk = createChunkLookup(chunksMap);
+
+  // Build texts array for reranking (O(1) lookup per candidate)
+  const texts: string[] = toRerank.map((c) => {
+    const chunk = getChunk(c.mirrorHash, c.seq);
+    return chunk?.text ?? '';
+  });
+
+  // Run reranking
+  const rerankResult = await rerankPort.rerank(query, texts);
+
+  if (!rerankResult.ok) {
+    // Graceful degradation - return normalized fusion scores
+    return {
+      candidates: candidates.map((c) => ({
+        ...c,
+        rerankScore: null,
+        blendedScore: normalizeFusionScore(c.fusionScore),
+      })),
+      reranked: false,
+    };
+  }
+
+  // Map rerank scores to candidates
+  // Note: We use normalizeFusionScore defined above (across ALL candidates)
+  // Build index->score map for O(1) lookup instead of O(n) find per candidate
+  const scoreByIndex = new Map(
+    rerankResult.value.map((s) => [s.index, s.score])
+  );
+  const rerankedCandidates: RerankedCandidate[] = toRerank.map((c, i) => {
+    const rerankScore = scoreByIndex.get(i) ?? null;
+
+    // Normalize rerank score to 0-1 range (models may return different scales)
+    const normalizedRerankScore =
+      rerankScore !== null ? Math.max(0, Math.min(1, rerankScore)) : null;
+
+    // Calculate blended score using normalized fusion score
+    const position = i + 1;
+    const normalizedFusion = normalizeFusionScore(c.fusionScore);
+    const blendedScore =
+      normalizedRerankScore !== null
+        ? blend(normalizedFusion, normalizedRerankScore, position, schedule)
+        : normalizedFusion;
+
+    return {
+      ...c,
+      rerankScore: normalizedRerankScore,
+      blendedScore,
+    };
+  });
+
+  // Add remaining candidates (not reranked)
+  // These get normalized fusion scores with penalty but clamped to [0,1]
+  const allCandidates: RerankedCandidate[] = [
+    ...rerankedCandidates,
+    ...remaining.map((c) => {
+      const base = normalizeFusionScore(c.fusionScore);
+      return {
+        ...c,
+        rerankScore: null,
+        // Apply 0.5x penalty and clamp to [0,1]
+        blendedScore: Math.max(0, Math.min(1, base * 0.5)),
+      };
+    }),
+  ];
+
+  // Sort by blended score
+  allCandidates.sort((a, b) => {
+    const scoreDiff = b.blendedScore - a.blendedScore;
+    if (Math.abs(scoreDiff) > 1e-9) {
+      return scoreDiff;
+    }
+    // Deterministic tie-breaking
+    const aKey = `${a.mirrorHash}:${a.seq}`;
+    const bKey = `${b.mirrorHash}:${b.seq}`;
+    return aKey.localeCompare(bKey);
+  });
+
+  return {
+    candidates: allCandidates,
+    reranked: true,
+  };
+}

package/src/pipeline/search.ts

@@ -0,0 +1,261 @@
+/**
+ * BM25 search pipeline.
+ * Wraps StorePort.searchFts() to produce SearchResults.
+ *
+ * @module src/pipeline/search
+ */
+
+import { join as pathJoin } from 'node:path'; // No Bun path utils equivalent
+import type { ChunkRow, FtsResult, StorePort } from '../store/types';
+import { err, ok } from '../store/types';
+import { createChunkLookup } from './chunk-lookup';
+import { detectQueryLanguage } from './query-language';
+import type {
+  SearchOptions,
+  SearchResult,
+  SearchResultSource,
+  SearchResults,
+} from './types';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Score Normalization
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Normalize BM25 scores to 0-1 range using min-max scaling.
+ * FTS5 bm25() returns negative scores where smaller (more negative) = better match.
+ * After normalization: 1 = best match, 0 = worst match in result set.
+ */
+function normalizeBm25Scores(results: SearchResult[]): void {
+  if (results.length === 0) {
+    return;
+  }
+
+  // Raw scores: smaller (more negative) is better
+  const scores = results.map((r) => r.score);
+  const best = Math.min(...scores); // Most negative = best
+  const worst = Math.max(...scores); // Least negative = worst
+  const range = worst - best;
+
+  // If all scores equal, assign 1.0 to all
+  if (range === 0) {
+    for (const r of results) {
+      r.score = 1;
+    }
+    return;
+  }
+
+  // Map: best -> 1, worst -> 0 (clamp for floating point safety)
+  for (const r of results) {
+    r.score = Math.max(0, Math.min(1, (worst - r.score) / range));
+  }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Result Building
+// ─────────────────────────────────────────────────────────────────────────────
+
+interface BuildResultContext {
+  fts: FtsResult;
+  chunk: ChunkRow | null;
+  collectionPath?: string;
+  options?: SearchOptions;
+  fullContent?: string;
+}
+
+/** Build SearchResult from FtsResult and related data */
+function buildSearchResult(ctx: BuildResultContext): SearchResult {
+  const { fts, chunk, collectionPath, options, fullContent } = ctx;
+  const source: SearchResultSource = {
+    relPath: fts.relPath ?? '',
+    // Use actual source metadata with fallback to markdown defaults
+    mime: fts.sourceMime ?? 'text/markdown',
+    ext: fts.sourceExt ?? '.md',
+    modifiedAt: fts.sourceMtime,
+    sizeBytes: fts.sourceSize,
+    sourceHash: fts.sourceHash,
+  };
+
+  // Add absPath if we have collection path (cross-platform safe)
+  if (collectionPath && fts.relPath) {
+    source.absPath = pathJoin(collectionPath, fts.relPath);
+  }
+
+  // Determine snippet content and range
+  let snippet: string;
+  let snippetRange: { startLine: number; endLine: number } | undefined;
+
+  if (options?.full && fullContent) {
+    // --full: use full content, no range (full doc)
+    snippet = fullContent;
+    snippetRange = undefined;
+  } else if (options?.lineNumbers && chunk) {
+    // --line-numbers: use raw chunk text (not FTS snippet with markers)
+    snippet = chunk.text;
+    snippetRange = { startLine: chunk.startLine, endLine: chunk.endLine };
+  } else {
+    // Default: use FTS snippet or chunk text
+    snippet = fts.snippet ?? chunk?.text ?? '';
+    snippetRange = chunk
+      ? { startLine: chunk.startLine, endLine: chunk.endLine }
+      : undefined;
+  }
+
+  return {
+    docid: fts.docid ?? '',
+    score: fts.score, // Raw score, normalized later as batch
+    uri: fts.uri ?? '',
+    title: fts.title,
+    snippet,
+    snippetLanguage: chunk?.language ?? undefined,
+    snippetRange,
+    source,
+    conversion: fts.mirrorHash ? { mirrorHash: fts.mirrorHash } : undefined,
+  };
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Search Function
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Execute BM25 search and return structured results.
+ */
+// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: BM25 search with pagination, filtering, and explain output
+export async function searchBm25(
+  store: StorePort,
+  query: string,
+  options: SearchOptions = {}
+): Promise<
+  ReturnType<typeof ok<SearchResults>> | ReturnType<typeof err<SearchResults>>
+> {
+  const limit = options.limit ?? 20;
+  const minScore = options.minScore ?? 0;
+
+  // Detect query language for metadata (DOES NOT affect retrieval filtering)
+  const detection = detectQueryLanguage(query);
+  const queryLanguage = options.lang ?? detection.bcp47;
+
+  // Run FTS search
+  // Disable FTS snippet when --full or --line-numbers (we use raw text instead)
+  const ftsResult = await store.searchFts(query, {
+    limit,
+    collection: options.collection,
+    language: options.lang,
+    snippet: !(options.full || options.lineNumbers),
+  });
+
+  if (!ftsResult.ok) {
+    // Adapter returns INVALID_INPUT for FTS syntax errors, pass through
+    const { code, message, cause } = ftsResult.error;
+    if (code === 'INVALID_INPUT') {
+      return err('INVALID_INPUT', `Invalid search query: ${message}`);
+    }
+    return err('QUERY_FAILED', message, cause);
+  }
+
+  // Get collection paths for absPath resolution
+  const collectionsResult = await store.getCollections();
+  const collectionPaths = new Map<string, string>();
+  if (collectionsResult.ok) {
+    for (const c of collectionsResult.value) {
+      collectionPaths.set(c.name, c.path);
+    }
+  }
+
+  // Build results
+  const results: SearchResult[] = [];
+
+  // Pre-fetch all chunks in one batch query (eliminates N+1)
+  const uniqueHashes = [
+    ...new Set(
+      ftsResult.value.map((f) => f.mirrorHash).filter((h): h is string => !!h)
+    ),
+  ];
+  const chunksMapResult = await store.getChunksBatch(uniqueHashes);
+  const getChunk = chunksMapResult.ok
+    ? createChunkLookup(chunksMapResult.value)
+    : () => undefined;
+
+  // Dedup: multiple docs can share mirror_hash (content-addressed storage)
+  // Track seen uri+seq to eliminate duplicate rows from join fan-out
+  // Robust key: use uri if present, else fall back to mirrorHash+relPath
+  const seenUriSeq = new Set<string>();
+  // For --full, track best score per docid to de-dupe
+  const bestByDocid = new Map<
+    string,
+    { fts: FtsResult; chunk: ChunkRow | null; score: number }
+  >();
+
+  for (const fts of ftsResult.value) {
+    // Dedup by uri+seq - eliminates rows from mirror_hash join fan-out
+    // Use robust key to avoid over-dedup if uri is unexpectedly missing
+    const uriSeqKey = fts.uri
+      ? `${fts.uri}:${fts.seq}`
+      : `${fts.mirrorHash ?? ''}:${fts.seq}:${fts.relPath ?? ''}`;
+    if (seenUriSeq.has(uriSeqKey)) {
+      continue;
+    }
+    seenUriSeq.add(uriSeqKey);
+
+    // Get chunk via O(1) lookup
+    const chunk = fts.mirrorHash
+      ? (getChunk(fts.mirrorHash, fts.seq) ?? null)
+      : null;
+
+    // For --full, de-dupe by docid (keep best scoring chunk per doc)
+    // Raw BM25: smaller (more negative) is better
+    if (options.full) {
+      const docid = fts.docid ?? '';
+      const existing = bestByDocid.get(docid);
+      if (!existing || fts.score < existing.score) {
+        bestByDocid.set(docid, { fts, chunk, score: fts.score });
+      }
+      continue;
+    }
+
+    const collectionPath = fts.collection
+      ? collectionPaths.get(fts.collection)
+      : undefined;
+
+    results.push(buildSearchResult({ fts, chunk, collectionPath, options }));
+  }
+
+  // For --full, fetch full content and build results
+  if (options.full) {
+    for (const { fts, chunk } of bestByDocid.values()) {
+      let fullContent: string | undefined;
+      if (fts.mirrorHash) {
+        const contentResult = await store.getContent(fts.mirrorHash);
+        if (contentResult.ok && contentResult.value) {
+          fullContent = contentResult.value;
+        }
+      }
+      const collectionPath = fts.collection
+        ? collectionPaths.get(fts.collection)
+        : undefined;
+      results.push(
+        buildSearchResult({ fts, chunk, collectionPath, options, fullContent })
+      );
+    }
+  }
+
+  // Normalize scores to 0-1 range (batch min-max)
+  normalizeBm25Scores(results);
+
+  // Apply minScore filter after normalization
+  const filteredResults =
+    minScore > 0 ? results.filter((r) => r.score >= minScore) : results;
+
+  return ok({
+    results: filteredResults,
+    meta: {
+      query,
+      mode: 'bm25',
+      totalResults: filteredResults.length,
+      collection: options.collection,
+      lang: options.lang,
+      queryLanguage,
+    },
+  });
+}