@gmickel/gno 0.3.0

package/src/pipeline/expansion.ts

@@ -0,0 +1,256 @@
+/**
+ * Query expansion for hybrid search.
+ * Uses GenerationPort to generate query variants.
+ *
+ * @module src/pipeline/expansion
+ */
+
+import { createHash } from 'node:crypto'; // No Bun alternative for hashing
+import type { GenerationPort } from '../llm/types';
+import type { StoreResult } from '../store/types';
+import { ok } from '../store/types';
+import type { ExpansionResult } from './types';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Constants
+// ─────────────────────────────────────────────────────────────────────────────
+
+const EXPANSION_PROMPT_VERSION = 'v1';
+const DEFAULT_TIMEOUT_MS = 5000;
+const JSON_EXTRACT_PATTERN = /\{[\s\S]*\}/;
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Cache Key Generation
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Generate cache key for expansion results.
+ * Key = SHA256(promptVersion || modelUri || query || lang)
+ */
+export function generateCacheKey(
+  modelUri: string,
+  query: string,
+  lang: string
+): string {
+  const data = [EXPANSION_PROMPT_VERSION, modelUri, query, lang].join('\0');
+  return createHash('sha256').update(data).digest('hex');
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Prompt Templates
+// ─────────────────────────────────────────────────────────────────────────────
+
+const EXPANSION_PROMPT_EN = `You are a query expansion assistant. Given a search query, generate alternative phrasings to improve search results.
+
+Input query: "{query}"
+
+Generate a JSON object with:
+- "lexicalQueries": array of 2-3 keyword-based variations (for BM25 search)
+- "vectorQueries": array of 2-3 semantic rephrasing (for embedding search)
+- "hyde": a short hypothetical document passage that would answer the query (optional)
+
+Respond ONLY with valid JSON, no explanation.
+
+Example:
+{
+  "lexicalQueries": ["deployment process", "how to deploy", "deploying application"],
+  "vectorQueries": ["steps to release software to production", "guide for application deployment"],
+  "hyde": "To deploy the application, first run the build command, then push to the staging environment..."
+}`;
+
+const EXPANSION_PROMPT_DE = `Du bist ein Query-Erweiterungs-Assistent. Generiere alternative Formulierungen für die Suchanfrage.
+
+Suchanfrage: "{query}"
+
+Generiere ein JSON-Objekt mit:
+- "lexicalQueries": Array mit 2-3 Keyword-Variationen (für BM25-Suche)
+- "vectorQueries": Array mit 2-3 semantischen Umformulierungen (für Vektor-Suche)
+- "hyde": Ein kurzer hypothetischer Dokumentenausschnitt, der die Anfrage beantworten würde (optional)
+
+Antworte NUR mit validem JSON, keine Erklärung.`;
+
+const EXPANSION_PROMPT_MULTILINGUAL = `You are a query expansion assistant. Generate alternative phrasings for the search query in the same language as the query.
+
+Input query: "{query}"
+
+Generate a JSON object with:
+- "lexicalQueries": array of 2-3 keyword-based variations
+- "vectorQueries": array of 2-3 semantic rephrasing
+- "hyde": a short hypothetical document passage (optional)
+
+Respond ONLY with valid JSON.`;
+
+/**
+ * Get prompt template for language.
+ */
+function getPromptTemplate(lang?: string): string {
+  switch (lang?.toLowerCase()) {
+    case 'en':
+    case 'en-us':
+    case 'en-gb':
+      return EXPANSION_PROMPT_EN;
+    case 'de':
+    case 'de-de':
+    case 'de-at':
+    case 'de-ch':
+      return EXPANSION_PROMPT_DE;
+    default:
+      return EXPANSION_PROMPT_MULTILINGUAL;
+  }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Schema Validation
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Validate and parse expansion result from LLM output.
+ */
+function parseExpansionResult(output: string): ExpansionResult | null {
+  try {
+    // Try to extract JSON from output (model may include extra text)
+    const jsonMatch = output.match(JSON_EXTRACT_PATTERN);
+    if (!jsonMatch) {
+      return null;
+    }
+
+    const parsed = JSON.parse(jsonMatch[0]) as Record<string, unknown>;
+
+    // Validate required fields
+    if (!Array.isArray(parsed.lexicalQueries)) {
+      return null;
+    }
+    if (!Array.isArray(parsed.vectorQueries)) {
+      return null;
+    }
+
+    // Validate array contents are strings
+    const lexicalQueries = parsed.lexicalQueries.filter(
+      (q): q is string => typeof q === 'string' && q.length > 0
+    );
+    const vectorQueries = parsed.vectorQueries.filter(
+      (q): q is string => typeof q === 'string' && q.length > 0
+    );
+
+    // Limit array sizes
+    const result: ExpansionResult = {
+      lexicalQueries: lexicalQueries.slice(0, 5),
+      vectorQueries: vectorQueries.slice(0, 5),
+    };
+
+    // Optional fields
+    if (typeof parsed.hyde === 'string' && parsed.hyde.length > 0) {
+      result.hyde = parsed.hyde;
+    }
+    if (typeof parsed.notes === 'string') {
+      result.notes = parsed.notes;
+    }
+
+    return result;
+  } catch {
+    return null;
+  }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Expansion Function
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface ExpansionOptions {
+  /** Language hint for prompt selection */
+  lang?: string;
+  /** Timeout in milliseconds */
+  timeout?: number;
+}
+
+/**
+ * Expand query using generation model.
+ * Returns null on timeout or parse failure (graceful degradation).
+ */
+export async function expandQuery(
+  genPort: GenerationPort,
+  query: string,
+  options: ExpansionOptions = {}
+): Promise<StoreResult<ExpansionResult | null>> {
+  const timeout = options.timeout ?? DEFAULT_TIMEOUT_MS;
+
+  // Build prompt
+  const template = getPromptTemplate(options.lang);
+  const prompt = template.replace('{query}', query);
+
+  // Run with timeout
+  const timeoutPromise = new Promise<null>((resolve) => {
+    setTimeout(() => resolve(null), timeout);
+  });
+
+  try {
+    const result = await Promise.race([
+      genPort.generate(prompt, {
+        temperature: 0,
+        seed: 42,
+        maxTokens: 512,
+      }),
+      timeoutPromise,
+    ]);
+
+    // Timeout
+    if (result === null) {
+      return ok(null);
+    }
+
+    // Generation failed
+    if (!result.ok) {
+      return ok(null); // Graceful degradation
+    }
+
+    // Parse result
+    const parsed = parseExpansionResult(result.value);
+    return ok(parsed);
+  } catch {
+    return ok(null); // Graceful degradation
+  }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Cached Expansion
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface CachedExpansionDeps {
+  genPort: GenerationPort;
+  getCache: (key: string) => Promise<string | null>;
+  setCache: (key: string, value: string) => Promise<void>;
+}
+
+/**
+ * Expand query with caching.
+ */
+export async function expandQueryCached(
+  deps: CachedExpansionDeps,
+  query: string,
+  options: ExpansionOptions = {}
+): Promise<StoreResult<ExpansionResult | null>> {
+  const lang = options.lang ?? 'auto';
+  const cacheKey = generateCacheKey(deps.genPort.modelUri, query, lang);
+
+  // Check cache
+  const cached = await deps.getCache(cacheKey);
+  if (cached) {
+    const parsed = parseExpansionResult(cached);
+    if (parsed) {
+      return ok(parsed);
+    }
+  }
+
+  // Generate
+  const result = await expandQuery(deps.genPort, query, options);
+  if (!result.ok) {
+    return result;
+  }
+
+  // Cache result
+  if (result.value) {
+    await deps.setCache(cacheKey, JSON.stringify(result.value));
+  }
+
+  return result;
+}

package/src/pipeline/explain.ts

@@ -0,0 +1,115 @@
+/**
+ * Explainability formatter for search pipeline.
+ * Outputs pipeline details to stderr.
+ *
+ * @module src/pipeline/explain
+ */
+
+import type {
+  ExpansionResult,
+  ExplainLine,
+  ExplainResult,
+  RerankedCandidate,
+} from './types';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Formatter
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Format explain lines for stderr output.
+ */
+export function formatExplain(lines: ExplainLine[]): string {
+  return lines.map((l) => `[explain] ${l.stage}: ${l.message}`).join('\n');
+}
+
+/**
+ * Format result explain for --explain output.
+ */
+export function formatResultExplain(results: ExplainResult[]): string {
+  const lines: string[] = [];
+  for (const r of results.slice(0, 10)) {
+    let msg = `score=${r.score.toFixed(2)}`;
+    if (r.bm25Score !== undefined) {
+      msg += ` (bm25=${r.bm25Score.toFixed(2)}`;
+      if (r.vecScore !== undefined) {
+        msg += `, vec=${r.vecScore.toFixed(2)}`;
+      }
+      if (r.rerankScore !== undefined) {
+        msg += `, rerank=${r.rerankScore.toFixed(2)}`;
+      }
+      msg += ')';
+    }
+    lines.push(`[explain] result ${r.rank}: ${r.docid} ${msg}`);
+  }
+  return lines.join('\n');
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Explain Line Builders
+// ─────────────────────────────────────────────────────────────────────────────
+
+export function explainExpansion(
+  enabled: boolean,
+  result: ExpansionResult | null
+): ExplainLine {
+  if (!enabled) {
+    return { stage: 'expansion', message: 'disabled' };
+  }
+  if (!result) {
+    return { stage: 'expansion', message: 'skipped (strong BM25 or timeout)' };
+  }
+  const lex = result.lexicalQueries.length;
+  const sem = result.vectorQueries.length;
+  const hyde = result.hyde ? ', 1 HyDE' : '';
+  return {
+    stage: 'expansion',
+    message: `enabled (${lex} lexical, ${sem} semantic variants${hyde})`,
+  };
+}
+
+export function explainBm25(count: number): ExplainLine {
+  return { stage: 'bm25', message: `${count} candidates` };
+}
+
+export function explainVector(count: number, available: boolean): ExplainLine {
+  if (!available) {
+    return { stage: 'vector', message: 'unavailable (sqlite-vec not loaded)' };
+  }
+  return { stage: 'vector', message: `${count} candidates` };
+}
+
+export function explainFusion(k: number, uniqueCount: number): ExplainLine {
+  return {
+    stage: 'fusion',
+    message: `RRF k=${k}, ${uniqueCount} unique candidates`,
+  };
+}
+
+export function explainRerank(enabled: boolean, count: number): ExplainLine {
+  if (!enabled) {
+    return { stage: 'rerank', message: 'disabled' };
+  }
+  return { stage: 'rerank', message: `top ${count} reranked` };
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Build ExplainResult from RerankedCandidate
+// ─────────────────────────────────────────────────────────────────────────────
+
+export function buildExplainResults(
+  candidates: RerankedCandidate[],
+  docidMap: Map<string, string>
+): ExplainResult[] {
+  return candidates.slice(0, 20).map((c, i) => {
+    const key = `${c.mirrorHash}:${c.seq}`;
+    return {
+      rank: i + 1,
+      docid: docidMap.get(key) ?? '#unknown',
+      score: c.blendedScore,
+      bm25Score: c.bm25Rank !== null ? 1 / (60 + c.bm25Rank) : undefined,
+      vecScore: c.vecRank !== null ? 1 / (60 + c.vecRank) : undefined,
+      rerankScore: c.rerankScore ?? undefined,
+    };
+  });
+}

package/src/pipeline/fusion.ts

@@ -0,0 +1,185 @@
+/**
+ * RRF (Reciprocal Rank Fusion) implementation.
+ * Combines BM25 and vector search results.
+ *
+ * @module src/pipeline/fusion
+ */
+
+import type { FusionCandidate, FusionSource, RrfConfig } from './types';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Types
+// ─────────────────────────────────────────────────────────────────────────────
+
+/** Input for fusion - ranked results from a single source */
+export interface RankedInput {
+  source: FusionSource;
+  results: Array<{
+    mirrorHash: string;
+    seq: number;
+    rank: number; // 1-based rank
+  }>;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// RRF Score Calculation
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Calculate RRF contribution for a single rank.
+ * Formula: weight / (k + rank)
+ */
+function rrfContribution(rank: number, k: number, weight: number): number {
+  return weight / (k + rank);
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Fusion Implementation
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Fuse multiple ranked lists using RRF.
+ *
+ * @param inputs - Array of ranked inputs from different sources
+ * @param config - RRF configuration
+ * @returns Fused and sorted candidates
+ */
+// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: RRF fusion with deduplication and scoring
+export function rrfFuse(
+  inputs: RankedInput[],
+  config: RrfConfig
+): FusionCandidate[] {
+  // Map: "mirrorHash:seq" -> FusionCandidate
+  const candidates = new Map<string, FusionCandidate>();
+
+  // Separate inputs by type for weight assignment
+  const bm25Inputs = inputs.filter(
+    (i) => i.source === 'bm25' || i.source === 'bm25_variant'
+  );
+  const vectorInputs = inputs.filter(
+    (i) =>
+      i.source === 'vector' ||
+      i.source === 'vector_variant' ||
+      i.source === 'hyde'
+  );
+
+  // Process BM25 sources
+  for (const input of bm25Inputs) {
+    const weight =
+      input.source === 'bm25' ? config.bm25Weight : config.bm25Weight * 0.5;
+
+    for (const result of input.results) {
+      const key = `${result.mirrorHash}:${result.seq}`;
+      let candidate = candidates.get(key);
+
+      if (!candidate) {
+        candidate = {
+          mirrorHash: result.mirrorHash,
+          seq: result.seq,
+          bm25Rank: null,
+          vecRank: null,
+          fusionScore: 0,
+          sources: [],
+        };
+        candidates.set(key, candidate);
+      }
+
+      // Track best BM25 rank
+      if (candidate.bm25Rank === null || result.rank < candidate.bm25Rank) {
+        candidate.bm25Rank = result.rank;
+      }
+
+      // Add RRF contribution
+      candidate.fusionScore += rrfContribution(result.rank, config.k, weight);
+      if (!candidate.sources.includes(input.source)) {
+        candidate.sources.push(input.source);
+      }
+    }
+  }
+
+  // Process vector sources
+  for (const input of vectorInputs) {
+    let weight = config.vecWeight;
+    if (input.source === 'vector_variant') {
+      weight = config.vecWeight * 0.5;
+    } else if (input.source === 'hyde') {
+      weight = config.vecWeight * 0.7;
+    }
+
+    for (const result of input.results) {
+      const key = `${result.mirrorHash}:${result.seq}`;
+      let candidate = candidates.get(key);
+
+      if (!candidate) {
+        candidate = {
+          mirrorHash: result.mirrorHash,
+          seq: result.seq,
+          bm25Rank: null,
+          vecRank: null,
+          fusionScore: 0,
+          sources: [],
+        };
+        candidates.set(key, candidate);
+      }
+
+      // Track best vector rank
+      if (candidate.vecRank === null || result.rank < candidate.vecRank) {
+        candidate.vecRank = result.rank;
+      }
+
+      // Add RRF contribution
+      candidate.fusionScore += rrfContribution(result.rank, config.k, weight);
+      if (!candidate.sources.includes(input.source)) {
+        candidate.sources.push(input.source);
+      }
+    }
+  }
+
+  // Apply top-rank bonus
+  for (const candidate of candidates.values()) {
+    if (
+      candidate.bm25Rank !== null &&
+      candidate.bm25Rank <= config.topRankThreshold &&
+      candidate.vecRank !== null &&
+      candidate.vecRank <= config.topRankThreshold
+    ) {
+      candidate.fusionScore += config.topRankBonus;
+    }
+  }
+
+  // Sort by fusion score (descending), then by mirrorHash:seq for determinism
+  const sorted = Array.from(candidates.values()).sort((a, b) => {
+    const scoreDiff = b.fusionScore - a.fusionScore;
+    if (Math.abs(scoreDiff) > 1e-9) {
+      return scoreDiff;
+    }
+    // Deterministic tie-breaking
+    const aKey = `${a.mirrorHash}:${a.seq}`;
+    const bKey = `${b.mirrorHash}:${b.seq}`;
+    return aKey.localeCompare(bKey);
+  });
+
+  return sorted;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Helper: Convert search results to ranked input
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Convert array of (mirrorHash, seq) tuples to RankedInput.
+ * Results are assumed to be in rank order (first = rank 1).
+ */
+export function toRankedInput(
+  source: FusionSource,
+  results: Array<{ mirrorHash: string; seq: number }>
+): RankedInput {
+  return {
+    source,
+    results: results.map((r, i) => ({
+      mirrorHash: r.mirrorHash,
+      seq: r.seq,
+      rank: i + 1,
+    })),
+  };
+}