@gmickel/gno 0.3.0

package/src/pipeline/types.ts

@@ -0,0 +1,328 @@
+/**
+ * Search pipeline types.
+ * Defines SearchPipelinePort and related types for search operations.
+ *
+ * @module src/pipeline/types
+ */
+
+import type { StoreResult } from '../store/types';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Search Result Types
+// ─────────────────────────────────────────────────────────────────────────────
+
+/** Source metadata for a search result */
+export interface SearchResultSource {
+  relPath: string;
+  absPath?: string;
+  mime: string;
+  ext: string;
+  modifiedAt?: string;
+  sizeBytes?: number;
+  sourceHash?: string;
+}
+
+/** Conversion metadata for a search result */
+export interface SearchResultConversion {
+  converterId?: string;
+  converterVersion?: string;
+  mirrorHash: string;
+  warnings?: { code: string; message: string }[];
+}
+
+/** Snippet range in mirror content */
+export interface SnippetRange {
+  startLine: number;
+  endLine: number;
+}
+
+/** Single search result matching output schema */
+export interface SearchResult {
+  docid: string;
+  score: number;
+  uri: string;
+  title?: string;
+  snippet: string;
+  snippetLanguage?: string;
+  snippetRange?: SnippetRange;
+  context?: string;
+  source: SearchResultSource;
+  conversion?: SearchResultConversion;
+}
+
+/** Search mode enum */
+export type SearchMode = 'bm25' | 'vector' | 'hybrid' | 'bm25_only';
+
+/** Search metadata */
+export interface SearchMeta {
+  query: string;
+  mode: SearchMode;
+  expanded?: boolean;
+  reranked?: boolean;
+  vectorsUsed?: boolean;
+  totalResults: number;
+  collection?: string;
+  lang?: string;
+  /** Detected/overridden query language for prompt selection (typically BCP-47; may be user-provided via --lang) */
+  queryLanguage?: string;
+  /** Explain data (when --explain is used) */
+  explain?: {
+    lines: ExplainLine[];
+    results: ExplainResult[];
+  };
+}
+
+/** Complete search results wrapper */
+export interface SearchResults {
+  results: SearchResult[];
+  meta: SearchMeta;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Search Options
+// ─────────────────────────────────────────────────────────────────────────────
+
+/** Common options for all search commands */
+export interface SearchOptions {
+  /** Max results */
+  limit?: number;
+  /** Min score threshold (0-1) */
+  minScore?: number;
+  /** Filter by collection */
+  collection?: string;
+  /** Language filter/hint (BCP-47) */
+  lang?: string;
+  /** Include full content instead of snippet */
+  full?: boolean;
+  /** Include line numbers */
+  lineNumbers?: boolean;
+}
+
+/** Options for hybrid search (gno query) */
+export type HybridSearchOptions = SearchOptions & {
+  /** Disable query expansion */
+  noExpand?: boolean;
+  /** Disable reranking */
+  noRerank?: boolean;
+  /** Enable explain output */
+  explain?: boolean;
+  /** Language hint for prompt selection (does NOT filter retrieval, only affects expansion prompts) */
+  queryLanguageHint?: string;
+};
+
+/** Options for ask command */
+export type AskOptions = HybridSearchOptions & {
+  /** Generate grounded answer */
+  answer?: boolean;
+  /** Force retrieval-only output */
+  noAnswer?: boolean;
+  /** Max tokens for answer */
+  maxAnswerTokens?: number;
+};
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Query Expansion Types
+// ─────────────────────────────────────────────────────────────────────────────
+
+/** Expansion result from LLM */
+export interface ExpansionResult {
+  lexicalQueries: string[];
+  vectorQueries: string[];
+  hyde?: string;
+  notes?: string;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Fusion Types
+// ─────────────────────────────────────────────────────────────────────────────
+
+/** RRF config */
+export interface RrfConfig {
+  /** RRF constant (default: 60) */
+  k: number;
+  /** Weight for BM25 source */
+  bm25Weight: number;
+  /** Weight for vector source */
+  vecWeight: number;
+  /** Bonus for top-rank in both modes */
+  topRankBonus: number;
+  /** Max rank for top-rank bonus */
+  topRankThreshold: number;
+}
+
+/** Default RRF configuration */
+export const DEFAULT_RRF_CONFIG: RrfConfig = {
+  k: 60,
+  bm25Weight: 1.0,
+  vecWeight: 1.0,
+  topRankBonus: 0.1,
+  topRankThreshold: 5,
+};
+
+/** Chunk identifier for fusion tracking */
+export interface ChunkId {
+  mirrorHash: string;
+  seq: number;
+}
+
+/** Source for a fusion candidate */
+export type FusionSource =
+  | 'bm25'
+  | 'vector'
+  | 'bm25_variant'
+  | 'vector_variant'
+  | 'hyde';
+
+/** Fusion candidate with ranks from different sources */
+export interface FusionCandidate {
+  mirrorHash: string;
+  seq: number;
+  bm25Rank: number | null;
+  vecRank: number | null;
+  fusionScore: number;
+  sources: FusionSource[];
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Rerank & Blending Types
+// ─────────────────────────────────────────────────────────────────────────────
+
+/** Blending tier config */
+export interface BlendingTier {
+  maxRank: number;
+  fusionWeight: number;
+  rerankWeight: number;
+}
+
+/** Default blending schedule */
+export const DEFAULT_BLENDING_SCHEDULE: BlendingTier[] = [
+  { maxRank: 3, fusionWeight: 0.75, rerankWeight: 0.25 },
+  { maxRank: 10, fusionWeight: 0.6, rerankWeight: 0.4 },
+  { maxRank: Number.POSITIVE_INFINITY, fusionWeight: 0.4, rerankWeight: 0.6 },
+];
+
+/** Result after reranking */
+export type RerankedCandidate = FusionCandidate & {
+  rerankScore: number | null;
+  blendedScore: number;
+};
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Pipeline Config
+// ─────────────────────────────────────────────────────────────────────────────
+
+/** Search pipeline configuration */
+export interface PipelineConfig {
+  /** Strong BM25 threshold to skip expansion */
+  strongBm25Threshold: number;
+  /** Expansion timeout in ms */
+  expansionTimeout: number;
+  /** Max candidates to rerank */
+  rerankCandidates: number;
+  /** RRF configuration */
+  rrf: RrfConfig;
+  /** Blending schedule */
+  blendingSchedule: BlendingTier[];
+}
+
+/** Default pipeline configuration */
+export const DEFAULT_PIPELINE_CONFIG: PipelineConfig = {
+  strongBm25Threshold: 0.7,
+  expansionTimeout: 5000,
+  rerankCandidates: 20,
+  rrf: DEFAULT_RRF_CONFIG,
+  blendingSchedule: DEFAULT_BLENDING_SCHEDULE,
+};
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Ask Types
+// ─────────────────────────────────────────────────────────────────────────────
+
+/** Citation reference */
+export interface Citation {
+  docid: string;
+  uri: string;
+  startLine?: number;
+  endLine?: number;
+}
+
+/** Ask result metadata */
+export interface AskMeta {
+  expanded: boolean;
+  reranked: boolean;
+  vectorsUsed: boolean;
+  answerGenerated?: boolean;
+  totalResults?: number;
+}
+
+/** Ask command result */
+export interface AskResult {
+  query: string;
+  mode: 'hybrid' | 'bm25_only';
+  queryLanguage: string;
+  answer?: string;
+  citations?: Citation[];
+  results: SearchResult[];
+  meta: AskMeta;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Port Interfaces
+// ─────────────────────────────────────────────────────────────────────────────
+
+/** BM25 search port */
+export interface Bm25SearchPort {
+  search(
+    query: string,
+    options?: SearchOptions
+  ): Promise<StoreResult<SearchResults>>;
+}
+
+/** Vector search port */
+export interface VectorSearchPort {
+  search(
+    query: string,
+    options?: SearchOptions
+  ): Promise<StoreResult<SearchResults>>;
+}
+
+/** Query expansion port */
+export interface ExpansionPort {
+  expand(
+    query: string,
+    lang?: string
+  ): Promise<StoreResult<ExpansionResult | null>>;
+}
+
+/** Hybrid search port */
+export interface HybridSearchPort {
+  search(
+    query: string,
+    options?: HybridSearchOptions
+  ): Promise<StoreResult<SearchResults>>;
+}
+
+/** Ask port */
+export interface AskPort {
+  ask(query: string, options?: AskOptions): Promise<StoreResult<AskResult>>;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Explain Types
+// ─────────────────────────────────────────────────────────────────────────────
+
+/** Explain output line */
+export interface ExplainLine {
+  stage: string;
+  message: string;
+}
+
+/** Detailed explain for a result */
+export interface ExplainResult {
+  rank: number;
+  docid: string;
+  score: number;
+  bm25Score?: number;
+  vecScore?: number;
+  rerankScore?: number;
+}

package/src/pipeline/vsearch.ts

@@ -0,0 +1,348 @@
+/**
+ * Vector search pipeline.
+ * Wraps VectorIndexPort.searchNearest() to produce SearchResults.
+ *
+ * @module src/pipeline/vsearch
+ */
+
+import type { Config } from '../config/types';
+import type { EmbeddingPort } from '../llm/types';
+import type { StorePort } from '../store/types';
+import { err, ok } from '../store/types';
+import type { VectorIndexPort } from '../store/vector/types';
+import { createChunkLookup } from './chunk-lookup';
+import { detectQueryLanguage } from './query-language';
+import type { SearchOptions, SearchResult, SearchResults } from './types';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Score Normalization
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Normalize cosine distance to 0-1 similarity score.
+ * Cosine distance: 0 = identical, 2 = opposite.
+ * Similarity = 1 - (distance / 2)
+ */
+function normalizeVectorScore(distance: number): number {
+  return Math.max(0, Math.min(1, 1 - distance / 2));
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Vector Search Dependencies
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface VectorSearchDeps {
+  store: StorePort;
+  vectorIndex: VectorIndexPort;
+  embedPort: EmbeddingPort;
+  config: Config;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Search Function (with pre-computed embedding)
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Execute vector search with pre-computed embedding.
+ * Use this to avoid double-embedding when caller already has the query vector.
+ */
+// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: search pipeline with expansion, reranking, and scoring
+export async function searchVectorWithEmbedding(
+  deps: VectorSearchDeps,
+  query: string,
+  queryEmbedding: Float32Array,
+  options: SearchOptions = {}
+): Promise<
+  ReturnType<typeof ok<SearchResults>> | ReturnType<typeof err<SearchResults>>
+> {
+  const { store, vectorIndex } = deps;
+  const limit = options.limit ?? 20;
+  const minScore = options.minScore ?? 0;
+
+  // Detect query language for metadata (DOES NOT affect retrieval filtering)
+  const detection = detectQueryLanguage(query);
+  const queryLanguage = options.lang ?? detection.bcp47;
+
+  // Check if vector search is available
+  if (!vectorIndex.searchAvailable) {
+    return err(
+      'VEC_SEARCH_UNAVAILABLE',
+      'Vector search requires sqlite-vec. Run: gno embed'
+    );
+  }
+
+  // Search nearest neighbors
+  const searchResult = await vectorIndex.searchNearest(queryEmbedding, limit, {
+    minScore,
+  });
+
+  if (!searchResult.ok) {
+    return err('QUERY_FAILED', searchResult.error.message);
+  }
+
+  const vecResults = searchResult.value;
+
+  // Get collection paths for absPath resolution
+  const collectionsResult = await store.getCollections();
+  const collectionPaths = new Map<string, string>();
+  if (collectionsResult.ok) {
+    for (const c of collectionsResult.value) {
+      collectionPaths.set(c.name, c.path);
+    }
+  }
+
+  // Cache docs to avoid N+1 queries
+  const docByMirrorHash = await buildDocumentMap(store, options.collection);
+
+  // Pre-fetch all chunks in one batch query (eliminates N+1)
+  const uniqueHashes = [...new Set(vecResults.map((v) => v.mirrorHash))];
+  const chunksMapResult = await store.getChunksBatch(uniqueHashes);
+  if (!chunksMapResult.ok) {
+    return err('QUERY_FAILED', chunksMapResult.error.message);
+  }
+  const chunksMap = chunksMapResult.value;
+  const getChunk = createChunkLookup(chunksMap);
+
+  // Build search results
+  const results: SearchResult[] = [];
+
+  // For --full, track best score per docid to de-dupe
+  const bestByDocid = new Map<
+    string,
+    { doc: DocumentInfo; chunk: ChunkInfo; score: number }
+  >();
+
+  for (const vec of vecResults) {
+    const score = normalizeVectorScore(vec.distance);
+    if (score < minScore) {
+      continue;
+    }
+
+    // Get chunk via O(1) lookup
+    const chunk = getChunk(vec.mirrorHash, vec.seq);
+    if (!chunk) {
+      continue;
+    }
+
+    // STRICT --lang filter: require exact match (excludes null/undefined)
+    if (options.lang && chunk.language !== options.lang) {
+      continue;
+    }
+
+    // Get document (cached)
+    const doc = docByMirrorHash.get(vec.mirrorHash);
+    if (!doc) {
+      continue;
+    }
+
+    // For --full, de-dupe by docid (keep best scoring chunk per doc)
+    if (options.full) {
+      const existing = bestByDocid.get(doc.docid);
+      if (!existing || score > existing.score) {
+        bestByDocid.set(doc.docid, {
+          doc,
+          chunk: {
+            text: chunk.text,
+            language: chunk.language,
+            startLine: chunk.startLine,
+            endLine: chunk.endLine,
+          },
+          score,
+        });
+      }
+      continue;
+    }
+
+    const collectionPath = collectionPaths.get(doc.collection);
+
+    results.push({
+      docid: doc.docid,
+      score,
+      uri: doc.uri,
+      title: doc.title ?? undefined,
+      snippet: chunk.text,
+      snippetLanguage: chunk.language ?? undefined,
+      snippetRange: {
+        startLine: chunk.startLine,
+        endLine: chunk.endLine,
+      },
+      source: {
+        relPath: doc.relPath,
+        absPath: collectionPath
+          ? `${collectionPath}/${doc.relPath}`
+          : undefined,
+        mime: doc.sourceMime,
+        ext: doc.sourceExt,
+        modifiedAt: doc.sourceMtime,
+        sizeBytes: doc.sourceSize,
+        sourceHash: doc.sourceHash,
+      },
+      conversion: doc.mirrorHash
+        ? {
+            mirrorHash: doc.mirrorHash,
+            converterId: doc.converterId ?? undefined,
+            converterVersion: doc.converterVersion ?? undefined,
+          }
+        : undefined,
+    });
+  }
+
+  // For --full, fetch full content and build results
+  if (options.full) {
+    for (const { doc, chunk, score } of bestByDocid.values()) {
+      let fullContent: string | undefined;
+      if (doc.mirrorHash) {
+        const contentResult = await store.getContent(doc.mirrorHash);
+        if (contentResult.ok && contentResult.value) {
+          fullContent = contentResult.value;
+        }
+      }
+
+      const collectionPath = collectionPaths.get(doc.collection);
+
+      results.push({
+        docid: doc.docid,
+        score,
+        uri: doc.uri,
+        title: doc.title ?? undefined,
+        snippet: fullContent ?? chunk.text,
+        snippetLanguage: chunk.language ?? undefined,
+        // --full: no snippetRange (full doc content)
+        snippetRange: fullContent
+          ? undefined
+          : { startLine: chunk.startLine, endLine: chunk.endLine },
+        source: {
+          relPath: doc.relPath,
+          absPath: collectionPath
+            ? `${collectionPath}/${doc.relPath}`
+            : undefined,
+          mime: doc.sourceMime,
+          ext: doc.sourceExt,
+          modifiedAt: doc.sourceMtime,
+          sizeBytes: doc.sourceSize,
+          sourceHash: doc.sourceHash,
+        },
+        conversion: doc.mirrorHash
+          ? {
+              mirrorHash: doc.mirrorHash,
+              converterId: doc.converterId ?? undefined,
+              converterVersion: doc.converterVersion ?? undefined,
+            }
+          : undefined,
+      });
+    }
+  }
+
+  return ok({
+    results,
+    meta: {
+      query,
+      mode: 'vector',
+      vectorsUsed: true,
+      totalResults: results.length,
+      collection: options.collection,
+      lang: options.lang,
+      queryLanguage,
+    },
+  });
+}
+
+/**
+ * Execute vector search and return structured results.
+ * Embeds the query internally - use searchVectorWithEmbedding if you already have the embedding.
+ */
+export async function searchVector(
+  deps: VectorSearchDeps,
+  query: string,
+  options: SearchOptions = {}
+): Promise<
+  ReturnType<typeof ok<SearchResults>> | ReturnType<typeof err<SearchResults>>
+> {
+  const { vectorIndex, embedPort } = deps;
+
+  // Check if vector search is available
+  if (!vectorIndex.searchAvailable) {
+    return err(
+      'VEC_SEARCH_UNAVAILABLE',
+      'Vector search requires sqlite-vec. Run: gno embed'
+    );
+  }
+
+  // Embed query
+  const embedResult = await embedPort.embed(query);
+  if (!embedResult.ok) {
+    return err(
+      'QUERY_FAILED',
+      `Failed to embed query: ${embedResult.error.message}`
+    );
+  }
+
+  const queryEmbedding = new Float32Array(embedResult.value);
+
+  return searchVectorWithEmbedding(deps, query, queryEmbedding, options);
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Helper Types
+// ─────────────────────────────────────────────────────────────────────────────
+
+interface ChunkInfo {
+  text: string;
+  language: string | null;
+  startLine: number;
+  endLine: number;
+}
+
+interface DocumentInfo {
+  docid: string;
+  uri: string;
+  title: string | null;
+  collection: string;
+  relPath: string;
+  sourceHash: string;
+  sourceMime: string;
+  sourceExt: string;
+  sourceMtime: string;
+  sourceSize: number;
+  mirrorHash: string | null;
+  converterId: string | null;
+  converterVersion: string | null;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Helper: Build document map by mirrorHash
+// ─────────────────────────────────────────────────────────────────────────────
+
+async function buildDocumentMap(
+  store: StorePort,
+  collectionFilter?: string
+): Promise<Map<string, DocumentInfo>> {
+  const result = new Map<string, DocumentInfo>();
+
+  const docs = await store.listDocuments(collectionFilter);
+  if (!docs.ok) {
+    return result;
+  }
+
+  for (const doc of docs.value) {
+    if (doc.mirrorHash && doc.active) {
+      result.set(doc.mirrorHash, {
+        docid: doc.docid,
+        uri: doc.uri,
+        title: doc.title,
+        collection: doc.collection,
+        relPath: doc.relPath,
+        sourceHash: doc.sourceHash,
+        sourceMime: doc.sourceMime,
+        sourceExt: doc.sourceExt,
+        sourceMtime: doc.sourceMtime,
+        sourceSize: doc.sourceSize,
+        mirrorHash: doc.mirrorHash,
+        converterId: doc.converterId,
+        converterVersion: doc.converterVersion,
+      });
+    }
+  }
+
+  return result;
+}