@gmickel/gno 0.16.0 → 0.18.0

package/src/pipeline/hybrid.ts

@@ -26,14 +26,28 @@ import {
   buildExplainResults,
   type ExpansionStatus,
   explainBm25,
+  explainCounters,
   explainExpansion,
   explainFusion,
+  explainQueryModes,
   explainRerank,
+  explainTimings,
   explainVector,
 } from "./explain";
 import { type RankedInput, rrfFuse, toRankedInput } from "./fusion";
+import { selectBestChunkForSteering } from "./intent";
 import { detectQueryLanguage } from "./query-language";
+import {
+  buildExpansionFromQueryModes,
+  summarizeQueryModes,
+} from "./query-modes";
 import { rerankCandidates } from "./rerank";
+import {
+  isWithinTemporalRange,
+  resolveRecencyTimestamp,
+  resolveTemporalRange,
+  shouldSortByRecency,
+} from "./temporal";
 import { DEFAULT_PIPELINE_CONFIG } from "./types";
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -94,6 +108,10 @@ async function checkBm25Strength(
     lang?: string;
     tagsAll?: string[];
     tagsAny?: string[];
+    since?: string;
+    until?: string;
+    categories?: string[];
+    author?: string;
   }
 ): Promise<boolean> {
   const result = await store.searchFts(query, {
@@ -102,6 +120,10 @@ async function checkBm25Strength(
     language: options?.lang,
     tagsAll: options?.tagsAll,
     tagsAny: options?.tagsAny,
+    since: options?.since,
+    until: options?.until,
+    categories: options?.categories,
+    author: options?.author,
   });
 
   if (!result.ok || result.value.length === 0) {
@@ -143,6 +165,10 @@ async function searchFtsChunks(
     lang?: string;
     tagsAll?: string[];
     tagsAny?: string[];
+    since?: string;
+    until?: string;
+    categories?: string[];
+    author?: string;
   }
 ): Promise<FtsChunksResult> {
   const result = await store.searchFts(query, {
@@ -151,6 +177,10 @@ async function searchFtsChunks(
     language: options.lang,
     tagsAll: options.tagsAll,
     tagsAny: options.tagsAny,
+    since: options.since,
+    until: options.until,
+    categories: options.categories,
+    author: options.author,
   });
   if (!result.ok) {
     // Propagate INVALID_INPUT for FTS syntax errors
@@ -217,23 +247,54 @@ export async function searchHybrid(
   query: string,
   options: HybridSearchOptions = {}
 ): Promise<ReturnType<typeof ok<SearchResults>>> {
+  const runStartedAt = performance.now();
   const { store, vectorIndex, embedPort, genPort, rerankPort } = deps;
   const pipelineConfig = deps.pipelineConfig ?? DEFAULT_PIPELINE_CONFIG;
 
   const limit = options.limit ?? 20;
+  const recencySort = shouldSortByRecency(query);
+  const temporalRange = resolveTemporalRange(
+    query,
+    options.since,
+    options.until
+  );
   const explainLines: ExplainLine[] = [];
   let expansion: ExpansionResult | null = null;
+  const timings = {
+    langMs: 0,
+    expansionMs: 0,
+    bm25Ms: 0,
+    vectorMs: 0,
+    fusionMs: 0,
+    rerankMs: 0,
+    assemblyMs: 0,
+    totalMs: 0,
+  };
+  const counters = {
+    expansionCacheHits: 0,
+    expansionCacheLookups: 0,
+    rerankCacheHits: 0,
+    rerankCacheLookups: 0,
+    fallbackEvents: [] as string[],
+  };
 
-  // When tag filters are present, increase retrieval limits since vector results
-  // are filtered post-retrieval and we need more candidates to fill the limit
-  const hasTagFilters = options.tagsAll?.length || options.tagsAny?.length;
-  const retrievalMultiplier = hasTagFilters ? 3 : 1;
+  // Increase retrieval limits when post-retrieval filters are active.
+  const hasPostFilters = Boolean(
+    options.tagsAll?.length ||
+    options.tagsAny?.length ||
+    options.categories?.length ||
+    options.author ||
+    temporalRange.since ||
+    temporalRange.until
+  );
+  const retrievalMultiplier = hasPostFilters || recencySort ? 3 : 1;
 
   // ─────────────────────────────────────────────────────────────────────────
   // 0. Detect query language for PROMPT SELECTION only
   //    CRITICAL: Detection does NOT change retrieval filters - options.lang does
   //    Priority: queryLanguageHint (MCP) > lang (CLI) > detection
   // ─────────────────────────────────────────────────────────────────────────
+  const langStartedAt = performance.now();
   const detection = detectQueryLanguage(query);
   // Use explicit hint > lang filter > detected language
   const queryLanguage =
@@ -250,50 +311,80 @@ export async function searchHybrid(
     langMessage = `queryLanguage=${queryLanguage} (detected${confidence})`;
   }
   explainLines.push({ stage: "lang", message: langMessage });
+  timings.langMs = performance.now() - langStartedAt;
 
   // ─────────────────────────────────────────────────────────────────────────
   // 1. Check if expansion needed
   // ─────────────────────────────────────────────────────────────────────────
+  const expansionStartedAt = performance.now();
   const shouldExpand = !options.noExpand && genPort !== null;
   let expansionStatus: ExpansionStatus = "disabled";
+  let queryModeSummary: ReturnType<typeof summarizeQueryModes> | undefined =
+    undefined;
+
+  if (options.queryModes?.length) {
+    queryModeSummary = summarizeQueryModes(options.queryModes);
+    explainLines.push(explainQueryModes(queryModeSummary));
+    expansion = buildExpansionFromQueryModes(options.queryModes);
+    expansionStatus = "provided";
+  }
 
-  if (shouldExpand) {
-    const hasStrongSignal = await checkBm25Strength(store, query, {
-      collection: options.collection,
-      lang: options.lang,
-      tagsAll: options.tagsAll,
-      tagsAny: options.tagsAny,
-    });
+  if (expansionStatus !== "provided" && shouldExpand) {
+    const hasStrongSignal = options.intent?.trim()
+      ? false
+      : await checkBm25Strength(store, query, {
+          collection: options.collection,
+          lang: options.lang,
+          tagsAll: options.tagsAll,
+          tagsAny: options.tagsAny,
+          since: temporalRange.since,
+          until: temporalRange.until,
+          categories: options.categories,
+          author: options.author,
+        });
 
     if (hasStrongSignal) {
       expansionStatus = "skipped_strong";
+      counters.fallbackEvents.push("expansion_skipped_strong");
     } else {
       expansionStatus = "attempted";
       const expandResult = await expandQuery(genPort, query, {
         // Use queryLanguage for prompt selection, NOT options.lang (retrieval filter)
         lang: queryLanguage,
         timeout: pipelineConfig.expansionTimeout,
+        intent: options.intent,
+        contextSize: deps.config.models?.expandContextSize,
       });
       if (expandResult.ok) {
         expansion = expandResult.value;
       }
     }
   }
+  if (expansionStatus === "disabled") {
+    counters.fallbackEvents.push("expansion_disabled");
+  }
 
   explainLines.push(explainExpansion(expansionStatus, expansion));
+  timings.expansionMs = performance.now() - expansionStartedAt;
 
   // ─────────────────────────────────────────────────────────────────────────
   // 2. Parallel retrieval using raw store/vector APIs for correct seq tracking
   // ─────────────────────────────────────────────────────────────────────────
   const rankedInputs: RankedInput[] = [];
 
+  const bm25StartedAt = performance.now();
+
   // BM25: original query
   const bm25Result = await searchFtsChunks(store, query, {
-    limit: limit * 2,
+    limit: limit * 2 * retrievalMultiplier,
     collection: options.collection,
     lang: options.lang,
     tagsAll: options.tagsAll,
     tagsAny: options.tagsAny,
+    since: temporalRange.since,
+    until: temporalRange.until,
+    categories: options.categories,
+    author: options.author,
   });
 
   // Propagate FTS syntax errors as INVALID_INPUT
@@ -308,21 +399,35 @@ export async function searchHybrid(
     rankedInputs.push(toRankedInput("bm25", bm25Chunks));
   }
 
-  // BM25: lexical variants (syntax errors here are ignored - variants are optional)
-  if (expansion?.lexicalQueries) {
-    for (const variant of expansion.lexicalQueries) {
-      const variantResult = await searchFtsChunks(store, variant, {
-        limit,
-        collection: options.collection,
-        lang: options.lang,
-        tagsAll: options.tagsAll,
-        tagsAny: options.tagsAny,
-      });
+  // BM25: lexical variants (optional; run in parallel and ignore failures)
+  if (expansion?.lexicalQueries?.length) {
+    const lexicalVariantResults = await Promise.allSettled(
+      expansion.lexicalQueries.map((variant) =>
+        searchFtsChunks(store, variant, {
+          limit: limit * retrievalMultiplier,
+          collection: options.collection,
+          lang: options.lang,
+          tagsAll: options.tagsAll,
+          tagsAny: options.tagsAny,
+          since: temporalRange.since,
+          until: temporalRange.until,
+          categories: options.categories,
+          author: options.author,
+        })
+      )
+    );
+
+    for (const settled of lexicalVariantResults) {
+      if (settled.status !== "fulfilled") {
+        continue;
+      }
+      const variantResult = settled.value;
       if (variantResult.ok && variantResult.chunks.length > 0) {
         rankedInputs.push(toRankedInput("bm25_variant", variantResult.chunks));
       }
     }
   }
+  timings.bm25Ms = performance.now() - bm25StartedAt;
 
   explainLines.push(explainBm25(bm25Count));
 
@@ -330,9 +435,14 @@ export async function searchHybrid(
   let vecCount = 0;
   const vectorAvailable =
     (vectorIndex?.searchAvailable && embedPort !== null) ?? false;
+  if (!vectorAvailable) {
+    counters.fallbackEvents.push("vector_unavailable");
+  }
+
+  const vectorStartedAt = performance.now();
 
   if (vectorAvailable && vectorIndex && embedPort) {
-    // Original query (increase limit when tag filters active since filtering is post-retrieval)
+    // Original query (increase limit when post-filters are active).
     const vecChunks = await searchVectorChunks(vectorIndex, embedPort, query, {
       limit: limit * 2 * retrievalMultiplier,
     });
@@ -342,41 +452,47 @@ export async function searchHybrid(
       rankedInputs.push(toRankedInput("vector", vecChunks));
     }
 
-    // Semantic variants
-    if (expansion?.vectorQueries) {
-      for (const variant of expansion.vectorQueries) {
-        const variantChunks = await searchVectorChunks(
-          vectorIndex,
-          embedPort,
-          variant,
-          { limit: limit * retrievalMultiplier }
-        );
-        if (variantChunks.length > 0) {
-          rankedInputs.push(toRankedInput("vector_variant", variantChunks));
-        }
-      }
-    }
-
-    // HyDE
-    if (expansion?.hyde) {
-      const hydeChunks = await searchVectorChunks(
-        vectorIndex,
-        embedPort,
-        expansion.hyde,
-        { limit: limit * retrievalMultiplier }
+    // Semantic variants + HyDE (optional; run in parallel and ignore failures)
+    const vectorVariantQueries = [
+      ...(expansion?.vectorQueries?.map((query) => ({
+        source: "vector_variant" as const,
+        query,
+      })) ?? []),
+      ...(expansion?.hyde
+        ? [{ source: "hyde" as const, query: expansion.hyde }]
+        : []),
+    ];
+
+    if (vectorVariantQueries.length > 0) {
+      const optionalVectorResults = await Promise.allSettled(
+        vectorVariantQueries.map((variant) =>
+          searchVectorChunks(vectorIndex, embedPort, variant.query, {
+            limit: limit * retrievalMultiplier,
+          })
+        )
       );
-      if (hydeChunks.length > 0) {
-        rankedInputs.push(toRankedInput("hyde", hydeChunks));
+
+      for (const [index, settled] of optionalVectorResults.entries()) {
+        if (settled.status !== "fulfilled" || settled.value.length === 0) {
+          continue;
+        }
+        const variant = vectorVariantQueries[index];
+        if (variant) {
+          rankedInputs.push(toRankedInput(variant.source, settled.value));
+        }
       }
     }
   }
+  timings.vectorMs = performance.now() - vectorStartedAt;
 
   explainLines.push(explainVector(vecCount, vectorAvailable));
 
   // ─────────────────────────────────────────────────────────────────────────
   // 3. RRF Fusion
   // ─────────────────────────────────────────────────────────────────────────
+  const fusionStartedAt = performance.now();
   const fusedCandidates = rrfFuse(rankedInputs, pipelineConfig.rrf);
+  timings.fusionMs = performance.now() - fusionStartedAt;
   explainLines.push(
     explainFusion(pipelineConfig.rrf.k, fusedCandidates.length)
   );
@@ -384,18 +500,28 @@ export async function searchHybrid(
   // ─────────────────────────────────────────────────────────────────────────
   // 4. Reranking
   // ─────────────────────────────────────────────────────────────────────────
+  const rerankStartedAt = performance.now();
+  const candidateLimit =
+    options.candidateLimit ?? pipelineConfig.rerankCandidates;
   const rerankResult = await rerankCandidates(
     { rerankPort: options.noRerank ? null : rerankPort, store },
     query,
     fusedCandidates,
-    { maxCandidates: pipelineConfig.rerankCandidates }
+    {
+      maxCandidates: candidateLimit,
+      blendingSchedule: pipelineConfig.blendingSchedule,
+      intent: options.intent,
+    }
   );
+  if (rerankResult.fallbackReason === "disabled") {
+    counters.fallbackEvents.push("rerank_disabled");
+  } else if (rerankResult.fallbackReason === "error") {
+    counters.fallbackEvents.push("rerank_error");
+  }
+  timings.rerankMs = performance.now() - rerankStartedAt;
 
   explainLines.push(
-    explainRerank(
-      !options.noRerank && rerankPort !== null,
-      pipelineConfig.rerankCandidates
-    )
+    explainRerank(!options.noRerank && rerankPort !== null, candidateLimit)
   );
 
   // ─────────────────────────────────────────────────────────────────────────
@@ -411,21 +537,50 @@ export async function searchHybrid(
   // 5. Build final results (optimized: batch lookups, no per-candidate queries)
   // ─────────────────────────────────────────────────────────────────────────
 
-  // Collect unique mirrorHashes needed from candidates
-  // TODO: For large corpora (100k+ docs), add store.getDocumentsByMirrorHashes
-  // batch lookup to avoid loading all documents into memory.
+  const assemblyStartedAt = performance.now();
+
+  // Collect unique mirrorHashes needed from candidates.
   const neededHashes = new Set(filteredCandidates.map((c) => c.mirrorHash));
 
-  // Fetch documents and collections
-  const docsResult = await store.listDocuments(options.collection);
+  // Fetch only needed documents and collections.
+  const docsResult = await store.getDocumentsByMirrorHashes([...neededHashes], {
+    collection: options.collection,
+    activeOnly: true,
+  });
   const collectionsResult = await store.getCollections();
 
   if (!docsResult.ok) {
     return err("QUERY_FAILED", docsResult.error.message);
   }
 
-  // Build lookup maps - only include docs needed by candidates
+  // Build lookup maps.
   const docByMirrorHash = new Map<string, (typeof docsResult.value)[number]>();
+  const matchesMetadataFilters = (
+    doc: (typeof docsResult.value)[number]
+  ): boolean => {
+    if (!isWithinTemporalRange(doc.sourceMtime, temporalRange)) {
+      return false;
+    }
+    if (
+      options.author &&
+      !doc.author?.toLowerCase().includes(options.author.toLowerCase())
+    ) {
+      return false;
+    }
+    if (options.categories?.length) {
+      const allowed = new Set(options.categories.map((c) => c.toLowerCase()));
+      const contentTypeMatch = doc.contentType
+        ? allowed.has(doc.contentType.toLowerCase())
+        : false;
+      const categoryMatch = (doc.categories ?? []).some((c) =>
+        allowed.has(c.toLowerCase())
+      );
+      if (!contentTypeMatch && !categoryMatch) {
+        return false;
+      }
+    }
+    return true;
+  };
 
   // Collect doc IDs that need tag filtering
   const needsTagFilter = options.tagsAll?.length || options.tagsAny?.length;
@@ -433,11 +588,14 @@ export async function searchHybrid(
   const candidateDocs: (typeof docsResult.value)[number][] = [];
 
   for (const doc of docsResult.value) {
-    if (doc.active && doc.mirrorHash && neededHashes.has(doc.mirrorHash)) {
-      if (needsTagFilter) {
-        docIdsForTagCheck.push(doc.id);
-        candidateDocs.push(doc);
-      } else {
+    if (!doc.mirrorHash) {
+      continue;
+    }
+    if (needsTagFilter) {
+      docIdsForTagCheck.push(doc.id);
+      candidateDocs.push(doc);
+    } else {
+      if (matchesMetadataFilters(doc)) {
         docByMirrorHash.set(doc.mirrorHash, doc);
       }
     }
@@ -465,7 +623,7 @@ export async function searchHybrid(
           if (!hasAny) continue;
         }
 
-        if (doc.mirrorHash) {
+        if (doc.mirrorHash && matchesMetadataFilters(doc)) {
           docByMirrorHash.set(doc.mirrorHash, doc);
         }
       }
@@ -494,6 +652,7 @@ export async function searchHybrid(
   >();
 
   const results: SearchResult[] = [];
+  const assemblyLimit = recencySort ? limit * 3 : limit;
   const docidMap = new Map<string, string>();
   // Track seen docids for --full de-duplication
   const seenDocids = new Set<string>();
@@ -501,7 +660,7 @@ export async function searchHybrid(
   // Iterate until we have enough results (don't slice early - deduping may skip candidates)
   for (const candidate of filteredCandidates) {
     // Stop when we have enough results
-    if (results.length >= limit) {
+    if (results.length >= assemblyLimit) {
       break;
     }
 
@@ -538,10 +697,23 @@ export async function searchHybrid(
     const collectionPath = collectionPaths.get(doc.collection);
 
     // For --full mode, fetch full mirror content
-    let snippet = chunk.text;
+    const snippetChunk =
+      options.full || !options.intent?.trim()
+        ? chunk
+        : (selectBestChunkForSteering(
+            chunksMap.get(candidate.mirrorHash) ?? [],
+            query,
+            options.intent,
+            {
+              preferredSeq: chunk.seq,
+              intentWeight: 0.3,
+            }
+          ) ?? chunk);
+
+    let snippet = snippetChunk.text;
     let snippetRange: { startLine: number; endLine: number } | undefined = {
-      startLine: chunk.startLine,
-      endLine: chunk.endLine,
+      startLine: snippetChunk.startLine,
+      endLine: snippetChunk.endLine,
     };
 
     if (options.full) {
@@ -577,6 +749,7 @@ export async function searchHybrid(
         mime: doc.sourceMime,
         ext: doc.sourceExt,
         modifiedAt: doc.sourceMtime,
+        documentDate: doc.frontmatterDate ?? undefined,
         sizeBytes: doc.sourceSize,
         sourceHash: doc.sourceHash,
       },
@@ -587,6 +760,10 @@ export async function searchHybrid(
       },
     });
   }
+  timings.assemblyMs = performance.now() - assemblyStartedAt;
+  timings.totalMs = performance.now() - runStartedAt;
+  explainLines.push(explainTimings(timings));
+  explainLines.push(explainCounters(counters));
 
   // ─────────────────────────────────────────────────────────────────────────
   // 6. Build explain data (if requested)
@@ -604,18 +781,44 @@ export async function searchHybrid(
   // ─────────────────────────────────────────────────────────────────────────
   // 7. Return results
   // ─────────────────────────────────────────────────────────────────────────
+  if (recencySort) {
+    results.sort((a, b) => {
+      const aTs = resolveRecencyTimestamp(
+        a.source.documentDate,
+        a.source.modifiedAt
+      );
+      const bTs = resolveRecencyTimestamp(
+        b.source.documentDate,
+        b.source.modifiedAt
+      );
+      if (aTs !== bTs) {
+        return bTs - aTs;
+      }
+      return b.score - a.score;
+    });
+  }
+
+  const finalResults = results.slice(0, limit);
+
   return ok({
-    results,
+    results: finalResults,
     meta: {
       query,
       mode: vectorAvailable ? "hybrid" : "bm25_only",
       expanded: expansion !== null,
       reranked: rerankResult.reranked,
       vectorsUsed: vectorAvailable,
-      totalResults: results.length,
+      totalResults: finalResults.length,
+      intent: options.intent,
       collection: options.collection,
       lang: options.lang,
+      since: temporalRange.since,
+      until: temporalRange.until,
+      categories: options.categories,
+      author: options.author,
+      candidateLimit,
       queryLanguage,
+      queryModes: queryModeSummary,
       explain: explainData,
     },
   });