@gmickel/gno 0.15.1 → 0.17.0

package/src/pipeline/hybrid.ts

@@ -26,14 +26,27 @@ import {
   buildExplainResults,
   type ExpansionStatus,
   explainBm25,
+  explainCounters,
   explainExpansion,
   explainFusion,
+  explainQueryModes,
   explainRerank,
+  explainTimings,
   explainVector,
 } from "./explain";
 import { type RankedInput, rrfFuse, toRankedInput } from "./fusion";
 import { detectQueryLanguage } from "./query-language";
+import {
+  buildExpansionFromQueryModes,
+  summarizeQueryModes,
+} from "./query-modes";
 import { rerankCandidates } from "./rerank";
+import {
+  isWithinTemporalRange,
+  resolveRecencyTimestamp,
+  resolveTemporalRange,
+  shouldSortByRecency,
+} from "./temporal";
 import { DEFAULT_PIPELINE_CONFIG } from "./types";
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -94,6 +107,10 @@ async function checkBm25Strength(
     lang?: string;
     tagsAll?: string[];
     tagsAny?: string[];
+    since?: string;
+    until?: string;
+    categories?: string[];
+    author?: string;
   }
 ): Promise<boolean> {
   const result = await store.searchFts(query, {
@@ -102,6 +119,10 @@ async function checkBm25Strength(
     language: options?.lang,
     tagsAll: options?.tagsAll,
     tagsAny: options?.tagsAny,
+    since: options?.since,
+    until: options?.until,
+    categories: options?.categories,
+    author: options?.author,
   });
 
   if (!result.ok || result.value.length === 0) {
@@ -143,6 +164,10 @@ async function searchFtsChunks(
     lang?: string;
     tagsAll?: string[];
     tagsAny?: string[];
+    since?: string;
+    until?: string;
+    categories?: string[];
+    author?: string;
   }
 ): Promise<FtsChunksResult> {
   const result = await store.searchFts(query, {
@@ -151,6 +176,10 @@ async function searchFtsChunks(
     language: options.lang,
     tagsAll: options.tagsAll,
     tagsAny: options.tagsAny,
+    since: options.since,
+    until: options.until,
+    categories: options.categories,
+    author: options.author,
   });
   if (!result.ok) {
     // Propagate INVALID_INPUT for FTS syntax errors
@@ -217,23 +246,54 @@ export async function searchHybrid(
   query: string,
   options: HybridSearchOptions = {}
 ): Promise<ReturnType<typeof ok<SearchResults>>> {
+  const runStartedAt = performance.now();
   const { store, vectorIndex, embedPort, genPort, rerankPort } = deps;
   const pipelineConfig = deps.pipelineConfig ?? DEFAULT_PIPELINE_CONFIG;
 
   const limit = options.limit ?? 20;
+  const recencySort = shouldSortByRecency(query);
+  const temporalRange = resolveTemporalRange(
+    query,
+    options.since,
+    options.until
+  );
   const explainLines: ExplainLine[] = [];
   let expansion: ExpansionResult | null = null;
+  const timings = {
+    langMs: 0,
+    expansionMs: 0,
+    bm25Ms: 0,
+    vectorMs: 0,
+    fusionMs: 0,
+    rerankMs: 0,
+    assemblyMs: 0,
+    totalMs: 0,
+  };
+  const counters = {
+    expansionCacheHits: 0,
+    expansionCacheLookups: 0,
+    rerankCacheHits: 0,
+    rerankCacheLookups: 0,
+    fallbackEvents: [] as string[],
+  };
 
-  // When tag filters are present, increase retrieval limits since vector results
-  // are filtered post-retrieval and we need more candidates to fill the limit
-  const hasTagFilters = options.tagsAll?.length || options.tagsAny?.length;
-  const retrievalMultiplier = hasTagFilters ? 3 : 1;
+  // Increase retrieval limits when post-retrieval filters are active.
+  const hasPostFilters = Boolean(
+    options.tagsAll?.length ||
+    options.tagsAny?.length ||
+    options.categories?.length ||
+    options.author ||
+    temporalRange.since ||
+    temporalRange.until
+  );
+  const retrievalMultiplier = hasPostFilters || recencySort ? 3 : 1;
 
   // ─────────────────────────────────────────────────────────────────────────
   // 0. Detect query language for PROMPT SELECTION only
   //    CRITICAL: Detection does NOT change retrieval filters - options.lang does
   //    Priority: queryLanguageHint (MCP) > lang (CLI) > detection
   // ─────────────────────────────────────────────────────────────────────────
+  const langStartedAt = performance.now();
   const detection = detectQueryLanguage(query);
   // Use explicit hint > lang filter > detected language
   const queryLanguage =
@@ -250,23 +310,39 @@ export async function searchHybrid(
     langMessage = `queryLanguage=${queryLanguage} (detected${confidence})`;
   }
   explainLines.push({ stage: "lang", message: langMessage });
+  timings.langMs = performance.now() - langStartedAt;
 
   // ─────────────────────────────────────────────────────────────────────────
   // 1. Check if expansion needed
   // ─────────────────────────────────────────────────────────────────────────
+  const expansionStartedAt = performance.now();
   const shouldExpand = !options.noExpand && genPort !== null;
   let expansionStatus: ExpansionStatus = "disabled";
+  let queryModeSummary: ReturnType<typeof summarizeQueryModes> | undefined =
+    undefined;
+
+  if (options.queryModes?.length) {
+    queryModeSummary = summarizeQueryModes(options.queryModes);
+    explainLines.push(explainQueryModes(queryModeSummary));
+    expansion = buildExpansionFromQueryModes(options.queryModes);
+    expansionStatus = "provided";
+  }
 
-  if (shouldExpand) {
+  if (expansionStatus !== "provided" && shouldExpand) {
     const hasStrongSignal = await checkBm25Strength(store, query, {
       collection: options.collection,
       lang: options.lang,
       tagsAll: options.tagsAll,
       tagsAny: options.tagsAny,
+      since: temporalRange.since,
+      until: temporalRange.until,
+      categories: options.categories,
+      author: options.author,
     });
 
     if (hasStrongSignal) {
       expansionStatus = "skipped_strong";
+      counters.fallbackEvents.push("expansion_skipped_strong");
     } else {
       expansionStatus = "attempted";
       const expandResult = await expandQuery(genPort, query, {
@@ -279,21 +355,31 @@ export async function searchHybrid(
       }
     }
   }
+  if (expansionStatus === "disabled") {
+    counters.fallbackEvents.push("expansion_disabled");
+  }
 
   explainLines.push(explainExpansion(expansionStatus, expansion));
+  timings.expansionMs = performance.now() - expansionStartedAt;
 
   // ─────────────────────────────────────────────────────────────────────────
   // 2. Parallel retrieval using raw store/vector APIs for correct seq tracking
   // ─────────────────────────────────────────────────────────────────────────
   const rankedInputs: RankedInput[] = [];
 
+  const bm25StartedAt = performance.now();
+
   // BM25: original query
   const bm25Result = await searchFtsChunks(store, query, {
-    limit: limit * 2,
+    limit: limit * 2 * retrievalMultiplier,
     collection: options.collection,
     lang: options.lang,
     tagsAll: options.tagsAll,
     tagsAny: options.tagsAny,
+    since: temporalRange.since,
+    until: temporalRange.until,
+    categories: options.categories,
+    author: options.author,
   });
 
   // Propagate FTS syntax errors as INVALID_INPUT
@@ -308,21 +394,35 @@ export async function searchHybrid(
     rankedInputs.push(toRankedInput("bm25", bm25Chunks));
   }
 
-  // BM25: lexical variants (syntax errors here are ignored - variants are optional)
-  if (expansion?.lexicalQueries) {
-    for (const variant of expansion.lexicalQueries) {
-      const variantResult = await searchFtsChunks(store, variant, {
-        limit,
-        collection: options.collection,
-        lang: options.lang,
-        tagsAll: options.tagsAll,
-        tagsAny: options.tagsAny,
-      });
+  // BM25: lexical variants (optional; run in parallel and ignore failures)
+  if (expansion?.lexicalQueries?.length) {
+    const lexicalVariantResults = await Promise.allSettled(
+      expansion.lexicalQueries.map((variant) =>
+        searchFtsChunks(store, variant, {
+          limit: limit * retrievalMultiplier,
+          collection: options.collection,
+          lang: options.lang,
+          tagsAll: options.tagsAll,
+          tagsAny: options.tagsAny,
+          since: temporalRange.since,
+          until: temporalRange.until,
+          categories: options.categories,
+          author: options.author,
+        })
+      )
+    );
+
+    for (const settled of lexicalVariantResults) {
+      if (settled.status !== "fulfilled") {
+        continue;
+      }
+      const variantResult = settled.value;
       if (variantResult.ok && variantResult.chunks.length > 0) {
         rankedInputs.push(toRankedInput("bm25_variant", variantResult.chunks));
       }
     }
   }
+  timings.bm25Ms = performance.now() - bm25StartedAt;
 
   explainLines.push(explainBm25(bm25Count));
 
@@ -330,9 +430,14 @@ export async function searchHybrid(
   let vecCount = 0;
   const vectorAvailable =
     (vectorIndex?.searchAvailable && embedPort !== null) ?? false;
+  if (!vectorAvailable) {
+    counters.fallbackEvents.push("vector_unavailable");
+  }
+
+  const vectorStartedAt = performance.now();
 
   if (vectorAvailable && vectorIndex && embedPort) {
-    // Original query (increase limit when tag filters active since filtering is post-retrieval)
+    // Original query (increase limit when post-filters are active).
     const vecChunks = await searchVectorChunks(vectorIndex, embedPort, query, {
       limit: limit * 2 * retrievalMultiplier,
     });
@@ -342,41 +447,47 @@ export async function searchHybrid(
       rankedInputs.push(toRankedInput("vector", vecChunks));
     }
 
-    // Semantic variants
-    if (expansion?.vectorQueries) {
-      for (const variant of expansion.vectorQueries) {
-        const variantChunks = await searchVectorChunks(
-          vectorIndex,
-          embedPort,
-          variant,
-          { limit: limit * retrievalMultiplier }
-        );
-        if (variantChunks.length > 0) {
-          rankedInputs.push(toRankedInput("vector_variant", variantChunks));
-        }
-      }
-    }
-
-    // HyDE
-    if (expansion?.hyde) {
-      const hydeChunks = await searchVectorChunks(
-        vectorIndex,
-        embedPort,
-        expansion.hyde,
-        { limit: limit * retrievalMultiplier }
+    // Semantic variants + HyDE (optional; run in parallel and ignore failures)
+    const vectorVariantQueries = [
+      ...(expansion?.vectorQueries?.map((query) => ({
+        source: "vector_variant" as const,
+        query,
+      })) ?? []),
+      ...(expansion?.hyde
+        ? [{ source: "hyde" as const, query: expansion.hyde }]
+        : []),
+    ];
+
+    if (vectorVariantQueries.length > 0) {
+      const optionalVectorResults = await Promise.allSettled(
+        vectorVariantQueries.map((variant) =>
+          searchVectorChunks(vectorIndex, embedPort, variant.query, {
+            limit: limit * retrievalMultiplier,
+          })
+        )
       );
-      if (hydeChunks.length > 0) {
-        rankedInputs.push(toRankedInput("hyde", hydeChunks));
+
+      for (const [index, settled] of optionalVectorResults.entries()) {
+        if (settled.status !== "fulfilled" || settled.value.length === 0) {
+          continue;
+        }
+        const variant = vectorVariantQueries[index];
+        if (variant) {
+          rankedInputs.push(toRankedInput(variant.source, settled.value));
+        }
       }
     }
   }
+  timings.vectorMs = performance.now() - vectorStartedAt;
 
   explainLines.push(explainVector(vecCount, vectorAvailable));
 
   // ─────────────────────────────────────────────────────────────────────────
   // 3. RRF Fusion
   // ─────────────────────────────────────────────────────────────────────────
+  const fusionStartedAt = performance.now();
   const fusedCandidates = rrfFuse(rankedInputs, pipelineConfig.rrf);
+  timings.fusionMs = performance.now() - fusionStartedAt;
   explainLines.push(
     explainFusion(pipelineConfig.rrf.k, fusedCandidates.length)
   );
@@ -384,12 +495,22 @@ export async function searchHybrid(
   // ─────────────────────────────────────────────────────────────────────────
   // 4. Reranking
   // ─────────────────────────────────────────────────────────────────────────
+  const rerankStartedAt = performance.now();
   const rerankResult = await rerankCandidates(
     { rerankPort: options.noRerank ? null : rerankPort, store },
     query,
     fusedCandidates,
-    { maxCandidates: pipelineConfig.rerankCandidates }
+    {
+      maxCandidates: pipelineConfig.rerankCandidates,
+      blendingSchedule: pipelineConfig.blendingSchedule,
+    }
   );
+  if (rerankResult.fallbackReason === "disabled") {
+    counters.fallbackEvents.push("rerank_disabled");
+  } else if (rerankResult.fallbackReason === "error") {
+    counters.fallbackEvents.push("rerank_error");
+  }
+  timings.rerankMs = performance.now() - rerankStartedAt;
 
   explainLines.push(
     explainRerank(
@@ -411,21 +532,50 @@ export async function searchHybrid(
   // 5. Build final results (optimized: batch lookups, no per-candidate queries)
   // ─────────────────────────────────────────────────────────────────────────
 
-  // Collect unique mirrorHashes needed from candidates
-  // TODO: For large corpora (100k+ docs), add store.getDocumentsByMirrorHashes
-  // batch lookup to avoid loading all documents into memory.
+  const assemblyStartedAt = performance.now();
+
+  // Collect unique mirrorHashes needed from candidates.
   const neededHashes = new Set(filteredCandidates.map((c) => c.mirrorHash));
 
-  // Fetch documents and collections
-  const docsResult = await store.listDocuments(options.collection);
+  // Fetch only needed documents and collections.
+  const docsResult = await store.getDocumentsByMirrorHashes([...neededHashes], {
+    collection: options.collection,
+    activeOnly: true,
+  });
   const collectionsResult = await store.getCollections();
 
   if (!docsResult.ok) {
     return err("QUERY_FAILED", docsResult.error.message);
   }
 
-  // Build lookup maps - only include docs needed by candidates
+  // Build lookup maps.
   const docByMirrorHash = new Map<string, (typeof docsResult.value)[number]>();
+  const matchesMetadataFilters = (
+    doc: (typeof docsResult.value)[number]
+  ): boolean => {
+    if (!isWithinTemporalRange(doc.sourceMtime, temporalRange)) {
+      return false;
+    }
+    if (
+      options.author &&
+      !doc.author?.toLowerCase().includes(options.author.toLowerCase())
+    ) {
+      return false;
+    }
+    if (options.categories?.length) {
+      const allowed = new Set(options.categories.map((c) => c.toLowerCase()));
+      const contentTypeMatch = doc.contentType
+        ? allowed.has(doc.contentType.toLowerCase())
+        : false;
+      const categoryMatch = (doc.categories ?? []).some((c) =>
+        allowed.has(c.toLowerCase())
+      );
+      if (!contentTypeMatch && !categoryMatch) {
+        return false;
+      }
+    }
+    return true;
+  };
 
   // Collect doc IDs that need tag filtering
   const needsTagFilter = options.tagsAll?.length || options.tagsAny?.length;
@@ -433,11 +583,14 @@ export async function searchHybrid(
   const candidateDocs: (typeof docsResult.value)[number][] = [];
 
   for (const doc of docsResult.value) {
-    if (doc.active && doc.mirrorHash && neededHashes.has(doc.mirrorHash)) {
-      if (needsTagFilter) {
-        docIdsForTagCheck.push(doc.id);
-        candidateDocs.push(doc);
-      } else {
+    if (!doc.mirrorHash) {
+      continue;
+    }
+    if (needsTagFilter) {
+      docIdsForTagCheck.push(doc.id);
+      candidateDocs.push(doc);
+    } else {
+      if (matchesMetadataFilters(doc)) {
         docByMirrorHash.set(doc.mirrorHash, doc);
       }
     }
@@ -465,7 +618,7 @@ export async function searchHybrid(
           if (!hasAny) continue;
         }
 
-        if (doc.mirrorHash) {
+        if (doc.mirrorHash && matchesMetadataFilters(doc)) {
           docByMirrorHash.set(doc.mirrorHash, doc);
         }
       }
@@ -494,6 +647,7 @@ export async function searchHybrid(
   >();
 
   const results: SearchResult[] = [];
+  const assemblyLimit = recencySort ? limit * 3 : limit;
   const docidMap = new Map<string, string>();
   // Track seen docids for --full de-duplication
   const seenDocids = new Set<string>();
@@ -501,7 +655,7 @@ export async function searchHybrid(
   // Iterate until we have enough results (don't slice early - deduping may skip candidates)
   for (const candidate of filteredCandidates) {
     // Stop when we have enough results
-    if (results.length >= limit) {
+    if (results.length >= assemblyLimit) {
       break;
     }
 
@@ -577,6 +731,7 @@ export async function searchHybrid(
         mime: doc.sourceMime,
         ext: doc.sourceExt,
         modifiedAt: doc.sourceMtime,
+        documentDate: doc.frontmatterDate ?? undefined,
         sizeBytes: doc.sourceSize,
         sourceHash: doc.sourceHash,
       },
@@ -587,6 +742,10 @@ export async function searchHybrid(
       },
     });
   }
+  timings.assemblyMs = performance.now() - assemblyStartedAt;
+  timings.totalMs = performance.now() - runStartedAt;
+  explainLines.push(explainTimings(timings));
+  explainLines.push(explainCounters(counters));
 
   // ─────────────────────────────────────────────────────────────────────────
   // 6. Build explain data (if requested)
@@ -604,18 +763,42 @@ export async function searchHybrid(
   // ─────────────────────────────────────────────────────────────────────────
   // 7. Return results
   // ─────────────────────────────────────────────────────────────────────────
+  if (recencySort) {
+    results.sort((a, b) => {
+      const aTs = resolveRecencyTimestamp(
+        a.source.documentDate,
+        a.source.modifiedAt
+      );
+      const bTs = resolveRecencyTimestamp(
+        b.source.documentDate,
+        b.source.modifiedAt
+      );
+      if (aTs !== bTs) {
+        return bTs - aTs;
+      }
+      return b.score - a.score;
+    });
+  }
+
+  const finalResults = results.slice(0, limit);
+
   return ok({
-    results,
+    results: finalResults,
     meta: {
       query,
       mode: vectorAvailable ? "hybrid" : "bm25_only",
       expanded: expansion !== null,
       reranked: rerankResult.reranked,
       vectorsUsed: vectorAvailable,
-      totalResults: results.length,
+      totalResults: finalResults.length,
       collection: options.collection,
       lang: options.lang,
+      since: temporalRange.since,
+      until: temporalRange.until,
+      categories: options.categories,
+      author: options.author,
       queryLanguage,
+      queryModes: queryModeSummary,
       explain: explainData,
     },
   });

package/src/pipeline/query-modes.ts

@@ -0,0 +1,125 @@
+/**
+ * Structured query mode parsing and normalization.
+ *
+ * @module src/pipeline/query-modes
+ */
+
+import type {
+  ExpansionResult,
+  QueryMode,
+  QueryModeInput,
+  QueryModeSummary,
+} from "./types";
+
+import { err, ok, type StoreResult } from "../store/types";
+
+const QUERY_MODE_ENTRY = /^\s*(term|intent|hyde)\s*:\s*([\s\S]*\S[\s\S]*)\s*$/i;
+
+/**
+ * Parse a single CLI/API query mode spec in `mode:text` form.
+ */
+export function parseQueryModeSpec(spec: string): StoreResult<QueryModeInput> {
+  const match = spec.match(QUERY_MODE_ENTRY);
+  if (!match) {
+    return err(
+      "INVALID_INPUT",
+      `Invalid --query-mode value "${spec}". Expected "term:<text>", "intent:<text>", or "hyde:<text>".`
+    );
+  }
+
+  const mode = match[1]?.toLowerCase() as QueryMode | undefined;
+  const text = match[2]?.trim();
+  if (!mode || !text) {
+    return err(
+      "INVALID_INPUT",
+      `Invalid --query-mode value "${spec}". Expected non-empty text after mode prefix.`
+    );
+  }
+
+  return ok({ mode, text });
+}
+
+/**
+ * Parse and validate repeated query mode specs.
+ */
+export function parseQueryModeSpecs(
+  specs: string[]
+): StoreResult<QueryModeInput[]> {
+  const parsed: QueryModeInput[] = [];
+  let hydeCount = 0;
+
+  for (const spec of specs) {
+    const entry = parseQueryModeSpec(spec);
+    if (!entry.ok) {
+      return entry;
+    }
+    if (entry.value.mode === "hyde") {
+      hydeCount += 1;
+      if (hydeCount > 1) {
+        return err(
+          "INVALID_INPUT",
+          "Only one hyde mode is allowed in structured query input."
+        );
+      }
+    }
+    parsed.push(entry.value);
+  }
+
+  return ok(parsed);
+}
+
+/**
+ * Normalize and summarize query modes for metadata/explain.
+ */
+export function summarizeQueryModes(
+  queryModes: QueryModeInput[]
+): QueryModeSummary {
+  const summary: QueryModeSummary = { term: 0, intent: 0, hyde: false };
+  for (const entry of queryModes) {
+    if (entry.mode === "term") {
+      summary.term += 1;
+    } else if (entry.mode === "intent") {
+      summary.intent += 1;
+    } else {
+      summary.hyde = true;
+    }
+  }
+  return summary;
+}
+
+/**
+ * Convert structured query modes into ExpansionResult shape used by hybrid pipeline.
+ */
+export function buildExpansionFromQueryModes(
+  queryModes: QueryModeInput[]
+): ExpansionResult | null {
+  if (queryModes.length === 0) {
+    return null;
+  }
+
+  const lexicalQueries: string[] = [];
+  const vectorQueries: string[] = [];
+  let hyde: string | undefined;
+
+  for (const entry of queryModes) {
+    if (entry.mode === "term") {
+      lexicalQueries.push(entry.text);
+    } else if (entry.mode === "intent") {
+      vectorQueries.push(entry.text);
+    } else if (!hyde) {
+      hyde = entry.text;
+    }
+  }
+
+  // Preserve existing expansion constraints (max 5 lexical/vector variants).
+  const result: ExpansionResult = {
+    lexicalQueries: [...new Set(lexicalQueries)].slice(0, 5),
+    vectorQueries: [...new Set(vectorQueries)].slice(0, 5),
+  };
+
+  if (hyde) {
+    result.hyde = hyde;
+  }
+
+  return result;
+}