@gmickel/gno 0.17.0 → 0.19.0

package/src/pipeline/intent.ts

@@ -0,0 +1,152 @@
+/**
+ * Intent-aware retrieval helpers.
+ *
+ * @module src/pipeline/intent
+ */
+
+import type { ChunkRow } from "../store/types";
+
+const TOKEN_PATTERN = /[A-Za-z0-9][A-Za-z0-9.+#/_-]*/g;
+
+const STOPWORDS = new Set([
+  "a",
+  "an",
+  "and",
+  "are",
+  "as",
+  "at",
+  "be",
+  "by",
+  "for",
+  "from",
+  "how",
+  "in",
+  "is",
+  "it",
+  "of",
+  "on",
+  "or",
+  "that",
+  "the",
+  "this",
+  "to",
+  "what",
+  "when",
+  "where",
+  "which",
+  "who",
+  "why",
+  "with",
+]);
+
+const MATCH_ANCHOR_BONUS = 0.2;
+
+function dedupe(values: string[]): string[] {
+  return [...new Set(values)];
+}
+
+function normalizeToken(token: string): string {
+  return token.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "").toLowerCase();
+}
+
+/**
+ * Extract meaningful steering terms from query/intent text.
+ * Keeps short domain tokens like API/SQL/LLM while dropping common stop words.
+ */
+export function extractSteeringTerms(text: string): string[] {
+  const matches = text.match(TOKEN_PATTERN) ?? [];
+  const terms: string[] = [];
+
+  for (const rawToken of matches) {
+    const token = normalizeToken(rawToken);
+    if (token.length < 2) {
+      continue;
+    }
+    if (STOPWORDS.has(token)) {
+      continue;
+    }
+    terms.push(token);
+  }
+
+  return dedupe(terms);
+}
+
+function scoreTextForTerms(text: string, terms: string[]): number {
+  if (terms.length === 0 || text.length === 0) {
+    return 0;
+  }
+
+  const haystack = text.toLowerCase();
+  let score = 0;
+
+  for (const term of terms) {
+    if (haystack.includes(term)) {
+      score += 1;
+    }
+  }
+
+  return score;
+}
+
+export interface ChunkSelectionOptions {
+  preferredSeq?: number | null;
+  intentWeight: number;
+}
+
+/**
+ * Choose the most query-relevant chunk in a document, with intent as a softer steer.
+ */
+export function selectBestChunkForSteering(
+  chunks: ChunkRow[],
+  query: string,
+  intent: string | undefined,
+  options: ChunkSelectionOptions
+): ChunkRow | null {
+  if (chunks.length === 0) {
+    return null;
+  }
+
+  const queryTerms = extractSteeringTerms(query);
+  const intentTerms = extractSteeringTerms(intent ?? "");
+  const preferredSeq = options.preferredSeq ?? null;
+  let bestChunk: ChunkRow | null = null;
+  let bestScore = Number.NEGATIVE_INFINITY;
+
+  for (const chunk of chunks) {
+    const queryScore = scoreTextForTerms(chunk.text, queryTerms);
+    const intentScore =
+      scoreTextForTerms(chunk.text, intentTerms) * options.intentWeight;
+    const preferredBonus =
+      preferredSeq !== null && chunk.seq === preferredSeq
+        ? MATCH_ANCHOR_BONUS
+        : 0;
+    const score = queryScore + intentScore + preferredBonus;
+
+    if (score > bestScore) {
+      bestScore = score;
+      bestChunk = chunk;
+      continue;
+    }
+
+    if (score === bestScore && bestChunk && chunk.seq < bestChunk.seq) {
+      bestChunk = chunk;
+    }
+  }
+
+  return bestChunk ?? chunks[0] ?? null;
+}
+
+/**
+ * Build a rerank query that provides intent as context without becoming a search term.
+ */
+export function buildIntentAwareRerankQuery(
+  query: string,
+  intent?: string
+): string {
+  const trimmedIntent = intent?.trim();
+  if (!trimmedIntent) {
+    return query;
+  }
+
+  return `Intent: ${trimmedIntent}\nQuery: ${query}`;
+}

package/src/pipeline/rerank.ts

@@ -9,6 +9,10 @@ import type { RerankPort } from "../llm/types";
 import type { ChunkRow, StorePort } from "../store/types";
 import type { BlendingTier, FusionCandidate, RerankedCandidate } from "./types";
 
+import {
+  buildIntentAwareRerankQuery,
+  selectBestChunkForSteering,
+} from "./intent";
 import { DEFAULT_BLENDING_SCHEDULE } from "./types";
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -20,6 +24,8 @@ export interface RerankOptions {
   maxCandidates?: number;
   /** Blending schedule */
   blendingSchedule?: BlendingTier[];
+  /** Optional disambiguating context for reranking */
+  intent?: string;
 }
 
 export interface RerankResult {
@@ -75,27 +81,6 @@ function blend(
 const MAX_CHUNK_CHARS = 4000;
 const PROTECT_BM25_TOP_RANK = 1;
 
-interface BestChunkInfo {
-  candidate: FusionCandidate;
-  seq: number;
-}
-
-/**
- * Extract best chunk per document for efficient reranking.
- */
-function selectBestChunks(
-  toRerank: FusionCandidate[]
-): Map<string, BestChunkInfo> {
-  const bestChunkPerDoc = new Map<string, BestChunkInfo>();
-  for (const c of toRerank) {
-    const existing = bestChunkPerDoc.get(c.mirrorHash);
-    if (!existing || c.fusionScore > existing.candidate.fusionScore) {
-      bestChunkPerDoc.set(c.mirrorHash, { candidate: c, seq: c.seq });
-    }
-  }
-  return bestChunkPerDoc;
-}
-
 function isProtectedLexicalTopHit(candidate: FusionCandidate): boolean {
   return (
     candidate.bm25Rank === PROTECT_BM25_TOP_RANK &&
@@ -108,31 +93,50 @@ function isProtectedLexicalTopHit(candidate: FusionCandidate): boolean {
  */
 async function fetchChunkTexts(
   store: StorePort,
-  bestChunkPerDoc: Map<string, BestChunkInfo>
+  toRerank: FusionCandidate[],
+  query: string,
+  intent: string | undefined
 ): Promise<{ texts: string[]; hashToIndex: Map<string, number> }> {
-  const uniqueHashes = [...bestChunkPerDoc.keys()];
+  const uniqueHashes = [
+    ...new Set(toRerank.map((candidate) => candidate.mirrorHash)),
+  ];
   const chunksBatchResult = await store.getChunksBatch(uniqueHashes);
   const chunksByHash: Map<string, ChunkRow[]> = chunksBatchResult.ok
     ? chunksBatchResult.value
     : new Map();
-  const chunkTexts = new Map<string, string>();
+  const preferredSeqByHash = new Map<string, number>();
+
+  for (const candidate of toRerank) {
+    const existingSeq = preferredSeqByHash.get(candidate.mirrorHash);
+    if (existingSeq !== undefined) {
+      const existingCandidate = toRerank.find(
+        (entry) =>
+          entry.mirrorHash === candidate.mirrorHash && entry.seq === existingSeq
+      );
+      if (
+        existingCandidate &&
+        existingCandidate.fusionScore >= candidate.fusionScore
+      ) {
+        continue;
+      }
+    }
+    preferredSeqByHash.set(candidate.mirrorHash, candidate.seq);
+  }
 
+  const chunkTexts = new Map<string, string>();
   for (const hash of uniqueHashes) {
-    const bestInfo = bestChunkPerDoc.get(hash);
     const chunks = chunksByHash.get(hash);
-
-    if (chunks && bestInfo) {
-      const chunk = chunks.find((c) => c.seq === bestInfo.seq);
-      const text = chunk?.text ?? "";
-      chunkTexts.set(
-        hash,
-        text.length > MAX_CHUNK_CHARS
-          ? `${text.slice(0, MAX_CHUNK_CHARS)}...`
-          : text
-      );
-    } else {
-      chunkTexts.set(hash, "");
-    }
+    const bestChunk = selectBestChunkForSteering(chunks ?? [], query, intent, {
+      preferredSeq: preferredSeqByHash.get(hash) ?? null,
+      intentWeight: 0.5,
+    });
+    const text = bestChunk?.text ?? "";
+    chunkTexts.set(
+      hash,
+      text.length > MAX_CHUNK_CHARS
+        ? `${text.slice(0, MAX_CHUNK_CHARS)}...`
+        : text
+    );
   }
 
   const hashToIndex = new Map<string, number>();
@@ -198,11 +202,40 @@ export async function rerankCandidates(
   const remaining = candidates.slice(maxCandidates);
 
   // Extract best chunk per document for efficient reranking
-  const bestChunkPerDoc = selectBestChunks(toRerank);
-  const { texts, hashToIndex } = await fetchChunkTexts(store, bestChunkPerDoc);
+  const { texts, hashToIndex } = await fetchChunkTexts(
+    store,
+    toRerank,
+    query,
+    options.intent
+  );
+
+  const uniqueTexts: string[] = [];
+  const docIndexToUniqueIndex = new Map<number, number>();
+  const uniqueIndexToDocIndices = new Map<number, number[]>();
+  const textToUniqueIndex = new Map<string, number>();
+
+  for (const [docIndex, text] of texts.entries()) {
+    const existingIndex = textToUniqueIndex.get(text);
+    if (existingIndex !== undefined) {
+      docIndexToUniqueIndex.set(docIndex, existingIndex);
+      const mapped = uniqueIndexToDocIndices.get(existingIndex) ?? [];
+      mapped.push(docIndex);
+      uniqueIndexToDocIndices.set(existingIndex, mapped);
+      continue;
+    }
+
+    const uniqueIndex = uniqueTexts.length;
+    uniqueTexts.push(text);
+    textToUniqueIndex.set(text, uniqueIndex);
+    docIndexToUniqueIndex.set(docIndex, uniqueIndex);
+    uniqueIndexToDocIndices.set(uniqueIndex, [docIndex]);
+  }
 
   // Run reranking on best chunks (much faster than full docs)
-  const rerankResult = await rerankPort.rerank(query, texts);
+  const rerankResult = await rerankPort.rerank(
+    buildIntentAwareRerankQuery(query, options.intent),
+    uniqueTexts
+  );
 
   if (!rerankResult.ok) {
     return {
@@ -217,9 +250,13 @@ export async function rerankCandidates(
   }
 
   // Normalize rerank scores using min-max
-  const scoreByDocIndex = new Map(
-    rerankResult.value.map((s) => [s.index, s.score])
-  );
+  const scoreByDocIndex = new Map<number, number>();
+  for (const score of rerankResult.value) {
+    const docIndices = uniqueIndexToDocIndices.get(score.index) ?? [];
+    for (const docIndex of docIndices) {
+      scoreByDocIndex.set(docIndex, score.score);
+    }
+  }
   const rerankScores = rerankResult.value.map((s) => s.score);
   const minRerank = Math.min(...rerankScores);
   const maxRerank = Math.max(...rerankScores);

package/src/pipeline/search.ts

@@ -17,6 +17,8 @@ import type {
 
 import { err, ok } from "../store/types";
 import { createChunkLookup } from "./chunk-lookup";
+import { matchesExcludedChunks, matchesExcludedText } from "./exclude";
+import { selectBestChunkForSteering } from "./intent";
 import { detectQueryLanguage } from "./query-language";
 import {
   resolveRecencyTimestamp,
@@ -218,9 +220,38 @@ export async function searchBm25(
     seenUriSeq.add(uriSeqKey);
 
     // Get chunk via O(1) lookup
-    const chunk = fts.mirrorHash
+    const rawChunk = fts.mirrorHash
       ? (getChunk(fts.mirrorHash, fts.seq) ?? null)
       : null;
+    const chunk =
+      options.intent && fts.mirrorHash
+        ? (selectBestChunkForSteering(
+            chunksMapResult.ok
+              ? (chunksMapResult.value.get(fts.mirrorHash) ?? [])
+              : [],
+            query,
+            options.intent,
+            {
+              preferredSeq: rawChunk?.seq ?? fts.seq,
+              intentWeight: 0.3,
+            }
+          ) ?? rawChunk)
+        : rawChunk;
+
+    const excluded =
+      matchesExcludedText(
+        [fts.title ?? "", fts.relPath ?? "", fts.snippet ?? ""],
+        options.exclude
+      ) ||
+      matchesExcludedChunks(
+        chunksMapResult.ok && fts.mirrorHash
+          ? (chunksMapResult.value.get(fts.mirrorHash) ?? [])
+          : [],
+        options.exclude
+      );
+    if (excluded) {
+      continue;
+    }
 
     // For --full, de-dupe by docid (keep best scoring chunk per doc)
     // Raw BM25: smaller (more negative) is better
@@ -293,6 +324,8 @@ export async function searchBm25(
       query,
       mode: "bm25",
       totalResults: Math.min(filteredResults.length, limit),
+      intent: options.intent,
+      exclude: options.exclude,
       collection: options.collection,
       lang: options.lang,
       since: temporalRange.since,

package/src/pipeline/types.ts

@@ -62,6 +62,7 @@ export interface SearchMeta {
   reranked?: boolean;
   vectorsUsed?: boolean;
   totalResults: number;
+  intent?: string;
   collection?: string;
   lang?: string;
   /** Detected/overridden query language for prompt selection (typically BCP-47; may be user-provided via --lang) */
@@ -76,6 +77,10 @@ export interface SearchMeta {
   categories?: string[];
   /** Author filter applied */
   author?: string;
+  /** Rerank candidate limit used */
+  candidateLimit?: number;
+  /** Explicit exclusion terms applied */
+  exclude?: string[];
   /** Explain data (when --explain is used) */
   explain?: {
     lines: ExplainLine[];
@@ -119,6 +124,10 @@ export interface SearchOptions {
   categories?: string[];
   /** Filter by author value */
   author?: string;
+  /** Optional disambiguating context that steers scoring/snippets, but is not searched directly */
+  intent?: string;
+  /** Explicit exclusion terms for hard candidate pruning */
+  exclude?: string[];
 }
 
 /** Structured query mode identifier */
@@ -145,6 +154,8 @@ export type HybridSearchOptions = SearchOptions & {
   noRerank?: boolean;
   /** Optional structured mode entries; when set, used as expansion inputs */
   queryModes?: QueryModeInput[];
+  /** Max candidates passed to reranking */
+  candidateLimit?: number;
   /** Enable explain output */
   explain?: boolean;
   /** Language hint for prompt selection (does NOT filter retrieval, only affects expansion prompts) */
@@ -308,6 +319,10 @@ export interface AskMeta {
   expanded: boolean;
   reranked: boolean;
   vectorsUsed: boolean;
+  intent?: string;
+  candidateLimit?: number;
+  exclude?: string[];
+  queryModes?: QueryModeSummary;
   answerGenerated?: boolean;
   totalResults?: number;
   answerContext?: AnswerContextExplain;

package/src/pipeline/vsearch.ts

@@ -14,6 +14,8 @@ import type { SearchOptions, SearchResult, SearchResults } from "./types";
 import { err, ok } from "../store/types";
 import { createChunkLookup } from "./chunk-lookup";
 import { formatQueryForEmbedding } from "./contextual";
+import { matchesExcludedChunks, matchesExcludedText } from "./exclude";
+import { selectBestChunkForSteering } from "./intent";
 import { detectQueryLanguage } from "./query-language";
 import {
   resolveRecencyTimestamp,
@@ -146,7 +148,18 @@ export async function searchVectorWithEmbedding(
     }
 
     // Get chunk via O(1) lookup
-    const chunk = getChunk(vec.mirrorHash, vec.seq);
+    const rawChunk = getChunk(vec.mirrorHash, vec.seq);
+    const chunk = options.intent
+      ? (selectBestChunkForSteering(
+          chunksMap.get(vec.mirrorHash) ?? [],
+          query,
+          options.intent,
+          {
+            preferredSeq: rawChunk?.seq ?? vec.seq,
+            intentWeight: 0.3,
+          }
+        ) ?? rawChunk)
+      : rawChunk;
     if (!chunk) {
       continue;
     }
@@ -162,6 +175,25 @@ export async function searchVectorWithEmbedding(
       continue;
     }
 
+    const excluded =
+      matchesExcludedText(
+        [
+          doc.title ?? "",
+          doc.relPath,
+          doc.author ?? "",
+          doc.contentType ?? "",
+          ...(doc.categories ?? []),
+        ],
+        options.exclude
+      ) ||
+      matchesExcludedChunks(
+        chunksMap.get(vec.mirrorHash) ?? [],
+        options.exclude
+      );
+    if (excluded) {
+      continue;
+    }
+
     // For --full, de-dupe by docid (keep best scoring chunk per doc)
     if (options.full) {
       const existing = bestByDocid.get(doc.docid);
@@ -288,6 +320,8 @@ export async function searchVectorWithEmbedding(
       mode: "vector",
       vectorsUsed: true,
       totalResults: finalResults.length,
+      intent: options.intent,
+      exclude: options.exclude,
       collection: options.collection,
       lang: options.lang,
       since: temporalRange.since,
@@ -349,6 +383,9 @@ interface DocumentInfo {
   title: string | null;
   collection: string;
   relPath: string;
+  author: string | null;
+  contentType: string | null;
+  categories: string[] | null;
   sourceHash: string;
   sourceMime: string;
   sourceExt: string;
@@ -478,6 +515,9 @@ async function buildDocumentMap(
       title: doc.title,
       collection: doc.collection,
       relPath: doc.relPath,
+      author: doc.author ?? null,
+      contentType: doc.contentType ?? null,
+      categories: doc.categories ?? null,
       sourceHash: doc.sourceHash,
       sourceMime: doc.sourceMime,
       sourceExt: doc.sourceExt,

package/src/serve/public/lib/retrieval-filters.ts

@@ -9,6 +9,9 @@ export interface QueryModeEntry {
 
 export interface RetrievalFiltersState {
   collection: string;
+  intent: string;
+  candidateLimit: string;
+  exclude: string;
   since: string;
   until: string;
   category: string;
@@ -121,6 +124,10 @@ export function parseFiltersFromSearch(
 
   return {
     collection: params.get("collection") ?? defaults.collection ?? "",
+    intent: params.get("intent") ?? defaults.intent ?? "",
+    candidateLimit:
+      params.get("candidateLimit") ?? defaults.candidateLimit ?? "",
+    exclude: params.get("exclude") ?? defaults.exclude ?? "",
     since: params.get("since") ?? defaults.since ?? "",
     until: params.get("until") ?? defaults.until ?? "",
     category: params.get("category") ?? defaults.category ?? "",
@@ -145,6 +152,9 @@ export function applyFiltersToUrl(
   };
 
   setOrDelete("collection", filters.collection);
+  setOrDelete("intent", filters.intent);
+  setOrDelete("candidateLimit", filters.candidateLimit);
+  setOrDelete("exclude", filters.exclude);
   setOrDelete("since", filters.since);
   setOrDelete("until", filters.until);
   setOrDelete("category", filters.category);