@gmickel/gno 0.15.1 → 0.17.0

package/src/pipeline/rerank.ts

@@ -6,7 +6,7 @@
  */
 
 import type { RerankPort } from "../llm/types";
-import type { StorePort } from "../store/types";
+import type { ChunkRow, StorePort } from "../store/types";
 import type { BlendingTier, FusionCandidate, RerankedCandidate } from "./types";
 
 import { DEFAULT_BLENDING_SCHEDULE } from "./types";
@@ -25,6 +25,7 @@ export interface RerankOptions {
 export interface RerankResult {
   candidates: RerankedCandidate[];
   reranked: boolean;
+  fallbackReason: "none" | "disabled" | "error";
 }
 
 export interface RerankDeps {
@@ -72,6 +73,7 @@ function blend(
 // ─────────────────────────────────────────────────────────────────────────────
 
 const MAX_CHUNK_CHARS = 4000;
+const PROTECT_BM25_TOP_RANK = 1;
 
 interface BestChunkInfo {
   candidate: FusionCandidate;
@@ -94,6 +96,13 @@ function selectBestChunks(
   return bestChunkPerDoc;
 }
 
+function isProtectedLexicalTopHit(candidate: FusionCandidate): boolean {
+  return (
+    candidate.bm25Rank === PROTECT_BM25_TOP_RANK &&
+    candidate.sources.includes("bm25")
+  );
+}
+
 /**
  * Fetch chunk texts for reranking.
  */
@@ -102,18 +111,18 @@ async function fetchChunkTexts(
   bestChunkPerDoc: Map<string, BestChunkInfo>
 ): Promise<{ texts: string[]; hashToIndex: Map<string, number> }> {
   const uniqueHashes = [...bestChunkPerDoc.keys()];
-  const chunkResults = await Promise.all(
-    uniqueHashes.map((hash) => store.getChunks(hash))
-  );
-
+  const chunksBatchResult = await store.getChunksBatch(uniqueHashes);
+  const chunksByHash: Map<string, ChunkRow[]> = chunksBatchResult.ok
+    ? chunksBatchResult.value
+    : new Map();
   const chunkTexts = new Map<string, string>();
-  for (let i = 0; i < uniqueHashes.length; i++) {
-    const hash = uniqueHashes[i] as string;
-    const result = chunkResults[i];
+
+  for (const hash of uniqueHashes) {
     const bestInfo = bestChunkPerDoc.get(hash);
+    const chunks = chunksByHash.get(hash);
 
-    if (result?.ok && result.value && bestInfo) {
-      const chunk = result.value.find((c) => c.seq === bestInfo.seq);
+    if (chunks && bestInfo) {
+      const chunk = chunks.find((c) => c.seq === bestInfo.seq);
       const text = chunk?.text ?? "";
       chunkTexts.set(
         hash,
@@ -151,7 +160,7 @@ export async function rerankCandidates(
   options: RerankOptions = {}
 ): Promise<RerankResult> {
   if (candidates.length === 0) {
-    return { candidates: [], reranked: false };
+    return { candidates: [], reranked: false, fallbackReason: "none" };
   }
 
   const { rerankPort, store } = deps;
@@ -181,6 +190,7 @@ export async function rerankCandidates(
         blendedScore: normalizeFusionScore(c.fusionScore),
       })),
       reranked: false,
+      fallbackReason: "disabled",
     };
   }
 
@@ -202,6 +212,7 @@ export async function rerankCandidates(
         blendedScore: normalizeFusionScore(c.fusionScore),
       })),
       reranked: false,
+      fallbackReason: "error",
     };
   }
 
@@ -239,7 +250,7 @@ export async function rerankCandidates(
   });
 
   // Add remaining candidates with penalty
-  const allCandidates: RerankedCandidate[] = [
+  let allCandidates: RerankedCandidate[] = [
     ...rerankedCandidates,
     ...remaining.map((c) => ({
       ...c,
@@ -260,5 +271,15 @@ export async function rerankCandidates(
     return `${a.mirrorHash}:${a.seq}`.localeCompare(`${b.mirrorHash}:${b.seq}`);
   });
 
-  return { candidates: allCandidates, reranked: true };
+  // Guardrail: keep strong original lexical #1 at the top.
+  // This avoids rerank-only demotions on clear exact-hit queries.
+  const protectedTopHit = allCandidates.find(isProtectedLexicalTopHit);
+  if (protectedTopHit && allCandidates[0] !== protectedTopHit) {
+    allCandidates = [
+      protectedTopHit,
+      ...allCandidates.filter((candidate) => candidate !== protectedTopHit),
+    ];
+  }
+
+  return { candidates: allCandidates, reranked: true, fallbackReason: "none" };
 }

package/src/pipeline/search.ts

@@ -18,6 +18,11 @@ import type {
 import { err, ok } from "../store/types";
 import { createChunkLookup } from "./chunk-lookup";
 import { detectQueryLanguage } from "./query-language";
+import {
+  resolveRecencyTimestamp,
+  resolveTemporalRange,
+  shouldSortByRecency,
+} from "./temporal";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Score Normalization
@@ -74,6 +79,7 @@ function buildSearchResult(ctx: BuildResultContext): SearchResult {
     mime: fts.sourceMime ?? "text/markdown",
     ext: fts.sourceExt ?? ".md",
     modifiedAt: fts.sourceMtime,
+    documentDate: fts.frontmatterDate,
     sizeBytes: fts.sourceSize,
     sourceHash: fts.sourceHash,
   };
@@ -131,6 +137,13 @@ export async function searchBm25(
 ): Promise<ReturnType<typeof ok<SearchResults>>> {
   const limit = options.limit ?? 20;
   const minScore = options.minScore ?? 0;
+  const recencySort = shouldSortByRecency(query);
+  const retrievalLimit = recencySort ? limit * 3 : limit;
+  const temporalRange = resolveTemporalRange(
+    query,
+    options.since,
+    options.until
+  );
 
   // Detect query language for metadata (DOES NOT affect retrieval filtering)
   const detection = detectQueryLanguage(query);
@@ -139,12 +152,16 @@ export async function searchBm25(
   // Run FTS search
   // Disable FTS snippet when --full or --line-numbers (we use raw text instead)
   const ftsResult = await store.searchFts(query, {
-    limit,
+    limit: retrievalLimit,
     collection: options.collection,
     language: options.lang,
     snippet: !(options.full || options.lineNumbers),
     tagsAll: options.tagsAll,
     tagsAny: options.tagsAny,
+    since: temporalRange.since,
+    until: temporalRange.until,
+    categories: options.categories,
+    author: options.author,
   });
 
   if (!ftsResult.ok) {
@@ -253,14 +270,35 @@ export async function searchBm25(
   const filteredResults =
     minScore > 0 ? results.filter((r) => r.score >= minScore) : results;
 
+  if (recencySort) {
+    filteredResults.sort((a, b) => {
+      const aTs = resolveRecencyTimestamp(
+        a.source.documentDate,
+        a.source.modifiedAt
+      );
+      const bTs = resolveRecencyTimestamp(
+        b.source.documentDate,
+        b.source.modifiedAt
+      );
+      if (aTs !== bTs) {
+        return bTs - aTs;
+      }
+      return b.score - a.score;
+    });
+  }
+
   return ok({
-    results: filteredResults,
+    results: filteredResults.slice(0, limit),
     meta: {
       query,
       mode: "bm25",
-      totalResults: filteredResults.length,
+      totalResults: Math.min(filteredResults.length, limit),
       collection: options.collection,
       lang: options.lang,
+      since: temporalRange.since,
+      until: temporalRange.until,
+      categories: options.categories,
+      author: options.author,
       queryLanguage,
     },
   });

package/src/pipeline/temporal.ts

@@ -0,0 +1,257 @@
+/**
+ * Temporal range parsing helpers for retrieval filters.
+ *
+ * @module src/pipeline/temporal
+ */
+
+const DATE_ONLY_RE = /^\d{4}-\d{2}-\d{2}$/;
+const RECENCY_SORT_RE =
+  /\b(latest|newest|most recent|recent|today|yesterday|this week|last week|this month|last month)\b/;
+
+export interface TemporalRange {
+  since?: string;
+  until?: string;
+}
+
+type BoundKind = "since" | "until";
+
+function startOfDay(d: Date): Date {
+  const out = new Date(d);
+  out.setUTCHours(0, 0, 0, 0);
+  return out;
+}
+
+function endOfDay(d: Date): Date {
+  const out = new Date(d);
+  out.setUTCHours(23, 59, 59, 999);
+  return out;
+}
+
+function startOfWeekUtc(d: Date): Date {
+  const out = startOfDay(d);
+  const day = out.getUTCDay(); // 0 = Sunday
+  const mondayOffset = day === 0 ? -6 : 1 - day;
+  out.setUTCDate(out.getUTCDate() + mondayOffset);
+  return out;
+}
+
+function endOfWeekUtc(d: Date): Date {
+  const start = startOfWeekUtc(d);
+  const out = new Date(start);
+  out.setUTCDate(out.getUTCDate() + 6);
+  return endOfDay(out);
+}
+
+function startOfMonthUtc(d: Date): Date {
+  return new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), 1, 0, 0, 0, 0));
+}
+
+function endOfMonthUtc(d: Date): Date {
+  return new Date(
+    Date.UTC(d.getUTCFullYear(), d.getUTCMonth() + 1, 0, 23, 59, 59, 999)
+  );
+}
+
+function normalizeParsedDate(value: string, kind: BoundKind): string | null {
+  const parsed = new Date(value);
+  if (Number.isNaN(parsed.getTime())) {
+    return null;
+  }
+
+  if (DATE_ONLY_RE.test(value)) {
+    return (
+      kind === "since" ? startOfDay(parsed) : endOfDay(parsed)
+    ).toISOString();
+  }
+
+  return parsed.toISOString();
+}
+
+function parseRelative(
+  value: string,
+  kind: BoundKind,
+  now: Date
+): string | null {
+  const v = value.trim().toLowerCase();
+
+  if (v === "today") {
+    return (kind === "since" ? startOfDay(now) : endOfDay(now)).toISOString();
+  }
+  if (v === "yesterday") {
+    const d = new Date(now);
+    d.setUTCDate(d.getUTCDate() - 1);
+    return (kind === "since" ? startOfDay(d) : endOfDay(d)).toISOString();
+  }
+  if (v === "this week") {
+    return (kind === "since" ? startOfWeekUtc(now) : now).toISOString();
+  }
+  if (v === "last week") {
+    const d = new Date(now);
+    d.setUTCDate(d.getUTCDate() - 7);
+    return (
+      kind === "since" ? startOfWeekUtc(d) : endOfWeekUtc(d)
+    ).toISOString();
+  }
+  if (v === "this month") {
+    return (kind === "since" ? startOfMonthUtc(now) : now).toISOString();
+  }
+  if (v === "last month") {
+    const d = new Date(now);
+    d.setUTCMonth(d.getUTCMonth() - 1);
+    return (
+      kind === "since" ? startOfMonthUtc(d) : endOfMonthUtc(d)
+    ).toISOString();
+  }
+  if (v === "recent") {
+    if (kind === "until") {
+      return now.toISOString();
+    }
+    const d = new Date(now);
+    d.setUTCDate(d.getUTCDate() - 30);
+    return d.toISOString();
+  }
+
+  return null;
+}
+
+function parseBound(
+  input: string | undefined,
+  kind: BoundKind,
+  now: Date
+): string | undefined {
+  if (!input) {
+    return undefined;
+  }
+  const relative = parseRelative(input, kind, now);
+  if (relative) {
+    return relative;
+  }
+  return normalizeParsedDate(input, kind) ?? undefined;
+}
+
+function inferFromQuery(query: string, now: Date): TemporalRange {
+  const q = query.toLowerCase();
+
+  if (/\btoday\b/.test(q)) {
+    return {
+      since: parseRelative("today", "since", now) ?? undefined,
+      until: parseRelative("today", "until", now) ?? undefined,
+    };
+  }
+  if (/\byesterday\b/.test(q)) {
+    return {
+      since: parseRelative("yesterday", "since", now) ?? undefined,
+      until: parseRelative("yesterday", "until", now) ?? undefined,
+    };
+  }
+  if (/\bthis week\b/.test(q)) {
+    return {
+      since: parseRelative("this week", "since", now) ?? undefined,
+      until: parseRelative("this week", "until", now) ?? undefined,
+    };
+  }
+  if (/\blast week\b/.test(q)) {
+    return {
+      since: parseRelative("last week", "since", now) ?? undefined,
+      until: parseRelative("last week", "until", now) ?? undefined,
+    };
+  }
+  if (/\bthis month\b/.test(q)) {
+    return {
+      since: parseRelative("this month", "since", now) ?? undefined,
+      until: parseRelative("this month", "until", now) ?? undefined,
+    };
+  }
+  if (/\blast month\b/.test(q)) {
+    return {
+      since: parseRelative("last month", "since", now) ?? undefined,
+      until: parseRelative("last month", "until", now) ?? undefined,
+    };
+  }
+  if (/\brecent\b/.test(q)) {
+    return {
+      since: parseRelative("recent", "since", now) ?? undefined,
+      until: parseRelative("recent", "until", now) ?? undefined,
+    };
+  }
+
+  return {};
+}
+
+/**
+ * Resolve temporal bounds from explicit flags or query text.
+ */
+export function resolveTemporalRange(
+  query: string,
+  sinceInput?: string,
+  untilInput?: string,
+  now = new Date()
+): TemporalRange {
+  const since = parseBound(sinceInput, "since", now);
+  const until = parseBound(untilInput, "until", now);
+
+  if (since || until) {
+    return { since, until };
+  }
+
+  return inferFromQuery(query, now);
+}
+
+/**
+ * Return true when timestamp falls inside optional range.
+ */
+export function isWithinTemporalRange(
+  timestamp: string | undefined,
+  range: TemporalRange
+): boolean {
+  if (!timestamp) {
+    return true;
+  }
+  const t = new Date(timestamp).getTime();
+  if (Number.isNaN(t)) {
+    return true;
+  }
+  if (range.since) {
+    const since = new Date(range.since).getTime();
+    if (!Number.isNaN(since) && t < since) {
+      return false;
+    }
+  }
+  if (range.until) {
+    const until = new Date(range.until).getTime();
+    if (!Number.isNaN(until) && t > until) {
+      return false;
+    }
+  }
+  return true;
+}
+
+/**
+ * Return true when query intent implies newest-first ordering.
+ */
+export function shouldSortByRecency(query: string): boolean {
+  return RECENCY_SORT_RE.test(query.toLowerCase());
+}
+
+/**
+ * Prefer canonical doc date; fallback to source modified time.
+ * Returns 0 when neither value is valid.
+ */
+export function resolveRecencyTimestamp(
+  docDate?: string | null,
+  sourceModifiedAt?: string | null
+): number {
+  if (docDate) {
+    const parsed = new Date(docDate).getTime();
+    if (!Number.isNaN(parsed)) {
+      return parsed;
+    }
+  }
+  if (sourceModifiedAt) {
+    const parsed = new Date(sourceModifiedAt).getTime();
+    if (!Number.isNaN(parsed)) {
+      return parsed;
+    }
+  }
+  return 0;
+}

package/src/pipeline/types.ts

@@ -18,6 +18,7 @@ export interface SearchResultSource {
   mime: string;
   ext: string;
   modifiedAt?: string;
+  documentDate?: string;
   sizeBytes?: number;
   sourceHash?: string;
 }
@@ -65,6 +66,16 @@ export interface SearchMeta {
   lang?: string;
   /** Detected/overridden query language for prompt selection (typically BCP-47; may be user-provided via --lang) */
   queryLanguage?: string;
+  /** Summary of structured query modes applied (if provided) */
+  queryModes?: QueryModeSummary;
+  /** Temporal filter lower bound (ISO 8601) */
+  since?: string;
+  /** Temporal filter upper bound (ISO 8601) */
+  until?: string;
+  /** Category filters applied */
+  categories?: string[];
+  /** Author filter applied */
+  author?: string;
   /** Explain data (when --explain is used) */
   explain?: {
     lines: ExplainLine[];
@@ -100,6 +111,30 @@ export interface SearchOptions {
   tagsAll?: string[];
   /** Filter to docs with ANY of these tags (OR) */
   tagsAny?: string[];
+  /** Filter by modified time lower bound (ISO 8601 or relative token) */
+  since?: string;
+  /** Filter by modified time upper bound (ISO 8601 or relative token) */
+  until?: string;
+  /** Filter to docs matching ANY category */
+  categories?: string[];
+  /** Filter by author value */
+  author?: string;
+}
+
+/** Structured query mode identifier */
+export type QueryMode = "term" | "intent" | "hyde";
+
+/** Structured query mode entry */
+export interface QueryModeInput {
+  mode: QueryMode;
+  text: string;
+}
+
+/** Structured query mode summary for metadata/explain */
+export interface QueryModeSummary {
+  term: number;
+  intent: number;
+  hyde: boolean;
 }
 
 /** Options for hybrid search (gno query) */
@@ -108,6 +143,8 @@ export type HybridSearchOptions = SearchOptions & {
   noExpand?: boolean;
   /** Disable reranking */
   noRerank?: boolean;
+  /** Optional structured mode entries; when set, used as expansion inputs */
+  queryModes?: QueryModeInput[];
   /** Enable explain output */
   explain?: boolean;
   /** Language hint for prompt selection (does NOT filter retrieval, only affects expansion prompts) */
@@ -247,6 +284,25 @@ export interface Citation {
   endLine?: number;
 }
 
+/** Source selection entry for answer-generation explain */
+export interface AnswerContextEntry {
+  docid: string;
+  uri: string;
+  score: number;
+  queryTokenHits: number;
+  facetHits: number;
+  reason: string;
+}
+
+/** Answer-generation context selection explain payload */
+export interface AnswerContextExplain {
+  strategy: "adaptive_coverage_v1";
+  targetSources: number;
+  facets: string[];
+  selected: AnswerContextEntry[];
+  dropped: AnswerContextEntry[];
+}
+
 /** Ask result metadata */
 export interface AskMeta {
   expanded: boolean;
@@ -254,6 +310,7 @@ export interface AskMeta {
   vectorsUsed: boolean;
   answerGenerated?: boolean;
   totalResults?: number;
+  answerContext?: AnswerContextExplain;
 }
 
 /** Ask command result */
@@ -323,6 +380,7 @@ export interface ExplainResult {
   rank: number;
   docid: string;
   score: number;
+  fusionScore?: number;
   bm25Score?: number;
   vecScore?: number;
   rerankScore?: number;