@gmickel/gno 1.18.0 → 1.19.0

package/src/pipeline/hybrid.ts

@@ -55,9 +55,10 @@ import {
   shouldSortByRecency,
 } from "./temporal";
 import {
-  DEFAULT_PIPELINE_CONFIG,
-  SEARCH_RESULT_PLANNER_METADATA,
-} from "./types";
+  attachSearchResultPlannerMetadata,
+  attachSearchResultsTraceMetadata,
+} from "./trace-metadata";
+import { DEFAULT_PIPELINE_CONFIG } from "./types";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Dependencies
@@ -121,11 +122,13 @@ async function checkBm25Strength(
     until?: string;
     categories?: string[];
     author?: string;
+    relPathPrefix?: string;
   }
 ): Promise<boolean> {
   const result = await store.searchFts(query, {
     limit: 5,
     collection: options?.collection,
+    relPathPrefix: options?.relPathPrefix,
     language: options?.lang,
     tagsAll: options?.tagsAll,
     tagsAny: options?.tagsAny,
@@ -179,11 +182,13 @@ async function searchFtsChunks(
     until?: string;
     categories?: string[];
     author?: string;
+    relPathPrefix?: string;
   }
 ): Promise<FtsChunksResult> {
   const result = await store.searchFts(query, {
     limit: options.limit,
     collection: options.collection,
+    relPathPrefix: options.relPathPrefix,
     language: options.lang,
     tagsAll: options.tagsAll,
     tagsAny: options.tagsAny,
@@ -216,7 +221,11 @@ async function searchVectorChunks(
   vectorIndex: VectorIndexPort,
   embedPort: EmbeddingPort,
   query: string,
-  options: { limit: number; minScore?: number }
+  options: {
+    limit: number;
+    minScore?: number;
+    allowedMirrorHashes?: string[];
+  }
 ): Promise<ChunkId[]> {
   if (!vectorIndex.searchAvailable) {
     return [];
@@ -234,7 +243,10 @@ async function searchVectorChunks(
   const searchResult = await vectorIndex.searchNearest(
     queryEmbedding,
     options.limit,
-    { minScore: options.minScore }
+    {
+      minScore: options.minScore,
+      allowedMirrorHashes: options.allowedMirrorHashes,
+    }
   );
 
   if (!searchResult.ok) {
@@ -381,6 +393,7 @@ export async function searchHybrid(
           until: temporalRange.until,
           categories: options.categories,
           author: options.author,
+          relPathPrefix: options.retrievalScope?.relPathPrefix,
         });
 
     if (hasStrongSignal) {
@@ -397,6 +410,8 @@ export async function searchHybrid(
       });
       if (expandResult.ok) {
         expansion = expandResult.value;
+      } else {
+        counters.fallbackEvents.push("expansion_error");
       }
     }
   }
@@ -428,6 +443,7 @@ export async function searchHybrid(
     until: temporalRange.until,
     categories: options.categories,
     author: options.author,
+    relPathPrefix: options.retrievalScope?.relPathPrefix,
   });
 
   // Propagate FTS syntax errors as INVALID_INPUT
@@ -462,6 +478,7 @@ export async function searchHybrid(
           until: temporalRange.until,
           categories: options.categories,
           author: options.author,
+          relPathPrefix: options.retrievalScope?.relPathPrefix,
         })
       )
     );
@@ -509,6 +526,7 @@ export async function searchHybrid(
         query,
         {
           limit: limit * 2 * retrievalMultiplier,
+          allowedMirrorHashes: options.retrievalScope?.allowedMirrorHashes,
         }
       );
 
@@ -552,7 +570,10 @@ export async function searchHybrid(
 
           const searchResult = await vectorIndex.searchNearest(
             new Float32Array(embedding),
-            variant.limit
+            variant.limit,
+            {
+              allowedMirrorHashes: options.retrievalScope?.allowedMirrorHashes,
+            }
           );
           if (!searchResult.ok || searchResult.value.length === 0) {
             continue;
@@ -615,6 +636,7 @@ export async function searchHybrid(
     limit,
     candidateLimit,
     disabled: !options.graph || options.noGraph,
+    relPathPrefix: options.retrievalScope?.relPathPrefix,
     lang: options.lang,
     tagsAll: options.tagsAll,
     tagsAny: options.tagsAny,
@@ -735,6 +757,14 @@ export async function searchHybrid(
   const matchesMetadataFilters = (
     doc: (typeof docsResult.value)[number]
   ): boolean => {
+    const relPathPrefix = options.retrievalScope?.relPathPrefix;
+    if (
+      relPathPrefix !== undefined &&
+      doc.relPath !== relPathPrefix &&
+      !doc.relPath.startsWith(`${relPathPrefix}/`)
+    ) {
+      return false;
+    }
     if (!isWithinTemporalRange(doc.sourceMtime, temporalRange)) {
       return false;
     }
@@ -923,42 +953,51 @@ export async function searchHybrid(
       if (!docidMap.has(docidKey)) docidMap.set(docidKey, doc.docid);
       const collectionPath = collectionPaths.get(doc.collection);
       seenDocids.add(doc.docid);
-      results.push({
-        docid: doc.docid,
-        score: candidate.blendedScore,
-        uri: doc.uri,
-        title: doc.title ?? undefined,
-        contentType: doc.contentType ?? undefined,
-        categories: doc.categories ?? undefined,
-        line: snippetChunk.startLine,
-        snippet,
-        snippetLanguage: chunk.language ?? undefined,
-        snippetRange,
-        source: {
-          relPath: doc.relPath,
-          absPath: collectionPath
-            ? `${collectionPath}/${doc.relPath}`
-            : undefined,
-          mime: doc.sourceMime,
-          ext: doc.sourceExt,
-          modifiedAt: doc.sourceMtime,
-          documentDate: doc.frontmatterDate ?? undefined,
-          sizeBytes: doc.sourceSize,
-          sourceHash: doc.sourceHash,
-        },
-        conversion: {
-          mirrorHash: candidate.mirrorHash,
-          converterId: doc.converterId ?? undefined,
-          converterVersion: doc.converterVersion ?? undefined,
-        },
-        [SEARCH_RESULT_PLANNER_METADATA]: {
-          retrievalRank: candidateIndex + 1,
-          mirrorHash: candidate.mirrorHash,
-          seq: snippetChunk.seq,
-          sources: [...candidate.sources].sort(),
-          graphExpanded: candidate.sources.includes("graph"),
-        },
-      });
+      results.push(
+        attachSearchResultPlannerMetadata(
+          {
+            docid: doc.docid,
+            score: candidate.blendedScore,
+            uri: doc.uri,
+            title: doc.title ?? undefined,
+            contentType: doc.contentType ?? undefined,
+            categories: doc.categories ?? undefined,
+            line: snippetChunk.startLine,
+            snippet,
+            snippetLanguage: chunk.language ?? undefined,
+            snippetRange,
+            source: {
+              relPath: doc.relPath,
+              absPath: collectionPath
+                ? `${collectionPath}/${doc.relPath}`
+                : undefined,
+              mime: doc.sourceMime,
+              ext: doc.sourceExt,
+              modifiedAt: doc.sourceMtime,
+              documentDate: doc.frontmatterDate ?? undefined,
+              sizeBytes: doc.sourceSize,
+              sourceHash: doc.sourceHash,
+            },
+            conversion: {
+              mirrorHash: candidate.mirrorHash,
+              converterId: doc.converterId ?? undefined,
+              converterVersion: doc.converterVersion ?? undefined,
+            },
+          },
+          {
+            retrievalRank: candidateIndex + 1,
+            mirrorHash: candidate.mirrorHash,
+            seq: snippetChunk.seq,
+            sources: [...candidate.sources].sort(),
+            graphExpanded: candidate.sources.includes("graph"),
+            startLine: snippetChunk.startLine,
+            endLine: snippetChunk.endLine,
+            passageHash: new Bun.CryptoHasher("sha256")
+              .update(snippetChunk.text)
+              .digest("hex"),
+          }
+        )
+      );
     }
   }
   timings.assemblyMs = performance.now() - assemblyStartedAt;
@@ -1002,7 +1041,7 @@ export async function searchHybrid(
   const finalResults = results.slice(0, limit);
   await attachSearchResultContexts(store, finalResults);
 
-  return ok({
+  const output: SearchResults = {
     results: finalResults,
     meta: {
       query,
@@ -1033,5 +1072,74 @@ export async function searchHybrid(
       explain: explainData,
       trace: diagnoseTrace,
     },
+  };
+  const fallbackCodes = [...new Set(counters.fallbackEvents)].sort();
+  const capabilityOutcomes = [
+    { capability: "lexical_search", status: "used" as const },
+    vectorAvailable
+      ? fallbackCodes.includes("vector_embed_error")
+        ? {
+            capability: "semantic_search",
+            status: "failed" as const,
+            reasonCode: "vector_embed_error",
+          }
+        : { capability: "semantic_search", status: "used" as const }
+      : {
+          capability: "semantic_search",
+          status: "unavailable" as const,
+          reasonCode: "vector_unavailable",
+        },
+    expansion !== null
+      ? { capability: "query_expansion", status: "used" as const }
+      : {
+          capability: "query_expansion",
+          status:
+            expansionStatus === "attempted"
+              ? ("failed" as const)
+              : ("unavailable" as const),
+          reasonCode:
+            expansionStatus === "attempted"
+              ? "expansion_error"
+              : expansionStatus === "skipped_strong"
+                ? "expansion_skipped_strong"
+                : "expansion_disabled",
+        },
+    rerankResult.reranked
+      ? { capability: "reranking", status: "used" as const }
+      : {
+          capability: "reranking",
+          status:
+            rerankResult.fallbackReason === "error"
+              ? ("failed" as const)
+              : ("unavailable" as const),
+          reasonCode:
+            rerankResult.fallbackReason === "error"
+              ? "rerank_error"
+              : "rerank_disabled",
+        },
+    graphExpansion.meta.enabled
+      ? { capability: "graph_expansion", status: "used" as const }
+      : {
+          capability: "graph_expansion",
+          status: "unavailable" as const,
+          reasonCode:
+            graphExpansion.meta.fallbackReasons[0] ?? "graph_disabled",
+        },
+  ];
+  attachSearchResultsTraceMetadata(output, {
+    capabilityOutcomes,
+    fallbackCodes,
   });
+  const traceResult = await options.traceSession?.recordRetrieval(
+    output,
+    timings.totalMs
+  );
+  if (traceResult && !traceResult.ok) {
+    return err(
+      "QUERY_FAILED",
+      `Trace recording failed: ${traceResult.error.message}`,
+      traceResult.error.cause
+    );
+  }
+  return ok(output);
 }

package/src/pipeline/search.ts

@@ -27,6 +27,8 @@ import {
   resolveTemporalRange,
   shouldSortByRecency,
 } from "./temporal";
+import { attachSearchResultPlannerMetadata } from "./trace-metadata";
+import { SEARCH_RESULT_PLANNER_METADATA } from "./types";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Score Normalization
@@ -113,7 +115,7 @@ function buildSearchResult(ctx: BuildResultContext): SearchResult {
       : undefined;
   }
 
-  return {
+  const result: SearchResult = {
     docid: fts.docid ?? "",
     score: fts.score, // Raw score, normalized later as batch
     uri: fts.uri ?? "",
@@ -127,6 +129,19 @@ function buildSearchResult(ctx: BuildResultContext): SearchResult {
     source,
     conversion: fts.mirrorHash ? { mirrorHash: fts.mirrorHash } : undefined,
   };
+  if (!(chunk && fts.mirrorHash)) return result;
+  return attachSearchResultPlannerMetadata(result, {
+    retrievalRank: 0,
+    mirrorHash: fts.mirrorHash,
+    seq: chunk.seq,
+    sources: ["bm25"],
+    graphExpanded: false,
+    startLine: chunk.startLine,
+    endLine: chunk.endLine,
+    passageHash: new Bun.CryptoHasher("sha256")
+      .update(chunk.text)
+      .digest("hex"),
+  });
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -142,6 +157,7 @@ export async function searchBm25(
   query: string,
   options: SearchOptions = {}
 ): Promise<ReturnType<typeof ok<SearchResults>>> {
+  const traceStartedAt = options.traceSession ? performance.now() : 0;
   const limit = options.limit ?? 20;
   const minScore = options.minScore ?? 0;
   const recencySort = shouldSortByRecency(query);
@@ -161,6 +177,7 @@ export async function searchBm25(
   const ftsResult = await store.searchFts(query, {
     limit: retrievalLimit,
     collection: options.collection,
+    relPathPrefix: options.retrievalScope?.relPathPrefix,
     language: options.lang,
     snippet: !(options.full || options.lineNumbers),
     tagsAll: options.tagsAll,
@@ -331,9 +348,13 @@ export async function searchBm25(
   }
 
   const finalResults = filteredResults.slice(0, limit);
+  for (const [index, result] of finalResults.entries()) {
+    const metadata = result[SEARCH_RESULT_PLANNER_METADATA];
+    if (metadata) metadata.retrievalRank = index + 1;
+  }
   await attachSearchResultContexts(store, finalResults);
 
-  return ok({
+  const output: SearchResults = {
     results: finalResults,
     meta: {
       query,
@@ -349,5 +370,17 @@ export async function searchBm25(
       author: options.author,
       queryLanguage,
     },
-  });
+  };
+  const traceResult = await options.traceSession?.recordRetrieval(
+    output,
+    performance.now() - traceStartedAt
+  );
+  if (traceResult && !traceResult.ok) {
+    return err(
+      "QUERY_FAILED",
+      `Trace recording failed: ${traceResult.error.message}`,
+      traceResult.error.cause
+    );
+  }
+  return ok(output);
 }

package/src/pipeline/trace-metadata.ts

@@ -0,0 +1,47 @@
+/** Non-canonical retrieval metadata attached without changing public payloads. */
+
+import type {
+  Citation,
+  CitationTraceMetadata,
+  SearchResult,
+  SearchResultPlannerMetadata,
+  SearchResults,
+  SearchResultsTraceMetadata,
+} from "./types";
+
+import {
+  CITATION_TRACE_METADATA,
+  SEARCH_RESULT_PLANNER_METADATA,
+  SEARCH_RESULTS_TRACE_METADATA,
+} from "./types";
+
+const attachHiddenMetadata = <Value extends object, Metadata>(
+  value: Value,
+  key: symbol,
+  metadata: Metadata
+): Value => {
+  Object.defineProperty(value, key, {
+    configurable: false,
+    enumerable: false,
+    value: metadata,
+    writable: false,
+  });
+  return value;
+};
+
+export const attachSearchResultPlannerMetadata = <Result extends SearchResult>(
+  result: Result,
+  metadata: SearchResultPlannerMetadata
+): Result =>
+  attachHiddenMetadata(result, SEARCH_RESULT_PLANNER_METADATA, metadata);
+
+export const attachCitationTraceMetadata = <Result extends Citation>(
+  citation: Result,
+  metadata: CitationTraceMetadata
+): Result => attachHiddenMetadata(citation, CITATION_TRACE_METADATA, metadata);
+
+export const attachSearchResultsTraceMetadata = <Result extends SearchResults>(
+  results: Result,
+  metadata: SearchResultsTraceMetadata
+): Result =>
+  attachHiddenMetadata(results, SEARCH_RESULTS_TRACE_METADATA, metadata);

package/src/pipeline/types.ts

@@ -5,6 +5,7 @@
  * @module src/pipeline/types
  */
 
+import type { RetrievalTraceSession } from "../core/retrieval-trace-session";
 import type { StoreResult } from "../store/types";
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -48,6 +49,11 @@ export interface SearchResultPlannerMetadata {
   seq: number;
   sources: FusionSource[];
   graphExpanded: boolean;
+  /** Exact canonical chunk coordinates, retained even for full-content output. */
+  startLine?: number;
+  endLine?: number;
+  /** SHA-256 of the complete canonical chunk lines. */
+  passageHash?: string;
 }
 
 /** Single search result matching output schema */
@@ -122,10 +128,26 @@ export interface SearchMeta {
   trace?: QueryDiagnoseTrace;
 }
 
+export const SEARCH_RESULTS_TRACE_METADATA = Symbol(
+  "gno.searchResultsTraceMetadata"
+);
+
+export interface SearchCapabilityOutcome {
+  capability: string;
+  status: "attempted" | "used" | "unavailable" | "failed";
+  reasonCode?: string;
+}
+
+export interface SearchResultsTraceMetadata {
+  capabilityOutcomes: SearchCapabilityOutcome[];
+  fallbackCodes: string[];
+}
+
 /** Complete search results wrapper */
 export interface SearchResults {
   results: SearchResult[];
   meta: SearchMeta;
+  [SEARCH_RESULTS_TRACE_METADATA]?: SearchResultsTraceMetadata;
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -134,12 +156,19 @@ export interface SearchResults {
 
 /** Common options for all search commands */
 export interface SearchOptions {
+  /** Internal receipt seam; never serialized or included in public schemas. */
+  traceSession?: RetrievalTraceSession;
   /** Max results */
   limit?: number;
   /** Min score threshold (0-1) */
   minScore?: number;
   /** Filter by collection */
   collection?: string;
+  /** Internal exact corpus scope used by deterministic retrieval replay. */
+  retrievalScope?: {
+    relPathPrefix?: string;
+    allowedMirrorHashes: string[];
+  };
   /** Language filter/hint (BCP-47) */
   lang?: string;
   /** Include full content instead of snippet */
@@ -357,11 +386,25 @@ export const DEFAULT_PIPELINE_CONFIG: PipelineConfig = {
 // ─────────────────────────────────────────────────────────────────────────────
 
 /** Citation reference */
+export const CITATION_TRACE_METADATA = Symbol("gno.citationTraceMetadata");
+
+export interface CitationTraceMetadata {
+  sourceHash: string;
+  mirrorHash: string;
+  passageHash: string;
+  seq?: number;
+  rank: number;
+  plannerRank?: number;
+  sources?: FusionSource[];
+  graphExpanded?: boolean;
+}
+
 export interface Citation {
   docid: string;
   uri: string;
   startLine?: number;
   endLine?: number;
+  [CITATION_TRACE_METADATA]?: CitationTraceMetadata;
 }
 
 /** Source selection entry for answer-generation explain */