@gmickel/gno 1.18.0 → 1.19.0

package/src/mcp/tools/vsearch.ts

@@ -6,11 +6,21 @@
 
 import { join as pathJoin } from "node:path";
 
+import type { EmbeddingPort } from "../../llm/types";
 import type { SearchResult, SearchResults } from "../../pipeline/types";
 import type { ToolContext } from "../server";
 
 import { decorateUriForIndex, parseUri } from "../../app/constants";
 import { createNonTtyProgressRenderer } from "../../cli/progress";
+import {
+  finishRetrievalTraceAfterError,
+  retrievalTraceFilters,
+  startRetrievalTraceRequest,
+} from "../../core/retrieval-trace-request";
+import {
+  attachRetrievalTraceMetadata,
+  type RetrievalTraceSession,
+} from "../../core/retrieval-trace-session";
 import { LlmAdapter } from "../../llm/nodeLlamaCpp/adapter";
 import { resolveDownloadPolicy } from "../../llm/policy";
 import { resolveModelUri } from "../../llm/registry";
@@ -125,29 +135,55 @@ export function handleVsearch(
         undefined,
         args.collection
       );
+      const options = {
+        limit: args.limit ?? 5,
+        minScore: args.minScore,
+        collection: args.collection,
+        intent: args.intent,
+        exclude: args.exclude,
+        since: args.since,
+        until: args.until,
+        categories: args.categories,
+        author: args.author,
+        tagsAll: normalizeTagFilters(args.tagsAll),
+        tagsAny: normalizeTagFilters(args.tagsAny),
+      };
+      let traceSession: RetrievalTraceSession | undefined;
+      const traceStart = await startRetrievalTraceRequest({
+        store: ctx.store,
+        config: ctx.config,
+        query: args.query,
+        filters: retrievalTraceFilters(options),
+        pipeline: "vector",
+        indexName: ctx.indexName,
+        modelUris: [modelUri],
+      });
+      if (!traceStart.ok) throw new Error(traceStart.error.message);
+      traceSession = traceStart.value ?? undefined;
 
-      // Resolve download policy from env (MCP has no CLI flags)
-      const policy = resolveDownloadPolicy(process.env, {});
+      let embedPort: EmbeddingPort | null = null;
 
-      // Non-TTY progress for MCP (periodic lines to stderr, not \r)
-      const downloadProgress = createNonTtyProgressRenderer();
+      try {
+        // Resolve download policy from env (MCP has no CLI flags)
+        const policy = resolveDownloadPolicy(process.env, {});
 
-      // Create LLM adapter for embeddings
-      const llm = new LlmAdapter(ctx.config);
-      const embedResult = await llm.createEmbeddingPort(modelUri, {
-        policy,
-        onProgress: (progress) => downloadProgress("embed", progress),
-      });
-      if (!embedResult.ok) {
-        throw new Error(
-          `Failed to load embedding model: ${embedResult.error.message}. ` +
-            "Ensure models are downloaded with: gno models pull"
-        );
-      }
+        // Non-TTY progress for MCP (periodic lines to stderr, not \r)
+        const downloadProgress = createNonTtyProgressRenderer();
 
-      const embedPort = embedResult.value;
+        // Create LLM adapter for embeddings
+        const llm = new LlmAdapter(ctx.config);
+        const embedResult = await llm.createEmbeddingPort(modelUri, {
+          policy,
+          onProgress: (progress) => downloadProgress("embed", progress),
+        });
+        if (!embedResult.ok) {
+          throw new Error(
+            `Failed to load embedding model: ${embedResult.error.message}. ` +
+              "Ensure models are downloaded with: gno models pull"
+          );
+        }
+        embedPort = embedResult.value;
 
-      try {
         // Embed query with contextual formatting
         const queryEmbedResult = await embedPort.embed(
           formatQueryForEmbedding(args.query, embedPort.modelUri)
@@ -196,17 +232,8 @@ export function handleVsearch(
           args.query,
           queryEmbedding,
           {
-            limit: args.limit ?? 5,
-            minScore: args.minScore,
-            collection: args.collection,
-            intent: args.intent,
-            exclude: args.exclude,
-            since: args.since,
-            until: args.until,
-            categories: args.categories,
-            author: args.author,
-            tagsAll: normalizeTagFilters(args.tagsAll),
-            tagsAny: normalizeTagFilters(args.tagsAny),
+            ...options,
+            traceSession,
           }
         );
 
@@ -217,17 +244,23 @@ export function handleVsearch(
         // Enrich with absPath
         const enrichedResults = enrichWithAbsPath(result.value.results, ctx);
 
-        return {
-          ...result.value,
-          results: enrichedResults,
-          meta: {
-            ...result.value.meta,
-            // Add queryLanguage hint if provided (per spec, lang is a hint for vsearch)
-            ...(args.lang ? { queryLanguage: args.lang } : {}),
+        return attachRetrievalTraceMetadata(
+          {
+            ...result.value,
+            results: enrichedResults,
+            meta: {
+              ...result.value.meta,
+              // Add queryLanguage hint if provided (per spec, lang is a hint for vsearch)
+              ...(args.lang ? { queryLanguage: args.lang } : {}),
+            },
           },
-        };
+          traceSession
+        );
+      } catch (cause) {
+        await finishRetrievalTraceAfterError(traceSession, cause);
+        throw cause;
       } finally {
-        await embedPort.dispose();
+        await embedPort?.dispose();
       }
     },
     formatSearchResults

package/src/pipeline/answer.ts

@@ -5,6 +5,7 @@
  * @module src/pipeline/answer
  */
 
+import type { RetrievalTraceSession } from "../core/retrieval-trace-session";
 import type { GenerationPort } from "../llm/types";
 import type { StorePort } from "../store/types";
 import type {
@@ -15,6 +16,11 @@ import type {
 } from "./types";
 
 import { buildAnswerPrompt, type AnswerPromptSource } from "./answer-prompt";
+import { attachCitationTraceMetadata } from "./trace-metadata";
+import {
+  CITATION_TRACE_METADATA,
+  SEARCH_RESULT_PLANNER_METADATA,
+} from "./types";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Constants
@@ -24,6 +30,11 @@ import { buildAnswerPrompt, type AnswerPromptSource } from "./answer-prompt";
 export const ABSTENTION_MESSAGE =
   "I don't have enough information in the provided sources to answer this question.";
 
+export const answerTraceTerminalStatus = (
+  citations: readonly Citation[] | undefined
+): "completed" | "partial" =>
+  citations && citations.length > 0 ? "completed" : "partial";
+
 /** Max characters per document (~8K tokens) */
 const MAX_DOC_CHARS = 32_000;
 
@@ -86,6 +97,65 @@ interface SelectedSource {
   reason: string;
 }
 
+interface ExactAnswerPassage {
+  text: string;
+  startLine: number;
+  endLine: number;
+  passageHash: string;
+}
+
+const sha256 = (value: string): string =>
+  new Bun.CryptoHasher("sha256").update(value).digest("hex");
+
+const completeLinePrefix = (
+  content: string,
+  startLine: number,
+  maxChars: number
+): ExactAnswerPassage | null => {
+  const lines = content.split("\n");
+  if (lines.length === 0) return null;
+  const selected: string[] = [];
+  let characters = 0;
+  for (const line of lines) {
+    const addition = selected.length === 0 ? line.length : line.length + 1;
+    if (characters + addition > maxChars) break;
+    selected.push(line);
+    characters += addition;
+  }
+  if (selected.length === 0) return null;
+  const text = selected.join("\n");
+  if (!text.trim()) return null;
+  return {
+    text,
+    startLine,
+    endLine: startLine + selected.length - 1,
+    passageHash: sha256(text),
+  };
+};
+
+const anchoredCompleteLines = (
+  content: string,
+  metadata: SearchResult[typeof SEARCH_RESULT_PLANNER_METADATA]
+): ExactAnswerPassage | null => {
+  if (
+    !metadata?.passageHash ||
+    metadata.startLine === undefined ||
+    metadata.endLine === undefined ||
+    metadata.endLine < metadata.startLine
+  ) {
+    return null;
+  }
+  const lines = content.split("\n");
+  const text = lines.slice(metadata.startLine - 1, metadata.endLine).join("\n");
+  if (
+    text.split("\n").length !== metadata.endLine - metadata.startLine + 1 ||
+    sha256(text) !== metadata.passageHash
+  ) {
+    return null;
+  }
+  return completeLinePrefix(text, metadata.startLine, MAX_DOC_CHARS);
+};
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Citation Processing
 // ─────────────────────────────────────────────────────────────────────────────
@@ -423,53 +493,78 @@ export async function generateGroundedAnswer(
 ): Promise<AnswerGenerationResult | null> {
   const { genPort, store } = deps;
   const sourceSelection = selectAdaptiveSources(query, results);
+  const finalRanks = new Map(
+    results.map((result, index) => [result, index + 1] as const)
+  );
   const promptSources: AnswerPromptSource[] = [];
   const citations: Citation[] = [];
   let citationIndex = 0;
 
   for (const r of sourceSelection.selected) {
-    let content: string | null = null;
-    let usedFullContent = false;
+    let passage: ExactAnswerPassage | null = null;
+    const plannerMetadata = r[SEARCH_RESULT_PLANNER_METADATA];
+    const sourceHash = r.source.sourceHash;
+    const mirrorHash = r.conversion?.mirrorHash;
 
     // Try to fetch full document content if store available
-    if (store && r.conversion?.mirrorHash) {
-      const contentResult = await store.getContent(r.conversion.mirrorHash);
+    if (store && mirrorHash) {
+      const contentResult = await store.getContent(mirrorHash);
       if (contentResult.ok && contentResult.value) {
-        content = contentResult.value;
-        usedFullContent = true;
-        // Truncate to max doc chars
-        if (content.length > MAX_DOC_CHARS) {
-          content = `${content.slice(0, MAX_DOC_CHARS)}\n\n[... truncated ...]`;
+        if (sha256(contentResult.value) === mirrorHash) {
+          passage =
+            anchoredCompleteLines(contentResult.value, plannerMetadata) ??
+            completeLinePrefix(contentResult.value, 1, MAX_DOC_CHARS);
         }
       }
     }
 
-    // Fallback to snippet if full content unavailable
-    if (!content) {
-      if (!r.snippet || r.snippet.trim().length === 0) {
-        continue;
-      }
-      content =
-        r.snippet.length > MAX_SNIPPET_CHARS
-          ? `${r.snippet.slice(0, MAX_SNIPPET_CHARS)}...`
-          : r.snippet;
+    // Snippet fallback is allowed only when the planner proves it is the exact
+    // complete canonical chunk, never an FTS marker/ellipsis presentation.
+    if (
+      !passage &&
+      plannerMetadata?.passageHash &&
+      plannerMetadata.startLine !== undefined &&
+      plannerMetadata.endLine !== undefined &&
+      sha256(r.snippet) === plannerMetadata.passageHash &&
+      r.snippet.split("\n").length ===
+        plannerMetadata.endLine - plannerMetadata.startLine + 1
+    ) {
+      passage = completeLinePrefix(
+        r.snippet,
+        plannerMetadata.startLine,
+        MAX_SNIPPET_CHARS
+      );
     }
+    if (!(passage && sourceHash && mirrorHash)) continue;
 
     citationIndex += 1;
     promptSources.push({
       index: citationIndex,
       docid: r.docid,
       uri: r.uri,
-      content,
+      content: passage.text,
       guidance: r.context,
     });
-    // Clear line range when citing full content (not a specific snippet)
-    citations.push({
-      docid: r.docid,
-      uri: r.uri,
-      startLine: usedFullContent ? undefined : r.snippetRange?.startLine,
-      endLine: usedFullContent ? undefined : r.snippetRange?.endLine,
-    });
+    citations.push(
+      attachCitationTraceMetadata(
+        {
+          docid: r.docid,
+          uri: r.uri,
+          startLine: passage.startLine,
+          endLine: passage.endLine,
+        },
+        {
+          sourceHash,
+          mirrorHash,
+          passageHash: passage.passageHash,
+          seq: plannerMetadata?.seq,
+          rank: finalRanks.get(r) ?? 1,
+          plannerRank: plannerMetadata?.retrievalRank,
+          sources: plannerMetadata?.sources,
+          graphExpanded: plannerMetadata?.graphExpanded,
+        }
+      )
+    );
   }
 
   if (promptSources.length === 0) {
@@ -528,3 +623,49 @@ export function processAnswerResult(rawResult: AnswerGenerationResult): {
     answerContext: rawResult.answerContext,
   };
 }
+
+/** Process first, then record only citations retained in the final answer. */
+export async function processAnswerResultWithTrace(
+  rawResult: AnswerGenerationResult,
+  traceSession?: RetrievalTraceSession
+): Promise<ReturnType<typeof processAnswerResult>> {
+  const processed = processAnswerResult(rawResult);
+  if (!traceSession || processed.citations.length === 0) return processed;
+  const evidence = processed.citations.flatMap((citation) => {
+    const metadata = citation[CITATION_TRACE_METADATA];
+    if (
+      !metadata ||
+      citation.startLine === undefined ||
+      citation.endLine === undefined
+    ) {
+      return [];
+    }
+    return [
+      {
+        docid: citation.docid,
+        uri: citation.uri,
+        sourceHash: metadata.sourceHash,
+        mirrorHash: metadata.mirrorHash,
+        passageHash: metadata.passageHash,
+        startLine: citation.startLine,
+        endLine: citation.endLine,
+        rank: metadata.rank,
+        ...(metadata.seq === undefined ? {} : { seq: metadata.seq }),
+        ...(metadata.plannerRank === undefined
+          ? {}
+          : { plannerRank: metadata.plannerRank }),
+        ...(metadata.sources === undefined
+          ? {}
+          : { sources: metadata.sources }),
+        ...(metadata.graphExpanded === undefined
+          ? {}
+          : { graphExpanded: metadata.graphExpanded }),
+      },
+    ];
+  });
+  const recorded = await traceSession.recordEvidence("cite", evidence);
+  if (!recorded.ok) {
+    throw new Error(`Trace recording failed: ${recorded.error.message}`);
+  }
+  return processed;
+}

package/src/pipeline/graph-retrieval.ts

@@ -62,6 +62,7 @@ const matchesDocumentFilters = (
     until?: string;
     categories?: string[];
     author?: string;
+    relPathPrefix?: string;
   }
 ): boolean => {
   if (
@@ -201,6 +202,7 @@ export async function expandGraphCandidates(
     until?: string;
     categories?: string[];
     author?: string;
+    relPathPrefix?: string;
   } = {}
 ): Promise<GraphRetrievalResult> {
   const maxCandidates = Math.max(
@@ -249,6 +251,13 @@ export async function expandGraphCandidates(
     if (!doc.mirrorHash) {
       continue;
     }
+    if (
+      options.relPathPrefix !== undefined &&
+      doc.relPath !== options.relPathPrefix &&
+      !doc.relPath.startsWith(`${options.relPathPrefix}/`)
+    ) {
+      continue;
+    }
     const rank =
       seedCandidates.findIndex(
         (candidate) => candidate.mirrorHash === doc.mirrorHash
@@ -325,7 +334,12 @@ export async function expandGraphCandidates(
   }
 
   const metadataFilteredDocs = docsResult.value.filter(
-    (doc) => doc.mirrorHash && matchesDocumentFilters(doc, options)
+    (doc) =>
+      doc.mirrorHash &&
+      (options.relPathPrefix === undefined ||
+        doc.relPath === options.relPathPrefix ||
+        doc.relPath.startsWith(`${options.relPathPrefix}/`)) &&
+      matchesDocumentFilters(doc, options)
   );
   const docs = await filterDocsByTags(store, metadataFilteredDocs, options);
   const docByDocid = new Map(docs.map((doc) => [doc.docid, doc]));