@gmickel/gno 1.16.0 → 1.17.0

package/src/core/sections.ts

@@ -14,6 +14,26 @@ export interface DocumentSection {
 }
 
 const HEADING_REGEX = /^(#{1,6})\s+(.+?)\s*#*\s*$/u;
+const FENCE_REGEX = /^ {0,3}(`{3,}|~{3,})(.*)$/u;
+const FENCE_CLOSE_REGEX = /^ {0,3}(`{3,}|~{3,})[\t ]*$/u;
+
+interface OpenFence {
+  marker: "`" | "~";
+  length: number;
+}
+
+const fenceOpener = (line: string): OpenFence | null => {
+  const match = FENCE_REGEX.exec(line);
+  const run = match?.[1];
+  const suffix = match?.[2] ?? "";
+  if (!run || (run[0] === "`" && suffix.includes("`"))) return null;
+  return { marker: run[0] as OpenFence["marker"], length: run.length };
+};
+
+const closesFence = (line: string, fence: OpenFence): boolean => {
+  const run = FENCE_CLOSE_REGEX.exec(line)?.[1];
+  return Boolean(run && run[0] === fence.marker && run.length >= fence.length);
+};
 
 export function slugifySectionTitle(title: string): string {
   return (
@@ -32,8 +52,18 @@ export function extractSections(content: string): DocumentSection[] {
   const sections: DocumentSection[] = [];
   const counts = new Map<string, number>();
   const lines = content.split("\n");
+  let openFence: OpenFence | null = null;
 
   for (const [index, line] of lines.entries()) {
+    if (openFence) {
+      if (closesFence(line, openFence)) openFence = null;
+      continue;
+    }
+    const opener = fenceOpener(line);
+    if (opener) {
+      openFence = opener;
+      continue;
+    }
     const match = HEADING_REGEX.exec(line);
     if (!match) {
       continue;
@@ -60,3 +90,36 @@ export function extractSections(content: string): DocumentSection[] {
 
   return sections;
 }
+
+/** Extract one inclusive, 1-based line range without normalizing source bytes. */
+export function extractInclusiveLines(
+  content: string,
+  startLine: number,
+  endLine: number
+): string | null {
+  if (
+    content.includes("\r") ||
+    !Number.isSafeInteger(startLine) ||
+    !Number.isSafeInteger(endLine) ||
+    startLine < 1 ||
+    endLine < startLine
+  ) {
+    return null;
+  }
+  const lines = content.split("\n");
+  if (endLine > lines.length) return null;
+  return lines.slice(startLine - 1, endLine).join("\n");
+}
+
+/** Find the nearest Markdown heading governing a 1-based source line. */
+export function headingForLine(
+  sections: readonly DocumentSection[],
+  line: number
+): string | null {
+  let heading: string | null = null;
+  for (const section of sections) {
+    if (section.line > line) break;
+    heading = section.title;
+  }
+  return heading;
+}

package/src/mcp/server.ts

@@ -13,7 +13,13 @@ import { dirname, join } from "node:path";
 import type { Collection, Config } from "../config/types";
 import type { SqliteAdapter } from "../store/sqlite/adapter";
 
-import { MCP_SERVER_NAME, VERSION, getIndexDbPath } from "../app/constants";
+import {
+  DEFAULT_INDEX_NAME,
+  MCP_SERVER_NAME,
+  VERSION,
+  getIndexDbPath,
+} from "../app/constants";
+import { canonicalizeIndexName } from "../app/index-name";
 import { JobManager } from "../core/job-manager";
 import { envIsSet } from "../llm/policy";
 import { MCP_ACTIVATION_VERIFICATION_ENV } from "./activation-verification-mode";
@@ -58,7 +64,7 @@ export interface ToolContext {
   config: Config;
   collections: Collection[];
   actualConfigPath: string;
-  indexName?: string;
+  indexName: string;
   toolMutex: Mutex;
   jobManager: JobManager;
   serverInstanceId: string;
@@ -120,7 +126,7 @@ export async function startMcpServer(options: McpServerOptions): Promise<void> {
     MCP_ACTIVATION_VERIFICATION_ENV
   );
   const init = await initStore({
-    indexName: options.indexName,
+    indexName: canonicalizeIndexName(options.indexName ?? DEFAULT_INDEX_NAME),
     configPath: options.configPath,
     syncConfig: !activationVerification,
   });
@@ -172,7 +178,7 @@ export async function startMcpServer(options: McpServerOptions): Promise<void> {
     config,
     collections,
     actualConfigPath,
-    indexName: options.indexName,
+    indexName: canonicalizeIndexName(options.indexName ?? DEFAULT_INDEX_NAME),
     toolMutex,
     jobManager,
     serverInstanceId,

package/src/mcp/tools/context.ts

@@ -0,0 +1,229 @@
+/** MCP Context Capsule tools over the shared application runtime. */
+
+import type { EmbeddingPort, RerankPort } from "../../llm/types";
+import type { VectorIndexPort } from "../../store/vector";
+import type { ToolContext } from "../server";
+import type { ToolResult } from "./index";
+
+import { formatContextCapsuleAgentJson } from "../../app/context-agent-projection";
+import { formatContextCapsuleVerificationMarkdown } from "../../app/context-format";
+import {
+  buildContextCapsule,
+  canonicalVerifiedContextCapsuleJson,
+  validateContextCapsuleBuildInput,
+  verifyContextCapsuleRuntime,
+} from "../../app/context-runtime";
+import {
+  contextSurfaceError,
+  parseContextBuildSurfaceInput,
+  parseContextVerifySurfaceInput,
+} from "../../app/context-surface";
+import { createNonTtyProgressRenderer } from "../../cli/progress";
+import { LlmAdapter } from "../../llm/nodeLlamaCpp/adapter";
+import { resolveDownloadPolicy } from "../../llm/policy";
+import { resolveModelUri } from "../../llm/registry";
+import { createVectorIndexPort } from "../../store/vector";
+
+interface ContextToolResultData {
+  structuredContent: Record<string, unknown>;
+  text: string;
+}
+
+const asToolResult = (data: ContextToolResultData): ToolResult => ({
+  content: [{ type: "text", text: data.text }],
+  structuredContent: data.structuredContent,
+});
+
+const asToolError = (error: unknown): ToolResult => {
+  const publicError = contextSurfaceError(error);
+  return {
+    isError: true,
+    content: [
+      {
+        type: "text",
+        text: `Error [${publicError.code}]: ${publicError.message}`,
+      },
+    ],
+    structuredContent: {
+      error: publicError.code,
+      message: publicError.message,
+    },
+  };
+};
+
+const runContextTool = async (
+  context: ToolContext,
+  operation: () => Promise<ContextToolResultData>
+): Promise<ToolResult> => {
+  if (context.isShuttingDown()) {
+    return asToolError(
+      Object.assign(new Error("Server is shutting down"), {
+        code: "runtime_error",
+      })
+    );
+  }
+  const release = await context.toolMutex.acquire();
+  try {
+    return asToolResult(await operation());
+  } catch (error) {
+    return asToolError(error);
+  } finally {
+    release();
+  }
+};
+
+interface McpModelPorts {
+  embedPort: EmbeddingPort | null;
+  rerankPort: RerankPort | null;
+  vectorIndex: VectorIndexPort | null;
+  dispose(): Promise<void>;
+}
+
+interface McpModelPortFactory {
+  createEmbeddingPort: LlmAdapter["createEmbeddingPort"];
+  createRerankPort: LlmAdapter["createRerankPort"];
+  dispose: LlmAdapter["dispose"];
+}
+
+export const disposeContextModelOwners = async (
+  portOwners: readonly { dispose(): Promise<void> }[],
+  managerOwner: { dispose(): Promise<void> }
+): Promise<void> => {
+  await Promise.allSettled(
+    portOwners.map((owner) => Promise.resolve().then(() => owner.dispose()))
+  );
+  await Promise.allSettled([
+    Promise.resolve().then(() => managerOwner.dispose()),
+  ]);
+};
+
+export const createMcpModelPorts = async (
+  context: ToolContext,
+  collection?: string,
+  factoryOverride?: McpModelPortFactory
+): Promise<McpModelPorts> => {
+  const llm = new LlmAdapter(context.config);
+  const factory = factoryOverride ?? llm;
+  const policy = resolveDownloadPolicy(process.env, {});
+  const progress = createNonTtyProgressRenderer();
+  const embedUri = resolveModelUri(
+    context.config,
+    "embed",
+    undefined,
+    collection
+  );
+  let embedPort: EmbeddingPort | null = null;
+  let ownedEmbedPort: EmbeddingPort | null = null;
+  let rerankPort: RerankPort | null = null;
+  let vectorIndex: VectorIndexPort | null = null;
+  try {
+    const embedResult = await factory.createEmbeddingPort(embedUri, {
+      policy,
+      onProgress: (value) => progress("embed", value),
+    });
+    if (embedResult.ok) {
+      // Take ownership before init: init failures must not leak the port.
+      ownedEmbedPort = embedResult.value;
+      const initialized = await ownedEmbedPort.init();
+      if (initialized.ok) {
+        embedPort = ownedEmbedPort;
+        const vectorResult = await createVectorIndexPort(
+          context.store.getRawDb(),
+          { model: embedUri, dimensions: ownedEmbedPort.dimensions() }
+        );
+        if (vectorResult.ok) vectorIndex = vectorResult.value;
+      }
+    }
+    const rerankResult = await factory.createRerankPort(
+      resolveModelUri(context.config, "rerank", undefined, collection),
+      {
+        policy,
+        onProgress: (value) => progress("rerank", value),
+      }
+    );
+    if (rerankResult.ok) rerankPort = rerankResult.value;
+    return {
+      embedPort,
+      rerankPort,
+      vectorIndex,
+      async dispose() {
+        await disposeContextModelOwners(
+          [ownedEmbedPort, rerankPort].filter(
+            (port): port is EmbeddingPort | RerankPort => port !== null
+          ),
+          factory
+        );
+      },
+    };
+  } catch (error) {
+    await disposeContextModelOwners(
+      [ownedEmbedPort, rerankPort].filter(
+        (port): port is EmbeddingPort | RerankPort => port !== null
+      ),
+      factory
+    );
+    throw error;
+  }
+};
+
+export const handleContext = (
+  args: unknown,
+  context: ToolContext
+): Promise<ToolResult> =>
+  runContextTool(context, async () => {
+    const parsed = parseContextBuildSurfaceInput(args, context.indexName);
+    // This guard is intentionally before any model construction/download.
+    validateContextCapsuleBuildInput(
+      parsed.input,
+      context.indexName,
+      context.config.collections.map((collection) => collection.name)
+    );
+    const useModels = parsed.input.depthPolicy !== "fast";
+    const modelPorts = useModels
+      ? await createMcpModelPorts(
+          context,
+          parsed.input.collections?.length === 1
+            ? parsed.input.collections[0]
+            : undefined
+        )
+      : null;
+    try {
+      const capsule = await buildContextCapsule(parsed.input, {
+        store: context.store,
+        config: context.config,
+        indexName: context.indexName,
+        vectorIndex: modelPorts?.vectorIndex ?? null,
+        embedPort: modelPorts?.embedPort ?? null,
+        rerankPort: modelPorts?.rerankPort ?? null,
+      });
+      return {
+        structuredContent: capsule as unknown as Record<string, unknown>,
+        // MCP model context receives this projection exactly once. The full
+        // canonical capsule remains available to application clients through
+        // structuredContent and is deliberately not duplicated in text.
+        text: formatContextCapsuleAgentJson(capsule),
+      };
+    } finally {
+      await modelPorts?.dispose();
+    }
+  });
+
+export const handleContextVerify = (
+  args: unknown,
+  context: ToolContext
+): Promise<ToolResult> =>
+  runContextTool(context, async () => {
+    const parsed = parseContextVerifySurfaceInput(args);
+    const receipt = await verifyContextCapsuleRuntime(parsed.capsule, {
+      store: context.store,
+      config: context.config,
+      indexName: context.indexName,
+    });
+    return {
+      structuredContent: receipt as unknown as Record<string, unknown>,
+      text:
+        parsed.format === "md"
+          ? formatContextCapsuleVerificationMarkdown(receipt)
+          : canonicalVerifiedContextCapsuleJson(receipt),
+    };
+  });

package/src/mcp/tools/index.ts

@@ -10,12 +10,17 @@ import { z } from "zod";
 
 import type { ToolContext } from "../server";
 
+import {
+  contextBuildSurfaceSchema,
+  contextVerifySurfaceSchema,
+} from "../../app/context-surface";
 import { CAPTURE_MAX_TEXT_BYTES } from "../../core/capture";
 import { NOTE_PRESETS, type NotePresetId } from "../../core/note-presets";
 import { normalizeTag } from "../../core/tags";
 import { handleAddCollection } from "./add-collection";
 import { handleCapture } from "./capture";
 import { handleClearCollectionEmbeddings } from "./clear-collection-embeddings";
+import { handleContext, handleContextVerify } from "./context";
 import { handleEmbed } from "./embed";
 import { handleGet } from "./get";
 import { handleIndex } from "./index-cmd";
@@ -72,6 +77,10 @@ export const MCP_TOOL_DESCRIPTIONS = {
     "Retrieve multiple documents by refs array or glob pattern. Use after gno_search/gno_query to batch top result URIs/docids; set maxBytes and lineNumbers to control context size.",
   status:
     "Get index health: collection count, document count, chunk count, embedding backlog, and per-collection stats. Check first when vector/hybrid results look stale or unavailable.",
+  context:
+    "Compile one deterministic, budgeted, extractive evidence Capsule with exact line spans, coverage gaps, omissions, provenance, and verification fingerprints. Raw search/get tools remain available for manual retrieval.",
+  contextVerify:
+    "Verify a saved Context Capsule without rebuilding or mutating it. Reports unchanged, stale, missing, reranked, and fingerprint drift states against the active index.",
 } as const;
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -899,6 +908,24 @@ function parseErrorMessage(message: string): { [x: string]: unknown } {
 
 export function registerTools(server: McpServer, ctx: ToolContext): void {
   // Tool IDs use underscores (MCP pattern: ^[a-zA-Z0-9_-]{1,64}$)
+  server.registerTool(
+    "gno_context",
+    {
+      description: MCP_TOOL_DESCRIPTIONS.context,
+      inputSchema: contextBuildSurfaceSchema,
+    },
+    (args) => handleContext(args, ctx)
+  );
+
+  server.registerTool(
+    "gno_context_verify",
+    {
+      description: MCP_TOOL_DESCRIPTIONS.contextVerify,
+      inputSchema: contextVerifySurfaceSchema,
+    },
+    (args) => handleContextVerify(args, ctx)
+  );
+
   server.tool(
     "gno_search",
     MCP_TOOL_DESCRIPTIONS.search,

package/src/pipeline/chunk-lookup.ts

@@ -42,3 +42,36 @@ export function createChunkLookup(
     return bySeq.get(seq);
   };
 }
+
+/**
+ * Prove that persisted chunk coordinates still address the exact stored
+ * canonical mirror bytes. Chunk text may begin or end mid-line; callers that
+ * need citation spans must expand it to full lines separately.
+ */
+export function chunkMatchesCanonicalContent(
+  chunk: ChunkRow,
+  content: string
+): boolean {
+  if (
+    content.includes("\r") ||
+    chunk.pos < 0 ||
+    chunk.startLine < 1 ||
+    chunk.endLine < chunk.startLine ||
+    chunk.text.length === 0 ||
+    chunk.pos + chunk.text.length > content.length
+  ) {
+    return false;
+  }
+  if (content.slice(chunk.pos, chunk.pos + chunk.text.length) !== chunk.text) {
+    return false;
+  }
+  let startLine = 1;
+  let endLine = 1;
+  const endOffset = chunk.pos + chunk.text.length - 1;
+  for (let offset = 0; offset < endOffset; offset += 1) {
+    if (content[offset] !== "\n") continue;
+    if (offset < chunk.pos) startLine += 1;
+    endLine += 1;
+  }
+  return startLine === chunk.startLine && endLine === chunk.endLine;
+}

package/src/pipeline/hybrid.ts

@@ -54,7 +54,10 @@ import {
   resolveTemporalRange,
   shouldSortByRecency,
 } from "./temporal";
-import { DEFAULT_PIPELINE_CONFIG } from "./types";
+import {
+  DEFAULT_PIPELINE_CONFIG,
+  SEARCH_RESULT_PLANNER_METADATA,
+} from "./types";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Dependencies
@@ -719,7 +722,16 @@ export async function searchHybrid(
   }
 
   // Build lookup maps.
-  const docByMirrorHash = new Map<string, (typeof docsResult.value)[number]>();
+  const docsByMirrorHash = new Map<
+    string,
+    (typeof docsResult.value)[number][]
+  >();
+  const addDocument = (doc: (typeof docsResult.value)[number]): void => {
+    if (!doc.mirrorHash) return;
+    const docs = docsByMirrorHash.get(doc.mirrorHash) ?? [];
+    docs.push(doc);
+    docsByMirrorHash.set(doc.mirrorHash, docs);
+  };
   const matchesMetadataFilters = (
     doc: (typeof docsResult.value)[number]
   ): boolean => {
@@ -761,7 +773,7 @@ export async function searchHybrid(
       candidateDocs.push(doc);
     } else {
       if (matchesMetadataFilters(doc)) {
-        docByMirrorHash.set(doc.mirrorHash, doc);
+        addDocument(doc);
       }
     }
   }
@@ -789,12 +801,20 @@ export async function searchHybrid(
         }
 
         if (doc.mirrorHash && matchesMetadataFilters(doc)) {
-          docByMirrorHash.set(doc.mirrorHash, doc);
+          addDocument(doc);
         }
       }
     }
   }
 
+  for (const docs of docsByMirrorHash.values()) {
+    docs.sort((left, right) => {
+      if (left.uri < right.uri) return -1;
+      if (left.uri > right.uri) return 1;
+      return left.docid < right.docid ? -1 : left.docid > right.docid ? 1 : 0;
+    });
+  }
+
   const collectionPaths = new Map<string, string>();
   if (collectionsResult.ok) {
     for (const c of collectionsResult.value) {
@@ -823,34 +843,19 @@ export async function searchHybrid(
   const seenDocids = new Set<string>();
 
   // Iterate until we have enough results (don't slice early - deduping may skip candidates)
-  for (const candidate of filteredCandidates) {
+  for (const [candidateIndex, candidate] of filteredCandidates.entries()) {
     // Stop when we have enough results
     if (results.length >= assemblyLimit) {
       break;
     }
 
     // Find document from pre-fetched map
-    const doc = docByMirrorHash.get(candidate.mirrorHash);
-    if (!doc) {
+    const candidateDocs = docsByMirrorHash.get(candidate.mirrorHash) ?? [];
+    if (candidateDocs.length === 0) {
       continue;
     }
 
     const docChunks = chunksMap.get(candidate.mirrorHash) ?? [];
-    const filterEval = evaluateDocumentChunkFilters(
-      query,
-      doc,
-      docChunks,
-      options
-    );
-    if (!filterEval.matches) {
-      continue;
-    }
-
-    // For --full mode, de-dupe by docid (keep best scoring candidate per doc)
-    if (options.full && seenDocids.has(doc.docid)) {
-      continue;
-    }
-
     // Get chunk via O(1) lookup
     // For doc-level FTS (seq=0), fall back to first available chunk if exact lookup fails
     let chunk = getChunk(candidate.mirrorHash, candidate.seq);
@@ -868,10 +873,6 @@ export async function searchHybrid(
       continue;
     }
 
-    docidMap.set(`${candidate.mirrorHash}:${candidate.seq}`, doc.docid);
-
-    const collectionPath = collectionPaths.get(doc.collection);
-
     // For --full mode, fetch full mirror content
     const snippetChunk =
       options.full || !options.intent?.trim()
@@ -907,37 +908,58 @@ export async function searchHybrid(
       // Fallback to chunk text if content unavailable
     }
 
-    seenDocids.add(doc.docid);
-
-    results.push({
-      docid: doc.docid,
-      score: candidate.blendedScore,
-      uri: doc.uri,
-      title: doc.title ?? undefined,
-      contentType: doc.contentType ?? undefined,
-      categories: doc.categories ?? undefined,
-      line: snippetChunk.startLine,
-      snippet,
-      snippetLanguage: chunk.language ?? undefined,
-      snippetRange,
-      source: {
-        relPath: doc.relPath,
-        absPath: collectionPath
-          ? `${collectionPath}/${doc.relPath}`
-          : undefined,
-        mime: doc.sourceMime,
-        ext: doc.sourceExt,
-        modifiedAt: doc.sourceMtime,
-        documentDate: doc.frontmatterDate ?? undefined,
-        sizeBytes: doc.sourceSize,
-        sourceHash: doc.sourceHash,
-      },
-      conversion: {
-        mirrorHash: candidate.mirrorHash,
-        converterId: doc.converterId ?? undefined,
-        converterVersion: doc.converterVersion ?? undefined,
-      },
-    });
+    for (const doc of candidateDocs) {
+      if (results.length >= assemblyLimit) break;
+      const filterEval = evaluateDocumentChunkFilters(
+        query,
+        doc,
+        docChunks,
+        options
+      );
+      if (!filterEval.matches || (options.full && seenDocids.has(doc.docid))) {
+        continue;
+      }
+      const docidKey = `${candidate.mirrorHash}:${candidate.seq}`;
+      if (!docidMap.has(docidKey)) docidMap.set(docidKey, doc.docid);
+      const collectionPath = collectionPaths.get(doc.collection);
+      seenDocids.add(doc.docid);
+      results.push({
+        docid: doc.docid,
+        score: candidate.blendedScore,
+        uri: doc.uri,
+        title: doc.title ?? undefined,
+        contentType: doc.contentType ?? undefined,
+        categories: doc.categories ?? undefined,
+        line: snippetChunk.startLine,
+        snippet,
+        snippetLanguage: chunk.language ?? undefined,
+        snippetRange,
+        source: {
+          relPath: doc.relPath,
+          absPath: collectionPath
+            ? `${collectionPath}/${doc.relPath}`
+            : undefined,
+          mime: doc.sourceMime,
+          ext: doc.sourceExt,
+          modifiedAt: doc.sourceMtime,
+          documentDate: doc.frontmatterDate ?? undefined,
+          sizeBytes: doc.sourceSize,
+          sourceHash: doc.sourceHash,
+        },
+        conversion: {
+          mirrorHash: candidate.mirrorHash,
+          converterId: doc.converterId ?? undefined,
+          converterVersion: doc.converterVersion ?? undefined,
+        },
+        [SEARCH_RESULT_PLANNER_METADATA]: {
+          retrievalRank: candidateIndex + 1,
+          mirrorHash: candidate.mirrorHash,
+          seq: snippetChunk.seq,
+          sources: [...candidate.sources].sort(),
+          graphExpanded: candidate.sources.includes("graph"),
+        },
+      });
+    }
   }
   timings.assemblyMs = performance.now() - assemblyStartedAt;
   timings.totalMs = performance.now() - runStartedAt;

package/src/pipeline/types.ts

@@ -37,6 +37,19 @@ export interface SnippetRange {
   endLine: number;
 }
 
+/** Symbol-keyed planner metadata; omitted from JSON/API projections. */
+export const SEARCH_RESULT_PLANNER_METADATA = Symbol(
+  "gno.searchResultPlannerMetadata"
+);
+
+export interface SearchResultPlannerMetadata {
+  retrievalRank: number;
+  mirrorHash: string;
+  seq: number;
+  sources: FusionSource[];
+  graphExpanded: boolean;
+}
+
 /** Single search result matching output schema */
 export interface SearchResult {
   docid: string;
@@ -53,6 +66,7 @@ export interface SearchResult {
   context?: string;
   source: SearchResultSource;
   conversion?: SearchResultConversion;
+  [SEARCH_RESULT_PLANNER_METADATA]?: SearchResultPlannerMetadata;
 }
 
 /** Search mode enum */