@gmickel/gno 1.12.2 → 1.12.4

package/src/ingestion/sync.ts

@@ -75,6 +75,7 @@ const MAX_CONCURRENCY = 16;
 export const INGEST_VERSION = 6;
 const EMPTY_CONTENT_TYPE_RULES_FINGERPRINT = fingerprintContentTypeRules([]);
 const RELATION_EDGE_TYPE_PATTERN = /^[a-z][a-z0-9_]*$/;
+const PROJECTION_YIELD_INTERVAL = 25;
 const NON_RETRYABLE_CONVERSION_ERROR_CODES = new Set([
   "CORRUPT",
   "PERMISSION",
@@ -1000,19 +1001,83 @@ export class SyncService {
     relPaths: string[],
     options: SyncOptions = {}
   ): Promise<FileSyncResult[]> {
+    const result = await this.syncPaths(collection, store, relPaths, options);
+    return result.files ?? [];
+  }
+
+  async syncPaths(
+    collection: Collection,
+    store: StorePort,
+    relPaths: string[],
+    options: SyncOptions = {}
+  ): Promise<CollectionSyncResult> {
+    const startedAt = Date.now();
+    const syncOptions: SyncOptions = {
+      ...options,
+      contentTypeRules: options.contentTypeRules ?? [],
+      contentTypeRulesFingerprint:
+        options.contentTypeRulesFingerprint ??
+        fingerprintContentTypeRules(options.contentTypeRules ?? []),
+    };
     const results: FileSyncResult[] = [];
+    const projectionSourceIds = new Set<number>();
+    let markedInactive = 0;
 
     for (const relPath of relPaths) {
+      const existingResult = await store.getDocument(collection.name, relPath);
+      const existingDoc = existingResult.ok ? existingResult.value : null;
+      if (existingDoc) {
+        await this.collectProjectionSourceIds(
+          store,
+          existingDoc.id,
+          projectionSourceIds
+        );
+      }
+
       const absPath = join(collection.path, relPath);
       let stats: Awaited<ReturnType<typeof stat>>;
       try {
         stats = await stat(absPath);
-      } catch {
+      } catch (error) {
+        const errorCode =
+          error && typeof error === "object" && "code" in error
+            ? String(error.code)
+            : undefined;
+        if (errorCode !== "ENOENT") {
+          results.push({
+            relPath,
+            status: "error",
+            errorCode: "STAT_FAILED",
+            errorMessage:
+              error instanceof Error ? error.message : "Failed to stat file",
+          });
+          continue;
+        }
+        if (existingDoc?.active) {
+          const inactiveResult = await store.markInactive(collection.name, [
+            relPath,
+          ]);
+          if (!inactiveResult.ok) {
+            results.push({
+              relPath,
+              status: "error",
+              errorCode: inactiveResult.error.code,
+              errorMessage: inactiveResult.error.message,
+            });
+            continue;
+          }
+          markedInactive += inactiveResult.value;
+          results.push({
+            relPath,
+            status: "updated",
+            docid: existingDoc.docid,
+          });
+          continue;
+        }
         results.push({
           relPath,
-          status: "error",
-          errorCode: "NOT_FOUND",
-          errorMessage: "File not found",
+          status: existingDoc ? "unchanged" : "skipped",
+          docid: existingDoc?.docid,
         });
         continue;
       }
@@ -1035,24 +1100,91 @@ export class SyncService {
         ctime: (stats.birthtime ?? stats.ctime ?? stats.mtime).toISOString(),
       };
 
-      const result = await this.processFile(collection, entry, store, options);
+      const result = await this.processFile(
+        collection,
+        entry,
+        store,
+        syncOptions
+      );
       results.push(result);
+      const currentResult = await store.getDocument(collection.name, relPath);
+      const currentDoc = currentResult.ok ? currentResult.value : null;
+      if (currentDoc?.active) {
+        await this.collectProjectionSourceIds(
+          store,
+          currentDoc.id,
+          projectionSourceIds
+        );
+      }
     }
 
-    await this.projectTypedEdges(collection, store, options);
+    const errors =
+      syncOptions.projectTypedEdges === false
+        ? []
+        : await this.projectTypedEdges(store, syncOptions, projectionSourceIds);
+    const added = results.filter((result) => result.status === "added").length;
+    const updated = results.filter(
+      (result) => result.status === "updated"
+    ).length;
+    const unchanged = results.filter(
+      (result) => result.status === "unchanged"
+    ).length;
+    const errored = results.filter(
+      (result) => result.status === "error"
+    ).length;
+    const skipped = results.filter(
+      (result) => result.status === "skipped"
+    ).length;
 
-    return results;
+    return {
+      collection: collection.name,
+      filesProcessed: results.length,
+      filesAdded: added,
+      filesUpdated: updated,
+      filesUnchanged: unchanged,
+      filesErrored: errored,
+      filesSkipped: skipped,
+      filesMarkedInactive: markedInactive,
+      durationMs: Date.now() - startedAt,
+      files: results,
+      errors,
+    };
+  }
+
+  private async collectProjectionSourceIds(
+    store: StorePort,
+    documentId: number,
+    sourceIds: Set<number>
+  ): Promise<void> {
+    sourceIds.add(documentId);
+    const [linkBacklinks, edgeBacklinks] = await Promise.all([
+      store.getBacklinksForDoc(documentId),
+      store.getEdgeBacklinksForDoc(documentId),
+    ]);
+    if (linkBacklinks.ok) {
+      for (const backlink of linkBacklinks.value) {
+        sourceIds.add(backlink.sourceDocId);
+      }
+    }
+    if (edgeBacklinks.ok) {
+      for (const backlink of edgeBacklinks.value) {
+        sourceIds.add(backlink.sourceDocId);
+      }
+    }
   }
 
   private async projectTypedEdges(
-    collection: Collection,
     store: StorePort,
-    options: SyncOptions
+    options: SyncOptions,
+    sourceDocumentIds?: Set<number>
   ): Promise<Array<{ relPath: string; code: string; message: string }>> {
     const errors: Array<{ relPath: string; code: string; message: string }> =
       [];
 
-    const backfillResult = await store.backfillDocEdges();
+    const selectedSourceIds = sourceDocumentIds
+      ? [...sourceDocumentIds]
+      : undefined;
+    const backfillResult = await store.backfillDocEdges(selectedSourceIds);
     if (!backfillResult.ok) {
       return [
         {
@@ -1075,8 +1207,22 @@ export class SyncService {
     }
 
     const activeDocs = docsResult.value.filter((doc) => doc.active);
+    const activeIds = new Set(activeDocs.map((doc) => doc.id));
+    if (selectedSourceIds) {
+      for (const documentId of selectedSourceIds) {
+        if (!activeIds.has(documentId)) {
+          await store.setDocEdges(documentId, [], "frontmatter-relation");
+        }
+      }
+    }
+    const projectedDocs = sourceDocumentIds
+      ? activeDocs.filter((doc) => sourceDocumentIds.has(doc.id))
+      : activeDocs;
 
-    for (const doc of activeDocs) {
+    for (const [docIndex, doc] of projectedDocs.entries()) {
+      if (docIndex > 0 && docIndex % PROJECTION_YIELD_INTERVAL === 0) {
+        await Bun.sleep(0);
+      }
       if (!doc.mirrorHash) {
         continue;
       }
@@ -1193,6 +1339,14 @@ export class SyncService {
     return errors;
   }
 
+  /** Run an exact global typed-edge reconciliation with cooperative yields. */
+  reconcileTypedEdges(
+    store: StorePort,
+    options: SyncOptions = {}
+  ): Promise<Array<{ relPath: string; code: string; message: string }>> {
+    return this.projectTypedEdges(store, options);
+  }
+
   /**
    * Sync a single collection.
    */
@@ -1419,9 +1573,9 @@ export class SyncService {
       }
     }
 
-    errors.push(
-      ...(await this.projectTypedEdges(collection, store, syncOptions))
-    );
+    if (syncOptions.projectTypedEdges !== false) {
+      errors.push(...(await this.projectTypedEdges(store, syncOptions)));
+    }
 
     return {
       collection: collection.name,
@@ -1448,12 +1602,25 @@ export class SyncService {
   ): Promise<SyncResult> {
     const startTime = Date.now();
     const results: CollectionSyncResult[] = [];
+    const deferredProjectionOptions: SyncOptions = {
+      ...options,
+      projectTypedEdges: false,
+    };
 
     for (const collection of collections) {
-      const result = await this.syncCollection(collection, store, options);
+      const result = await this.syncCollection(
+        collection,
+        store,
+        deferredProjectionOptions
+      );
       results.push(result);
     }
 
+    if (results.length > 0) {
+      const projectionErrors = await this.projectTypedEdges(store, options);
+      results.at(-1)?.errors.push(...projectionErrors);
+    }
+
     // Aggregate totals
     const totals = results.reduce(
       (acc, r) => ({

package/src/ingestion/types.ts

@@ -137,6 +137,8 @@ export interface SyncOptions {
   contentTypeRules?: NormalizedContentTypeRule[];
   /** Stable hash of the normalized content type rules, used for re-derivation. */
   contentTypeRulesFingerprint?: string;
+  /** Internal orchestration flag: defer graph projection to an outer sync. */
+  projectTypedEdges?: boolean;
 }
 
 export type ContentTypeSource =

package/src/mcp/resources/index.ts

@@ -20,8 +20,10 @@ import {
   URI_PREFIX,
 } from "../../app/constants";
 import { MCP_ERRORS } from "../../core/errors";
+import { resolveEffectiveIndex } from "../../core/indexed-reference";
 import { normalizeTag, validateTag } from "../../core/tags";
 import { normalizeCollectionName } from "../../core/validation";
+import { openScopedIndexStore } from "../../store/sqlite/scoped-index";
 
 // Tags resource URI prefix
 const TAGS_URI = `${URI_PREFIX}tags`;
@@ -51,7 +53,8 @@ function formatTagsContent(
 function formatResourceContent(
   doc: DocumentRow,
   content: string,
-  ctx: ToolContext
+  ctx: ToolContext,
+  indexName = ctx.indexName
 ): string {
   // Find collection for absPath
   const uriParsed = parseUri(doc.uri);
@@ -69,7 +72,7 @@ function formatResourceContent(
   const langLine = doc.languageHint
     ? `\n     language: ${doc.languageHint}`
     : "";
-  const displayUri = decorateUriForIndex(doc.uri, ctx.indexName);
+  const displayUri = decorateUriForIndex(doc.uri, indexName);
   const header = `<!-- ${displayUri}
      docid: ${doc.docid}
      source: ${absPath}
@@ -125,61 +128,82 @@ export function registerResources(server: McpServer, ctx: ToolContext): void {
         throw new Error(`Invalid gno:// URI: ${uri.href}`);
       }
 
-      const { collection, path } = parsed;
-
-      // Validate collection exists
-      const collectionExists = ctx.collections.some(
-        (c) => c.name === collection
-      );
-      if (!collectionExists) {
-        throw new Error(`Collection not found: ${collection}`);
+      const resolution = resolveEffectiveIndex([uri.href], ctx.indexName);
+      if (!resolution.ok) {
+        throw new Error(resolution.error);
       }
+      const scoped = await openScopedIndexStore({
+        activeStore: ctx.store,
+        activeIndexName: ctx.indexName,
+        requestedIndexName: resolution.value.indexName,
+        config: ctx.config,
+        configPath: ctx.actualConfigPath,
+      });
+
+      try {
+        const { collection, path } = parsed;
 
-      // Look up document (path is properly decoded by parseUri)
-      const docResult = await ctx.store.getDocument(collection, path);
-      if (!docResult.ok) {
-        throw new Error(
-          `Failed to lookup document: ${docResult.error.message}`
+        // Validate collection exists
+        const collectionExists = ctx.collections.some(
+          (c) => c.name === collection
         );
-      }
+        if (!collectionExists) {
+          throw new Error(`Collection not found: ${collection}`);
+        }
 
-      const doc = docResult.value;
-      if (!doc) {
-        throw new Error(`Document not found: ${uri.href}`);
-      }
+        // Look up document (path is properly decoded by parseUri)
+        const docResult = await scoped.store.getDocument(collection, path);
+        if (!docResult.ok) {
+          throw new Error(
+            `Failed to lookup document: ${docResult.error.message}`
+          );
+        }
 
-      // Get content
-      if (!doc.mirrorHash) {
-        throw new Error(`Document has no indexed content: ${uri.href}`);
-      }
+        const doc = docResult.value;
+        if (!doc) {
+          throw new Error(`Document not found: ${uri.href}`);
+        }
 
-      const contentResult = await ctx.store.getContent(doc.mirrorHash);
-      if (!contentResult.ok) {
-        throw new Error(
-          `Failed to read content: ${contentResult.error.message}`
-        );
-      }
+        // Get content
+        if (!doc.mirrorHash) {
+          throw new Error(`Document has no indexed content: ${uri.href}`);
+        }
 
-      const content = contentResult.value ?? "";
+        const contentResult = await scoped.store.getContent(doc.mirrorHash);
+        if (!contentResult.ok) {
+          throw new Error(
+            `Failed to read content: ${contentResult.error.message}`
+          );
+        }
 
-      // Format with header and line numbers
-      const formattedContent = formatResourceContent(doc, content, ctx);
+        const content = contentResult.value ?? "";
 
-      // Build canonical URI
-      const canonicalUri = decorateUriForIndex(
-        buildUri(collection, path),
-        parsed.indexName ?? ctx.indexName
-      );
+        // Format with header and line numbers
+        const formattedContent = formatResourceContent(
+          doc,
+          content,
+          ctx,
+          scoped.indexName
+        );
 
-      return {
-        contents: [
-          {
-            uri: canonicalUri,
-            mimeType: "text/markdown",
-            text: formattedContent,
-          },
-        ],
-      };
+        // Build canonical URI
+        const canonicalUri = decorateUriForIndex(
+          buildUri(collection, path),
+          scoped.indexName
+        );
+
+        return {
+          contents: [
+            {
+              uri: canonicalUri,
+              mimeType: "text/markdown",
+              text: formattedContent,
+            },
+          ],
+        };
+      } finally {
+        await scoped.close();
+      }
     } finally {
       release();
     }

package/src/mcp/tools/get.ts

@@ -14,7 +14,9 @@ import {
   getDocumentCapabilities,
   type DocumentCapabilities,
 } from "../../core/document-capabilities";
+import { resolveEffectiveIndex } from "../../core/indexed-reference";
 import { parseRef } from "../../core/ref-parser";
+import { openScopedIndexStore } from "../../store/sqlite/scoped-index";
 import { runTool, type ToolResult } from "./index";
 
 interface GetInput {
@@ -128,109 +130,122 @@ export function handleGet(
         throw new Error(parsed.error);
       }
 
-      // Lookup document
-      const doc = await lookupDocument(ctx.store, parsed);
-      if (!doc) {
-        throw new Error(`Document not found: ${args.ref}`);
+      const resolution = resolveEffectiveIndex([args.ref], ctx.indexName);
+      if (!resolution.ok) {
+        throw new Error(resolution.error);
       }
+      const scoped = await openScopedIndexStore({
+        activeStore: ctx.store,
+        activeIndexName: ctx.indexName,
+        requestedIndexName: resolution.value.indexName,
+        config: ctx.config,
+        configPath: ctx.actualConfigPath,
+      });
+
+      try {
+        // Lookup document
+        const doc = await lookupDocument(scoped.store, parsed);
+        if (!doc) {
+          throw new Error(`Document not found: ${args.ref}`);
+        }
 
-      // Get content
-      if (!doc.mirrorHash) {
-        throw new Error("Document has no indexed content");
-      }
+        // Get content
+        if (!doc.mirrorHash) {
+          throw new Error("Document has no indexed content");
+        }
 
-      const contentResult = await ctx.store.getContent(doc.mirrorHash);
-      if (!contentResult.ok) {
-        throw new Error(contentResult.error.message);
-      }
+        const contentResult = await scoped.store.getContent(doc.mirrorHash);
+        if (!contentResult.ok) {
+          throw new Error(contentResult.error.message);
+        }
 
-      const fullContent = contentResult.value ?? "";
-      const contentLines = fullContent.split("\n");
-      const totalLines = contentLines.length;
-
-      // Apply line range if specified
-      let content = fullContent;
-      let returnedLines: { start: number; end: number } | undefined;
-
-      // lineNumbers defaults to true per spec
-      const showLineNumbers = args.lineNumbers !== false;
-
-      if (args.fromLine || args.lineCount) {
-        const startLine = args.fromLine ?? 1;
-        // Clamp startLine to valid range
-        if (startLine > totalLines) {
-          // Return empty content for out-of-range request
-          content = "";
-          returnedLines = undefined;
-        } else {
-          const count = args.lineCount ?? totalLines - startLine + 1;
-          const endLine = Math.min(startLine + count - 1, totalLines);
-
-          const slicedLines = contentLines.slice(startLine - 1, endLine);
-
-          if (showLineNumbers) {
-            content = slicedLines
-              .map((line, i) => `${startLine + i}: ${line}`)
-              .join("\n");
+        const fullContent = contentResult.value ?? "";
+        const contentLines = fullContent.split("\n");
+        const totalLines = contentLines.length;
+
+        // Apply line range if specified
+        let content = fullContent;
+        let returnedLines: { start: number; end: number } | undefined;
+
+        // lineNumbers defaults to true per spec
+        const showLineNumbers = args.lineNumbers !== false;
+
+        if (args.fromLine || args.lineCount) {
+          const startLine = args.fromLine ?? 1;
+          // Clamp startLine to valid range
+          if (startLine > totalLines) {
+            // Return empty content for out-of-range request
+            content = "";
+            returnedLines = undefined;
           } else {
-            content = slicedLines.join("\n");
-          }
+            const count = args.lineCount ?? totalLines - startLine + 1;
+            const endLine = Math.min(startLine + count - 1, totalLines);
 
-          returnedLines = { start: startLine, end: endLine };
+            const slicedLines = contentLines.slice(startLine - 1, endLine);
+
+            if (showLineNumbers) {
+              content = slicedLines
+                .map((line, i) => `${startLine + i}: ${line}`)
+                .join("\n");
+            } else {
+              content = slicedLines.join("\n");
+            }
+
+            returnedLines = { start: startLine, end: endLine };
+          }
+        } else if (showLineNumbers) {
+          content = contentLines
+            .map((line, i) => `${i + 1}: ${line}`)
+            .join("\n");
         }
-      } else if (showLineNumbers) {
-        content = contentLines.map((line, i) => `${i + 1}: ${line}`).join("\n");
-      }
 
-      // Build absPath
-      const uriParsed = parseUri(doc.uri);
-      let absPath: string | undefined;
-      if (uriParsed) {
-        const collection = ctx.collections.find(
-          (c) => c.name === uriParsed.collection
-        );
-        if (collection) {
-          absPath = pathJoin(collection.path, doc.relPath);
+        // Build absPath
+        const uriParsed = parseUri(doc.uri);
+        let absPath: string | undefined;
+        if (uriParsed) {
+          const collection = ctx.collections.find(
+            (c) => c.name === uriParsed.collection
+          );
+          if (collection) {
+            absPath = pathJoin(collection.path, doc.relPath);
+          }
         }
-      }
 
-      const response: GetResponse = {
-        docid: doc.docid,
-        uri: decorateUriForIndex(
-          doc.uri,
-          parsed.type === "uri"
-            ? (parseUri(parsed.value)?.indexName ?? ctx.indexName)
-            : ctx.indexName
-        ),
-        title: doc.title ?? undefined,
-        content,
-        totalLines,
-        returnedLines,
-        language: doc.languageHint ?? undefined,
-        source: {
-          absPath,
-          relPath: doc.relPath,
-          mime: doc.sourceMime,
-          ext: doc.sourceExt,
-          modifiedAt: doc.sourceMtime,
-          sizeBytes: doc.sourceSize,
-          sourceHash: doc.sourceHash,
-        },
-        conversion: doc.mirrorHash
-          ? {
-              converterId: doc.converterId ?? undefined,
-              converterVersion: doc.converterVersion ?? undefined,
-              mirrorHash: doc.mirrorHash,
-            }
-          : undefined,
-        capabilities: getDocumentCapabilities({
-          sourceExt: doc.sourceExt,
-          sourceMime: doc.sourceMime,
-          contentAvailable: doc.mirrorHash !== null,
-        }),
-      };
-
-      return response;
+        const response: GetResponse = {
+          docid: doc.docid,
+          uri: decorateUriForIndex(doc.uri, scoped.indexName),
+          title: doc.title ?? undefined,
+          content,
+          totalLines,
+          returnedLines,
+          language: doc.languageHint ?? undefined,
+          source: {
+            absPath,
+            relPath: doc.relPath,
+            mime: doc.sourceMime,
+            ext: doc.sourceExt,
+            modifiedAt: doc.sourceMtime,
+            sizeBytes: doc.sourceSize,
+            sourceHash: doc.sourceHash,
+          },
+          conversion: doc.mirrorHash
+            ? {
+                converterId: doc.converterId ?? undefined,
+                converterVersion: doc.converterVersion ?? undefined,
+                mirrorHash: doc.mirrorHash,
+              }
+            : undefined,
+          capabilities: getDocumentCapabilities({
+            sourceExt: doc.sourceExt,
+            sourceMime: doc.sourceMime,
+            contentAvailable: doc.mirrorHash !== null,
+          }),
+        };
+
+        return response;
+      } finally {
+        await scoped.close();
+      }
     },
     formatGetResponse
   );

package/src/mcp/tools/index.ts

@@ -60,11 +60,11 @@ export function normalizeTagFilters(tags?: string[]): string[] | undefined {
 
 export const MCP_TOOL_DESCRIPTIONS = {
   search:
-    "BM25 keyword search. Fast exact-term lookup for names, identifiers, error text, and known phrases. Results include uri/docid and line when available; use gno_get with fromLine/lineCount or gno_multi_get for full context. Use gno_query when wording is uncertain.",
+    "BM25 keyword search. Fast exact-term lookup for names, identifiers, error text, and known phrases. Structured results include uri/docid, line when available, and optional user-configured context guidance; use gno_get with fromLine/lineCount or gno_multi_get for full context. Use gno_query when wording is uncertain.",
   vsearch:
-    "Vector semantic search. Finds conceptually similar docs with different wording. Best after embeddings are current; use intent to disambiguate short terms. Use gno_query for default hybrid retrieval.",
+    "Vector semantic search. Finds conceptually similar docs with different wording. Structured results preserve optional user-configured context guidance. Best after embeddings are current; use intent to disambiguate short terms. Use gno_query for default hybrid retrieval.",
   query:
-    "Hybrid search (BM25 + vector + optional expansion/reranking). Recommended default. Use intent for ambiguous terms, queryModes to combine term/intent/hyde strategies, fast=true for quick lookup, thorough=true when recall matters, and candidateLimit to trade latency for coverage.",
+    "Hybrid search (BM25 + vector + optional expansion/reranking). Recommended default. Structured results preserve optional user-configured context guidance with source identity. Use intent for ambiguous terms, queryModes to combine term/intent/hyde strategies, fast=true for quick lookup, thorough=true when recall matters, and candidateLimit to trade latency for coverage.",
   queryDiagnose:
     "Diagnose why one target document does or does not appear for a query. Use when an important doc is missing, a filter may exclude it, or you need stage-by-stage BM25/vector/fusion/graph/rerank evidence before changing retrieval strategy.",
   get: "Retrieve one document by gno:// URI, docid (#abc123), or collection/path. After search results include line, pass fromLine and lineCount to fetch only the relevant range before expanding to the full document.",