@gmickel/gno 2.12.2 → 2.14.0

package/src/core/reindex-paths.ts

@@ -0,0 +1,167 @@
+/**
+ * Validate caller-supplied paths for a path-level reindex.
+ *
+ * @module src/core/reindex-paths
+ */
+
+// node:fs/promises realpath, lstat/stat (symlinks, directories) and node:path
+// have no Bun equivalent.
+import { lstat, readlink, realpath, stat } from "node:fs/promises";
+import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
+
+import type { Collection } from "../config/types";
+
+import { collectionToWalkConfig } from "../ingestion/types";
+import { matchesWalkPath } from "../ingestion/walker";
+import { MCP_ERRORS } from "./errors";
+
+const escapes = (rel: string): boolean =>
+  rel === "" || rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel);
+
+const toPosix = (rel: string): string => rel.split(sep).join("/");
+
+const missing = (error: unknown): boolean =>
+  error !== null &&
+  typeof error === "object" &&
+  "code" in error &&
+  (error.code === "ENOENT" || error.code === "ENOTDIR");
+
+const MAX_SYMLINK_HOPS = 32;
+
+/**
+ * Most paths one request may list, and most a queued job keeps after merging.
+ * Past it, reindexing the whole collection is the better call.
+ */
+export const MAX_REINDEX_PATHS = 256;
+
+/**
+ * Realpath of `path`, or of its nearest existing ancestor below `root`. A
+ * dangling symlink on the way resolves to where it points, so a missing file
+ * cannot leave the root through a link whose target does not exist.
+ */
+async function nearestRealpath(
+  path: string,
+  root: string,
+  hops = 0
+): Promise<string | null> {
+  let current = path;
+  for (;;) {
+    try {
+      return await realpath(current);
+    } catch (error) {
+      if (!missing(error)) throw error;
+    }
+    const isLink = await lstat(current).then(
+      (stats) => stats.isSymbolicLink(),
+      () => false
+    );
+    if (isLink) {
+      const target = resolve(dirname(current), await readlink(current));
+      if (hops >= MAX_SYMLINK_HOPS) return target;
+      return (await nearestRealpath(target, root, hops + 1)) ?? target;
+    }
+    const parent = dirname(current);
+    if (parent === current || relative(root, parent) === "") return null;
+    current = parent;
+  }
+}
+
+/**
+ * Resolve paths (relative to the collection root, or absolute inside it) to
+ * deduplicated collection-relative POSIX paths. A path that no longer exists
+ * is kept so the sync can mark it inactive. Throws `INVALID_INPUT: ...` for an
+ * empty list or more than `MAX_REINDEX_PATHS` paths, and `INVALID_PATH: ...`
+ * for a directory or a path outside the root, including one that leaves it
+ * through a symlink.
+ */
+export async function resolveReindexPaths(
+  collection: Collection,
+  paths: readonly unknown[]
+): Promise<string[]> {
+  if (paths.length === 0) {
+    throw new Error(
+      `${MCP_ERRORS.INVALID_INPUT.code}: paths must list at least one path`
+    );
+  }
+  if (paths.length > MAX_REINDEX_PATHS) {
+    throw new Error(
+      `${MCP_ERRORS.INVALID_INPUT.code}: paths lists ${paths.length} paths (at most ${MAX_REINDEX_PATHS}); reindex the whole collection instead`
+    );
+  }
+  const root = resolve(collection.path);
+  let realRoot: string;
+  try {
+    realRoot = await realpath(root);
+  } catch {
+    realRoot = root;
+  }
+  const resolved = new Set<string>();
+  for (const path of paths) {
+    if (typeof path !== "string" || path.trim() === "") {
+      throw new Error(
+        `${MCP_ERRORS.INVALID_INPUT.code}: paths must be non-empty strings`
+      );
+    }
+    const absolute = resolve(root, path);
+    const rel = relative(root, absolute);
+    if (escapes(rel)) {
+      throw new Error(
+        `${MCP_ERRORS.INVALID_PATH.code}: ${path} is outside collection ${collection.name}`
+      );
+    }
+    // A missing file is checked through its nearest existing folder, so a
+    // deletion path cannot leave the root through a symlinked parent.
+    const real = await nearestRealpath(absolute, root);
+    if (
+      real !== null &&
+      escapes(relative(realRoot, real)) &&
+      real !== realRoot
+    ) {
+      throw new Error(
+        `${MCP_ERRORS.INVALID_PATH.code}: ${path} resolves outside collection ${collection.name}`
+      );
+    }
+    const isDirectory = await stat(absolute).then(
+      (stats) => stats.isDirectory(),
+      () => false
+    );
+    if (isDirectory) {
+      throw new Error(
+        `${MCP_ERRORS.INVALID_PATH.code}: ${path} is a directory; list the files in it, or reindex the whole collection`
+      );
+    }
+    // A listed path obeys the collection's own pattern, include and exclude
+    // rules: an excluded file must not become searchable by naming it.
+    if (!matchesWalkPath(toPosix(rel), collectionToWalkConfig(collection, 0))) {
+      throw new Error(
+        `${MCP_ERRORS.INVALID_PATH.code}: ${path} is not part of collection ${collection.name} (its pattern, include or exclude rules leave it out)`
+      );
+    }
+    resolved.add(toPosix(rel));
+  }
+  return [...resolved];
+}
+
+/**
+ * Resolve an optional path list for a reindex request. Paths need a single
+ * collection and apply to a plain sync: git pull and update commands act on
+ * the whole collection, so they cannot be combined with paths.
+ */
+export async function resolveReindexScope(
+  collection: Collection | null,
+  paths: readonly unknown[] | undefined,
+  options: { gitPull?: boolean; runUpdateCmd?: boolean } = {}
+): Promise<string[] | null> {
+  if (paths === undefined) return null;
+  if (!collection) {
+    throw new Error(
+      `${MCP_ERRORS.INVALID_INPUT.code}: paths require a collection`
+    );
+  }
+  if (options.gitPull || options.runUpdateCmd) {
+    throw new Error(
+      `${MCP_ERRORS.INVALID_INPUT.code}: paths cannot be combined with gitPull or an update command`
+    );
+  }
+  return resolveReindexPaths(collection, paths);
+}

package/src/core/retrieval-trace-request.ts

@@ -117,6 +117,7 @@ export const retrievalTraceFilters = (
       limit: options.limit,
       minScore: options.minScore,
       collection: options.collection,
+      collections: options.collections,
       lang: options.lang,
       full: options.full,
       lineNumbers: options.lineNumbers,

package/src/ingestion/convert-file.ts

@@ -0,0 +1,181 @@
+/**
+ * Convert one file to the Markdown GNO would index for it, with nothing
+ * persisted (`gno convert`, SDK `convertFile`).
+ *
+ * The file goes through the indexing path: extension MIME detection and the
+ * record-adapter check, then a sniffed MIME and `prepareFile` in the
+ * terminable file processor under the default time and memory budget. The
+ * Markdown and mirror hash therefore equal what `gno update` stores for the
+ * same bytes. No config, index, store or cache is read or written.
+ *
+ * @module src/ingestion/convert-file
+ */
+
+// node:fs/promises for stat (no Bun equivalent for file stats)
+import { stat } from "node:fs/promises";
+// node:path for basename/resolve (no Bun path utils)
+import { basename, resolve } from "node:path";
+
+import { getDefaultMimeDetector } from "../converters/mime";
+import { markdownConverter } from "../converters/native/markdown";
+import { getDefaultPipeline } from "../converters/pipeline";
+import { DEFAULT_LIMITS } from "../converters/types";
+import {
+  defaultConversionMemoryMb,
+  prepareInProcessor,
+} from "./file-processor";
+import { DEFAULT_CHUNK_PARAMS } from "./types";
+
+export const CONVERT_SCHEMA_VERSION = "convert@1.0";
+
+const BYTES_PER_MB = 1_048_576;
+const SNIFF_BYTES = 512;
+
+export interface ConvertFileResult {
+  schemaVersion: typeof CONVERT_SCHEMA_VERSION;
+  source: { path: string; mime: string; ext: string; sizeBytes: number };
+  converter: { id: string; version: string };
+  title: string | null;
+  languageHint: string | null;
+  markdown: string;
+  mirrorHash: string;
+  warnings: { code: string; message: string }[];
+}
+
+/**
+ * Why a file was not converted. `validation` failures (missing file, over
+ * the size limit, no converter, multi-record format) exit 1 on the CLI;
+ * converter and processor failures exit 2.
+ */
+export interface ConvertFileFailure {
+  reason: string;
+  message: string;
+  validation: boolean;
+  details?: Record<string, unknown>;
+}
+
+export type ConvertFileOutcome =
+  | { ok: true; value: ConvertFileResult }
+  | { ok: false; error: ConvertFileFailure };
+
+const fail = (
+  reason: string,
+  message: string,
+  validation: boolean,
+  details?: Record<string, unknown>
+): ConvertFileOutcome => ({
+  ok: false,
+  error: { reason, message, validation, ...(details ? { details } : {}) },
+});
+
+/** Convert one file; see the module comment. */
+export async function convertFile(path: string): Promise<ConvertFileOutcome> {
+  const absPath = resolve(path);
+  let sizeBytes: number;
+  try {
+    // stat follows symlinks: a link to a regular file converts, a link to
+    // a directory does not.
+    const stats = await stat(absPath);
+    if (!stats.isFile()) {
+      return fail("NOT_FOUND", `Not a regular file: ${path}`, true);
+    }
+    sizeBytes = stats.size;
+  } catch {
+    return fail("NOT_FOUND", `File not found: ${path}`, true);
+  }
+
+  // Refused before reading, so an oversized file is never loaded.
+  if (sizeBytes > DEFAULT_LIMITS.maxBytes) {
+    return fail(
+      "TOO_LARGE",
+      `File size ${sizeBytes} exceeds the indexing limit ${DEFAULT_LIMITS.maxBytes}`,
+      true,
+      { sizeBytes, limitBytes: DEFAULT_LIMITS.maxBytes }
+    );
+  }
+
+  const detector = getDefaultMimeDetector();
+  const byExtension = detector.detect(absPath, new Uint8Array());
+  const recordAdapter = await getDefaultPipeline().selectRecordAdapter(
+    byExtension.mime,
+    byExtension.ext
+  );
+  if (recordAdapter) {
+    return fail(
+      "MULTI_DOCUMENT",
+      `${recordAdapter.id} files hold several records; index them in a collection instead`,
+      true,
+      { format: recordAdapter.id }
+    );
+  }
+
+  let bytes: Uint8Array;
+  try {
+    bytes = await Bun.file(absPath).bytes();
+  } catch (error) {
+    // Readable at stat but not at read: removed meanwhile, or no permission.
+    const code = (error as { code?: unknown } | null)?.code;
+    if (code === "ENOENT") {
+      return fail("NOT_FOUND", `File not found: ${path}`, true);
+    }
+    const denied = code === "EACCES" || code === "EPERM";
+    return fail(
+      denied ? "PERMISSION" : "IO",
+      `Cannot read ${path}: ${error instanceof Error ? error.message : String(error)}`,
+      false
+    );
+  }
+  const { mime, ext } = detector.detect(
+    absPath,
+    bytes.subarray(0, SNIFF_BYTES)
+  );
+  const prepared = await prepareInProcessor(
+    {
+      input: {
+        sourcePath: absPath,
+        relativePath: basename(absPath),
+        collection: "",
+        bytes,
+        mime,
+        ext,
+        limits: { ...DEFAULT_LIMITS },
+      },
+      metadataExt: ext,
+      contentTypeRules: [],
+      chunkParams: DEFAULT_CHUNK_PARAMS,
+      memoryManaged: false,
+      markdownSource: markdownConverter.canHandle(mime, ext),
+      previous: null,
+    },
+    {
+      timeoutMs: DEFAULT_LIMITS.timeoutMs,
+      maxMemoryBytes: defaultConversionMemoryMb() * BYTES_PER_MB,
+    },
+    { onPrevious: () => undefined }
+  );
+  if (!prepared.ok) {
+    const { code, message, details } = prepared.error;
+    return fail(code, message, code === "UNSUPPORTED", details);
+  }
+
+  const { artifact } = prepared.value;
+  return {
+    ok: true,
+    value: {
+      schemaVersion: CONVERT_SCHEMA_VERSION,
+      source: { path: absPath, mime, ext, sizeBytes },
+      converter: {
+        id: artifact.meta.converterId,
+        version: artifact.meta.converterVersion,
+      },
+      title: artifact.title ?? null,
+      languageHint: artifact.languageHint ?? null,
+      markdown: artifact.markdown,
+      mirrorHash: artifact.mirrorHash,
+      warnings: (artifact.meta.warnings ?? []).map(({ code, message }) => ({
+        code,
+        message,
+      })),
+    },
+  };
+}

package/src/llm/registry.ts

@@ -134,6 +134,26 @@ export function resolveModelUri(
   return preset[type];
 }
 
+/**
+ * Collection to resolve models for when a request lists several collections:
+ * the first listed one when every listed collection resolves `type` to the
+ * same model, otherwise undefined (the unscoped resolution).
+ */
+export function modelCollectionForList(
+  config: Config,
+  type: ModelType,
+  collections: readonly string[] | undefined
+): string | undefined {
+  const [first] = collections ?? [];
+  if (first === undefined) return undefined;
+  const uri = resolveModelUri(config, type, undefined, first);
+  return collections?.every(
+    (collection) => resolveModelUri(config, type, undefined, collection) === uri
+  )
+    ? first
+    : undefined;
+}
+
 export function resolveModelSource(
   config: Config,
   type: ModelType,

package/src/mcp/http-egress.ts

@@ -140,6 +140,19 @@ const GRAPH_RESULT_TOOLS = new Set([
   "gno_impact",
 ]);
 
+/**
+ * Tools whose `collections` argument bounds the collections they read. The
+ * retrieval tools reject `collection` with `collections`, so the union of both
+ * is never wider than what the handler serves.
+ */
+const COLLECTIONS_SCOPE_TOOLS = new Set([
+  "gno_audit",
+  "gno_impact",
+  "gno_query",
+  "gno_search",
+  "gno_vsearch",
+]);
+
 const requestedCollections = (
   params: unknown,
   collections: readonly Collection[]
@@ -155,7 +168,8 @@ const requestedCollections = (
   if (typeof direct === "string" && direct.trim())
     names.add(direct.trim().toLowerCase());
   if (
-    (record?.name === "gno_audit" || record?.name === "gno_impact") &&
+    typeof record?.name === "string" &&
+    COLLECTIONS_SCOPE_TOOLS.has(record.name) &&
     Array.isArray(args.collections)
   ) {
     for (const value of args.collections) {
@@ -192,7 +206,12 @@ const requestedCollections = (
       if (collection) names.add(collection);
     }
   }
-  return names.size > 0 ? [...names] : collections.map(({ name }) => name);
+  // A name that is not configured serves nothing (the handler refuses it), so
+  // it never narrows or widens the check; with no configured name left, every
+  // collection is checked (fail closed).
+  const configured = new Set(collections.map(({ name }) => name.toLowerCase()));
+  const scoped = [...names].filter((name) => configured.has(name));
+  return scoped.length > 0 ? scoped : collections.map(({ name }) => name);
 };
 
 const enforceMessage = (

package/src/mcp/tools/collection-scope.ts

@@ -0,0 +1,25 @@
+/**
+ * MCP adapter for the shared request collection allowlist.
+ *
+ * @module src/mcp/tools/collection-scope
+ */
+
+import type { ToolContext } from "../server";
+
+import { resolveCollectionScope } from "../../core/collection-scope";
+
+/**
+ * Resolve `collections` for gno_search, gno_vsearch and gno_query. Throws a
+ * `VALIDATION:` error for malformed input or `collection` plus `collections`,
+ * and the existing unknown-collection error for names not configured.
+ */
+export function requestCollections(
+  args: { collection?: string; collections?: string[] },
+  ctx: Pick<ToolContext, "collections">
+): string[] | undefined {
+  const scope = resolveCollectionScope(args, ctx.collections);
+  if (scope.ok) return scope.collections;
+  throw new Error(
+    scope.kind === "validation" ? `VALIDATION: ${scope.message}` : scope.message
+  );
+}

package/src/mcp/tools/index-cmd.ts

@@ -8,12 +8,12 @@ import type { Collection } from "../../config/types";
 import type { ToolContext } from "../server";
 
 import { MCP_ERRORS } from "../../core/errors";
-import { acquireWriteLock, type WriteLockHandle } from "../../core/file-lock";
 import { JobError } from "../../core/job-manager";
 import {
   recordContentMutation,
   recordIndexMutation,
 } from "../../core/mutation-generations";
+import { resolveReindexScope } from "../../core/reindex-paths";
 import { normalizeCollectionName } from "../../core/validation";
 import { embedBacklog } from "../../embed";
 import { defaultSyncService, withContentTypeRules } from "../../ingestion";
@@ -29,12 +29,14 @@ import { runTool, type ToolResult } from "./index";
 interface IndexInput {
   collection?: string;
   gitPull?: boolean;
+  paths?: string[];
 }
 
 interface IndexResultOutput {
   jobId: string;
-  status: "started";
+  status: "started" | "queued";
   collections: string[];
+  paths?: string[];
   phases: ["sync", "embed"];
   options: {
     gitPull: boolean;
@@ -47,6 +49,7 @@ function formatIndexResult(result: IndexResultOutput): string {
   lines.push(`Job: ${result.jobId}`);
   lines.push(`Status: ${result.status}`);
   lines.push(`Collections: ${result.collections.join(", ") || "(none)"}`);
+  if (result.paths) lines.push(`Paths: ${result.paths.join(", ")}`);
   lines.push(`Phases: ${result.phases.join(" → ")}`);
   lines.push(`Git pull: ${result.options.gitPull ? "yes" : "no"}`);
   return lines.join("\n");
@@ -74,17 +77,7 @@ export function handleIndex(
         throw new Error("Write tools disabled. Start MCP with --enable-write.");
       }
 
-      let lock: WriteLockHandle | null = null;
-      let handedOff = false;
-
       try {
-        lock = await acquireWriteLock(ctx.writeLockPath);
-        if (!lock) {
-          throw new Error(
-            `${MCP_ERRORS.LOCKED.code}: ${MCP_ERRORS.LOCKED.message}`
-          );
-        }
-
         const requestedCollection = args.collection?.trim();
         const collection = requestedCollection
           ? resolveCollection(requestedCollection, ctx.collections)
@@ -108,47 +101,77 @@ export function handleIndex(
           },
           ctx.config
         );
+        const paths = await resolveReindexScope(
+          collection,
+          args.paths,
+          options
+        );
         enforceSyncCommandEgress(ctx, {
           collectionNames: collections,
           gitPull: options.gitPull,
           runUpdateCmd: options.runUpdateCmd,
         });
 
-        const modelUri = resolveModelUri(
-          ctx.config,
-          "embed",
-          undefined,
-          collection?.name
-        );
-
-        const jobId = await ctx.jobManager.startTypedJobWithLock(
-          "index",
-          lock,
-          async () => {
+        // Captured now: a queued pass runs after this call's context is gone.
+        const config = ctx.config;
+        const allCollections = ctx.collections;
+        const accepted = await ctx.jobManager.submitReindex({
+          type: "index",
+          collection: collection?.name ?? null,
+          paths,
+          options: {
+            gitPull: options.gitPull ?? false,
+            runUpdateCmd: options.runUpdateCmd ?? false,
+          },
+          run: async (scope) => {
             const lease = ctx.acquireModelLease?.();
+            const passOptions = withContentTypeRules(
+              { ...scope.options },
+              config
+            );
             try {
               // Phase 1: Sync
               const syncResult = collection
-                ? await defaultSyncService
-                    .syncCollection(collection, ctx.store, options)
-                    .then((r) => ({
-                      collections: [r],
-                      totalDurationMs: r.durationMs,
-                      totalFilesProcessed: r.filesProcessed,
-                      totalFilesAdded: r.filesAdded,
-                      totalFilesUpdated: r.filesUpdated,
-                      totalFilesErrored: r.filesErrored,
-                      totalFilesSkipped: r.filesSkipped,
-                    }))
+                ? await (
+                    scope.paths
+                      ? defaultSyncService.syncPaths(
+                          collection,
+                          ctx.store,
+                          scope.paths,
+                          passOptions
+                        )
+                      : defaultSyncService.syncCollection(
+                          collection,
+                          ctx.store,
+                          passOptions
+                        )
+                  ).then((r) => ({
+                    collections: [r],
+                    totalDurationMs: r.durationMs,
+                    totalFilesProcessed: r.filesProcessed,
+                    totalFilesAdded: r.filesAdded,
+                    totalFilesUpdated: r.filesUpdated,
+                    totalFilesErrored: r.filesErrored,
+                    totalFilesSkipped: r.filesSkipped,
+                  }))
                 : await defaultSyncService.syncAll(
-                    ctx.collections,
+                    allCollections,
                     ctx.store,
-                    options
+                    passOptions
                   );
               recordContentMutation(syncResult, ctx.markContentMutation);
 
-              // Phase 2: Embed
-              const llm = ctx.getModelAdapter?.() ?? new LlmAdapter(ctx.config);
+              // Phase 2: Embed. The model is looked up when the pass runs: a
+              // queued pass outlives a server reload, which disposes the
+              // adapter that was current at submit.
+              const liveConfig = ctx.config;
+              const modelUri = resolveModelUri(
+                liveConfig,
+                "embed",
+                undefined,
+                collection?.name
+              );
+              const llm = ctx.getModelAdapter?.() ?? new LlmAdapter(liveConfig);
               const embedResult = await llm.createEmbeddingPort(modelUri, {
                 egressCollections: collections,
                 policy: { offline: true, allowDownload: false },
@@ -217,15 +240,14 @@ export function handleIndex(
             } finally {
               lease?.release();
             }
-          }
-        );
-
-        handedOff = true;
+          },
+        });
 
         const result: IndexResultOutput = {
-          jobId,
+          jobId: accepted.jobId,
           collections,
-          status: "started",
+          status: accepted.status === "running" ? "started" : "queued",
+          ...(paths ? { paths } : {}),
           phases: ["sync", "embed"],
           options: {
             gitPull: options.gitPull ?? false,
@@ -239,10 +261,6 @@ export function handleIndex(
           throw new Error(`${error.code}: ${error.message}`);
         }
         throw error;
-      } finally {
-        if (lock && !handedOff) {
-          await lock.release();
-        }
       }
     },
     formatIndexResult