@gmickel/gno 2.12.2 → 2.13.0

package/src/ingestion/convert-file.ts

@@ -0,0 +1,181 @@
+/**
+ * Convert one file to the Markdown GNO would index for it, with nothing
+ * persisted (`gno convert`, SDK `convertFile`).
+ *
+ * The file goes through the indexing path: extension MIME detection and the
+ * record-adapter check, then a sniffed MIME and `prepareFile` in the
+ * terminable file processor under the default time and memory budget. The
+ * Markdown and mirror hash therefore equal what `gno update` stores for the
+ * same bytes. No config, index, store or cache is read or written.
+ *
+ * @module src/ingestion/convert-file
+ */
+
+// node:fs/promises for stat (no Bun equivalent for file stats)
+import { stat } from "node:fs/promises";
+// node:path for basename/resolve (no Bun path utils)
+import { basename, resolve } from "node:path";
+
+import { getDefaultMimeDetector } from "../converters/mime";
+import { markdownConverter } from "../converters/native/markdown";
+import { getDefaultPipeline } from "../converters/pipeline";
+import { DEFAULT_LIMITS } from "../converters/types";
+import {
+  defaultConversionMemoryMb,
+  prepareInProcessor,
+} from "./file-processor";
+import { DEFAULT_CHUNK_PARAMS } from "./types";
+
+export const CONVERT_SCHEMA_VERSION = "convert@1.0";
+
+const BYTES_PER_MB = 1_048_576;
+const SNIFF_BYTES = 512;
+
+export interface ConvertFileResult {
+  schemaVersion: typeof CONVERT_SCHEMA_VERSION;
+  source: { path: string; mime: string; ext: string; sizeBytes: number };
+  converter: { id: string; version: string };
+  title: string | null;
+  languageHint: string | null;
+  markdown: string;
+  mirrorHash: string;
+  warnings: { code: string; message: string }[];
+}
+
+/**
+ * Why a file was not converted. `validation` failures (missing file, over
+ * the size limit, no converter, multi-record format) exit 1 on the CLI;
+ * converter and processor failures exit 2.
+ */
+export interface ConvertFileFailure {
+  reason: string;
+  message: string;
+  validation: boolean;
+  details?: Record<string, unknown>;
+}
+
+export type ConvertFileOutcome =
+  | { ok: true; value: ConvertFileResult }
+  | { ok: false; error: ConvertFileFailure };
+
+const fail = (
+  reason: string,
+  message: string,
+  validation: boolean,
+  details?: Record<string, unknown>
+): ConvertFileOutcome => ({
+  ok: false,
+  error: { reason, message, validation, ...(details ? { details } : {}) },
+});
+
+/** Convert one file; see the module comment. */
+export async function convertFile(path: string): Promise<ConvertFileOutcome> {
+  const absPath = resolve(path);
+  let sizeBytes: number;
+  try {
+    // stat follows symlinks: a link to a regular file converts, a link to
+    // a directory does not.
+    const stats = await stat(absPath);
+    if (!stats.isFile()) {
+      return fail("NOT_FOUND", `Not a regular file: ${path}`, true);
+    }
+    sizeBytes = stats.size;
+  } catch {
+    return fail("NOT_FOUND", `File not found: ${path}`, true);
+  }
+
+  // Refused before reading, so an oversized file is never loaded.
+  if (sizeBytes > DEFAULT_LIMITS.maxBytes) {
+    return fail(
+      "TOO_LARGE",
+      `File size ${sizeBytes} exceeds the indexing limit ${DEFAULT_LIMITS.maxBytes}`,
+      true,
+      { sizeBytes, limitBytes: DEFAULT_LIMITS.maxBytes }
+    );
+  }
+
+  const detector = getDefaultMimeDetector();
+  const byExtension = detector.detect(absPath, new Uint8Array());
+  const recordAdapter = await getDefaultPipeline().selectRecordAdapter(
+    byExtension.mime,
+    byExtension.ext
+  );
+  if (recordAdapter) {
+    return fail(
+      "MULTI_DOCUMENT",
+      `${recordAdapter.id} files hold several records; index them in a collection instead`,
+      true,
+      { format: recordAdapter.id }
+    );
+  }
+
+  let bytes: Uint8Array;
+  try {
+    bytes = await Bun.file(absPath).bytes();
+  } catch (error) {
+    // Readable at stat but not at read: removed meanwhile, or no permission.
+    const code = (error as { code?: unknown } | null)?.code;
+    if (code === "ENOENT") {
+      return fail("NOT_FOUND", `File not found: ${path}`, true);
+    }
+    const denied = code === "EACCES" || code === "EPERM";
+    return fail(
+      denied ? "PERMISSION" : "IO",
+      `Cannot read ${path}: ${error instanceof Error ? error.message : String(error)}`,
+      false
+    );
+  }
+  const { mime, ext } = detector.detect(
+    absPath,
+    bytes.subarray(0, SNIFF_BYTES)
+  );
+  const prepared = await prepareInProcessor(
+    {
+      input: {
+        sourcePath: absPath,
+        relativePath: basename(absPath),
+        collection: "",
+        bytes,
+        mime,
+        ext,
+        limits: { ...DEFAULT_LIMITS },
+      },
+      metadataExt: ext,
+      contentTypeRules: [],
+      chunkParams: DEFAULT_CHUNK_PARAMS,
+      memoryManaged: false,
+      markdownSource: markdownConverter.canHandle(mime, ext),
+      previous: null,
+    },
+    {
+      timeoutMs: DEFAULT_LIMITS.timeoutMs,
+      maxMemoryBytes: defaultConversionMemoryMb() * BYTES_PER_MB,
+    },
+    { onPrevious: () => undefined }
+  );
+  if (!prepared.ok) {
+    const { code, message, details } = prepared.error;
+    return fail(code, message, code === "UNSUPPORTED", details);
+  }
+
+  const { artifact } = prepared.value;
+  return {
+    ok: true,
+    value: {
+      schemaVersion: CONVERT_SCHEMA_VERSION,
+      source: { path: absPath, mime, ext, sizeBytes },
+      converter: {
+        id: artifact.meta.converterId,
+        version: artifact.meta.converterVersion,
+      },
+      title: artifact.title ?? null,
+      languageHint: artifact.languageHint ?? null,
+      markdown: artifact.markdown,
+      mirrorHash: artifact.mirrorHash,
+      warnings: (artifact.meta.warnings ?? []).map(({ code, message }) => ({
+        code,
+        message,
+      })),
+    },
+  };
+}

package/src/llm/registry.ts

@@ -134,6 +134,26 @@ export function resolveModelUri(
   return preset[type];
 }
 
+/**
+ * Collection to resolve models for when a request lists several collections:
+ * the first listed one when every listed collection resolves `type` to the
+ * same model, otherwise undefined (the unscoped resolution).
+ */
+export function modelCollectionForList(
+  config: Config,
+  type: ModelType,
+  collections: readonly string[] | undefined
+): string | undefined {
+  const [first] = collections ?? [];
+  if (first === undefined) return undefined;
+  const uri = resolveModelUri(config, type, undefined, first);
+  return collections?.every(
+    (collection) => resolveModelUri(config, type, undefined, collection) === uri
+  )
+    ? first
+    : undefined;
+}
+
 export function resolveModelSource(
   config: Config,
   type: ModelType,

package/src/mcp/http-egress.ts

@@ -140,6 +140,19 @@ const GRAPH_RESULT_TOOLS = new Set([
   "gno_impact",
 ]);
 
+/**
+ * Tools whose `collections` argument bounds the collections they read. The
+ * retrieval tools reject `collection` with `collections`, so the union of both
+ * is never wider than what the handler serves.
+ */
+const COLLECTIONS_SCOPE_TOOLS = new Set([
+  "gno_audit",
+  "gno_impact",
+  "gno_query",
+  "gno_search",
+  "gno_vsearch",
+]);
+
 const requestedCollections = (
   params: unknown,
   collections: readonly Collection[]
@@ -155,7 +168,8 @@ const requestedCollections = (
   if (typeof direct === "string" && direct.trim())
     names.add(direct.trim().toLowerCase());
   if (
-    (record?.name === "gno_audit" || record?.name === "gno_impact") &&
+    typeof record?.name === "string" &&
+    COLLECTIONS_SCOPE_TOOLS.has(record.name) &&
     Array.isArray(args.collections)
   ) {
     for (const value of args.collections) {
@@ -192,7 +206,12 @@ const requestedCollections = (
       if (collection) names.add(collection);
     }
   }
-  return names.size > 0 ? [...names] : collections.map(({ name }) => name);
+  // A name that is not configured serves nothing (the handler refuses it), so
+  // it never narrows or widens the check; with no configured name left, every
+  // collection is checked (fail closed).
+  const configured = new Set(collections.map(({ name }) => name.toLowerCase()));
+  const scoped = [...names].filter((name) => configured.has(name));
+  return scoped.length > 0 ? scoped : collections.map(({ name }) => name);
 };
 
 const enforceMessage = (

package/src/mcp/tools/collection-scope.ts

@@ -0,0 +1,25 @@
+/**
+ * MCP adapter for the shared request collection allowlist.
+ *
+ * @module src/mcp/tools/collection-scope
+ */
+
+import type { ToolContext } from "../server";
+
+import { resolveCollectionScope } from "../../core/collection-scope";
+
+/**
+ * Resolve `collections` for gno_search, gno_vsearch and gno_query. Throws a
+ * `VALIDATION:` error for malformed input or `collection` plus `collections`,
+ * and the existing unknown-collection error for names not configured.
+ */
+export function requestCollections(
+  args: { collection?: string; collections?: string[] },
+  ctx: Pick<ToolContext, "collections">
+): string[] | undefined {
+  const scope = resolveCollectionScope(args, ctx.collections);
+  if (scope.ok) return scope.collections;
+  throw new Error(
+    scope.kind === "validation" ? `VALIDATION: ${scope.message}` : scope.message
+  );
+}

package/src/mcp/tools/index-cmd.ts

@@ -8,12 +8,12 @@ import type { Collection } from "../../config/types";
 import type { ToolContext } from "../server";
 
 import { MCP_ERRORS } from "../../core/errors";
-import { acquireWriteLock, type WriteLockHandle } from "../../core/file-lock";
 import { JobError } from "../../core/job-manager";
 import {
   recordContentMutation,
   recordIndexMutation,
 } from "../../core/mutation-generations";
+import { resolveReindexScope } from "../../core/reindex-paths";
 import { normalizeCollectionName } from "../../core/validation";
 import { embedBacklog } from "../../embed";
 import { defaultSyncService, withContentTypeRules } from "../../ingestion";
@@ -29,12 +29,14 @@ import { runTool, type ToolResult } from "./index";
 interface IndexInput {
   collection?: string;
   gitPull?: boolean;
+  paths?: string[];
 }
 
 interface IndexResultOutput {
   jobId: string;
-  status: "started";
+  status: "started" | "queued";
   collections: string[];
+  paths?: string[];
   phases: ["sync", "embed"];
   options: {
     gitPull: boolean;
@@ -47,6 +49,7 @@ function formatIndexResult(result: IndexResultOutput): string {
   lines.push(`Job: ${result.jobId}`);
   lines.push(`Status: ${result.status}`);
   lines.push(`Collections: ${result.collections.join(", ") || "(none)"}`);
+  if (result.paths) lines.push(`Paths: ${result.paths.join(", ")}`);
   lines.push(`Phases: ${result.phases.join(" → ")}`);
   lines.push(`Git pull: ${result.options.gitPull ? "yes" : "no"}`);
   return lines.join("\n");
@@ -74,17 +77,7 @@ export function handleIndex(
         throw new Error("Write tools disabled. Start MCP with --enable-write.");
       }
 
-      let lock: WriteLockHandle | null = null;
-      let handedOff = false;
-
       try {
-        lock = await acquireWriteLock(ctx.writeLockPath);
-        if (!lock) {
-          throw new Error(
-            `${MCP_ERRORS.LOCKED.code}: ${MCP_ERRORS.LOCKED.message}`
-          );
-        }
-
         const requestedCollection = args.collection?.trim();
         const collection = requestedCollection
           ? resolveCollection(requestedCollection, ctx.collections)
@@ -108,47 +101,77 @@ export function handleIndex(
           },
           ctx.config
         );
+        const paths = await resolveReindexScope(
+          collection,
+          args.paths,
+          options
+        );
         enforceSyncCommandEgress(ctx, {
           collectionNames: collections,
           gitPull: options.gitPull,
           runUpdateCmd: options.runUpdateCmd,
         });
 
-        const modelUri = resolveModelUri(
-          ctx.config,
-          "embed",
-          undefined,
-          collection?.name
-        );
-
-        const jobId = await ctx.jobManager.startTypedJobWithLock(
-          "index",
-          lock,
-          async () => {
+        // Captured now: a queued pass runs after this call's context is gone.
+        const config = ctx.config;
+        const allCollections = ctx.collections;
+        const accepted = await ctx.jobManager.submitReindex({
+          type: "index",
+          collection: collection?.name ?? null,
+          paths,
+          options: {
+            gitPull: options.gitPull ?? false,
+            runUpdateCmd: options.runUpdateCmd ?? false,
+          },
+          run: async (scope) => {
             const lease = ctx.acquireModelLease?.();
+            const passOptions = withContentTypeRules(
+              { ...scope.options },
+              config
+            );
             try {
               // Phase 1: Sync
               const syncResult = collection
-                ? await defaultSyncService
-                    .syncCollection(collection, ctx.store, options)
-                    .then((r) => ({
-                      collections: [r],
-                      totalDurationMs: r.durationMs,
-                      totalFilesProcessed: r.filesProcessed,
-                      totalFilesAdded: r.filesAdded,
-                      totalFilesUpdated: r.filesUpdated,
-                      totalFilesErrored: r.filesErrored,
-                      totalFilesSkipped: r.filesSkipped,
-                    }))
+                ? await (
+                    scope.paths
+                      ? defaultSyncService.syncPaths(
+                          collection,
+                          ctx.store,
+                          scope.paths,
+                          passOptions
+                        )
+                      : defaultSyncService.syncCollection(
+                          collection,
+                          ctx.store,
+                          passOptions
+                        )
+                  ).then((r) => ({
+                    collections: [r],
+                    totalDurationMs: r.durationMs,
+                    totalFilesProcessed: r.filesProcessed,
+                    totalFilesAdded: r.filesAdded,
+                    totalFilesUpdated: r.filesUpdated,
+                    totalFilesErrored: r.filesErrored,
+                    totalFilesSkipped: r.filesSkipped,
+                  }))
                 : await defaultSyncService.syncAll(
-                    ctx.collections,
+                    allCollections,
                     ctx.store,
-                    options
+                    passOptions
                   );
               recordContentMutation(syncResult, ctx.markContentMutation);
 
-              // Phase 2: Embed
-              const llm = ctx.getModelAdapter?.() ?? new LlmAdapter(ctx.config);
+              // Phase 2: Embed. The model is looked up when the pass runs: a
+              // queued pass outlives a server reload, which disposes the
+              // adapter that was current at submit.
+              const liveConfig = ctx.config;
+              const modelUri = resolveModelUri(
+                liveConfig,
+                "embed",
+                undefined,
+                collection?.name
+              );
+              const llm = ctx.getModelAdapter?.() ?? new LlmAdapter(liveConfig);
               const embedResult = await llm.createEmbeddingPort(modelUri, {
                 egressCollections: collections,
                 policy: { offline: true, allowDownload: false },
@@ -217,15 +240,14 @@ export function handleIndex(
             } finally {
               lease?.release();
             }
-          }
-        );
-
-        handedOff = true;
+          },
+        });
 
         const result: IndexResultOutput = {
-          jobId,
+          jobId: accepted.jobId,
           collections,
-          status: "started",
+          status: accepted.status === "running" ? "started" : "queued",
+          ...(paths ? { paths } : {}),
           phases: ["sync", "embed"],
           options: {
             gitPull: options.gitPull ?? false,
@@ -239,10 +261,6 @@ export function handleIndex(
           throw new Error(`${error.code}: ${error.message}`);
         }
         throw error;
-      } finally {
-        if (lock && !handedOff) {
-          await lock.release();
-        }
       }
     },
     formatIndexResult

package/src/mcp/tools/index.ts

@@ -27,6 +27,7 @@ import {
   readMetadataSummary,
 } from "../../core/metadata-discovery";
 import { NOTE_PRESETS, type NotePresetId } from "../../core/note-presets";
+import { MAX_REINDEX_PATHS } from "../../core/reindex-paths";
 import { RETRIEVAL_TRACE_METADATA } from "../../core/retrieval-trace-session";
 import { normalizeTag } from "../../core/tags";
 import {
@@ -255,6 +256,12 @@ export const searchInputSchema = z.object({
     .string()
     .optional()
     .describe("Filter to a single collection name"),
+  collections: z
+    .array(z.string())
+    .optional()
+    .describe(
+      "Filter to any of these collection names (1-256); use instead of collection"
+    ),
   limit: z
     .number()
     .int()
@@ -437,6 +444,13 @@ const syncInputSchema = z.object({
     .boolean()
     .default(false)
     .describe("Run the collection's configured update command before syncing"),
+  paths: z
+    .array(z.string())
+    .max(MAX_REINDEX_PATHS)
+    .optional()
+    .describe(
+      "Reindex only these files (1-256, relative to the collection root, or absolute inside it). Requires collection"
+    ),
 });
 
 const embedInputSchema = z.object({
@@ -452,6 +466,13 @@ const indexInputSchema = z.object({
     .optional()
     .describe("Collection name to index. Omit to index all collections"),
   gitPull: z.boolean().default(false).describe("Run git pull before indexing"),
+  paths: z
+    .array(z.string())
+    .max(MAX_REINDEX_PATHS)
+    .optional()
+    .describe(
+      "Reindex only these files (1-256, relative to the collection root, or absolute inside it). Requires collection"
+    ),
 });
 
 const removeCollectionInputSchema = z.object({
@@ -496,6 +517,12 @@ export const vsearchInputSchema = z.object({
     .string()
     .optional()
     .describe("Filter to a single collection name"),
+  collections: z
+    .array(z.string())
+    .optional()
+    .describe(
+      "Filter to any of these collection names (1-256); use instead of collection"
+    ),
   limit: z
     .number()
     .int()
@@ -570,6 +597,12 @@ export const queryInputSchema = z.object({
     .string()
     .optional()
     .describe("Filter to a single collection name"),
+  collections: z
+    .array(z.string())
+    .optional()
+    .describe(
+      "Filter to any of these collection names (1-256); use instead of collection"
+    ),
   limit: z
     .number()
     .int()
@@ -672,15 +705,17 @@ export const queryInputSchema = z.object({
   tagsAny: z.array(z.string()).optional().describe("Require ANY of these tags"),
 });
 
-export const queryDiagnoseInputSchema = queryInputSchema.extend({
-  target: z
-    .string()
-    .trim()
-    .min(1, "Target reference cannot be empty")
-    .describe(
-      "Target document reference to diagnose (gno URI, docid, or collection/path)"
-    ),
-});
+export const queryDiagnoseInputSchema = queryInputSchema
+  .omit({ collections: true })
+  .extend({
+    target: z
+      .string()
+      .trim()
+      .min(1, "Target reference cannot be empty")
+      .describe(
+        "Target document reference to diagnose (gno URI, docid, or collection/path)"
+      ),
+  });
 
 const getInputSchema = z.object({
   ref: z
@@ -1019,7 +1054,8 @@ export async function runTool<T>(
   try {
     assertConfiguredCollections(ctx, name);
     assertInferenceActive();
-    const raw = await (ctx.runWithSnapshot?.(fn) ?? fn());
+    const read = READ_CONNECTION_TOOLS.has(name) ? readConnection(ctx, fn) : fn;
+    const raw = await (ctx.runWithSnapshot?.(read) ?? read());
     assertInferenceActive();
     const traceMetadata =
       raw !== null && typeof raw === "object"
@@ -1048,6 +1084,34 @@ export async function runTool<T>(
   }
 }
 
+/**
+ * Read-only retrieval tools. They read on the store's read connection, so a
+ * sync batch that is still writing is never visible to them.
+ */
+const READ_CONNECTION_TOOLS = new Set([
+  "gno_search",
+  "gno_vsearch",
+  "gno_query",
+  "gno_get",
+  "gno_multi_get",
+  "gno_status",
+  "gno_list_tags",
+  "gno_links",
+  "gno_backlinks",
+  "gno_similar",
+]);
+
+/** Store test doubles may lack the read connection; they read as before. */
+function readConnection<T>(
+  ctx: ToolContext,
+  fn: () => Promise<T>
+): () => Promise<T> {
+  const store = ctx.store as Partial<ToolContext["store"]> | undefined;
+  return typeof store?.withReadConnection === "function"
+    ? () => ctx.store.withReadConnection(fn)
+    : fn;
+}
+
 /**
  * Run a tool without acquiring the mutex.
  * For read-only in-memory tools (job_status, list_jobs) that should not block.

package/src/mcp/tools/job-status.ts

@@ -24,6 +24,7 @@ interface JobStatusResult {
   result?: SyncResult;
   typedResult?: JobResult;
   error?: string;
+  scope?: JobRecord["scope"];
   serverInstanceId: string;
 }
 
@@ -34,6 +35,11 @@ function formatJobStatus(result: JobStatusResult): string {
   lines.push(`Type: ${result.type}`);
   lines.push(`Status: ${result.status}`);
   lines.push(`Started: ${result.startedAt}`);
+  if (result.scope) {
+    lines.push(`Collection: ${result.scope.collection ?? "(all)"}`);
+    if (result.scope.paths)
+      lines.push(`Paths: ${result.scope.paths.join(", ")}`);
+  }
 
   if (result.completedAt) {
     lines.push(`Completed: ${result.completedAt}`);
@@ -92,6 +98,7 @@ function toJobStatusResult(job: JobRecord): JobStatusResult {
     result: job.result,
     typedResult: job.typedResult,
     error: job.error,
+    ...(job.scope ? { scope: job.scope } : {}),
     serverInstanceId: job.serverInstanceId,
   };
 }