@gmickel/gno 1.34.6 → 1.36.0

package/src/ingestion/walker.ts

@@ -1,16 +1,20 @@
 /**
  * File walker implementation.
- * Walks collection directories using Bun.Glob with include/exclude filtering.
+ * Walks collection directories using Bun.Glob (`any`) or hierarchical
+ * availability-aware descent (`local`).
  *
  * @module src/ingestion/walker
  */
 
-// node:fs/promises - Bun has no realpath equivalent
-import { realpath } from "node:fs/promises";
+// node:fs - Bun has no synchronous Dirent enumeration for the guarded local walk.
+import { readdirSync } from "node:fs";
+// node:fs/promises - Bun has no realpath equivalent for symlink-safe containment.
+import { lstat, realpath } from "node:fs/promises";
 // node:path - Bun has no path manipulation module
 import {
   extname,
   isAbsolute,
+  join,
   normalize as normalizePath,
   relative,
   resolve,
@@ -20,8 +24,16 @@ import {
 import type { SkippedEntry, WalkConfig, WalkEntry, WalkerPort } from "./types";
 
 import { SUPPORTED_EXTENSIONS } from "../converters/mime";
-import { matchesCollectionExclusion } from "../core/path-rules";
+import {
+  matchesCollectionExclusion,
+  matchesCollectionSubtreeExclusion,
+} from "../core/path-rules";
 import { isRecordVirtualPath } from "./record-path";
+import {
+  createDirectoryAvailability,
+  type DirectoryAvailabilityPort,
+  isUnprovenDirectoryResult,
+} from "./source-availability";
 
 /**
  * Regex to detect dangerous patterns with parent directory traversal.
@@ -218,8 +230,27 @@ export function matchesWalkPath(
   );
 }
 
+function pushUnprovenDirectorySkip(
+  skipped: SkippedEntry[],
+  absPath: string,
+  relPath: string,
+  result: Exclude<
+    Awaited<ReturnType<DirectoryAvailabilityPort["classify"]>>,
+    { kind: "available" }
+  >
+): void {
+  skipped.push({
+    absPath,
+    relPath,
+    reason: result.code,
+    unprovenPrefix: true,
+    message: result.message,
+  });
+}
+
 /**
- * File walker implementation using Bun.Glob.
+ * File walker implementation using Bun.Glob (`any`) or hierarchical local-mode
+ * descent that refuses dataless / availability-unknown directories.
  *
  * Security: Validates patterns and ensures all matched files are within
  * the collection root directory. Files outside root are silently ignored.
@@ -228,6 +259,18 @@ export class FileWalker implements WalkerPort {
   async walk(config: WalkConfig): Promise<{
     entries: WalkEntry[];
     skipped: SkippedEntry[];
+  }> {
+    const mode = config.sourceAvailability ?? "any";
+    if (mode === "local") {
+      return this.walkLocal(config);
+    }
+    return this.walkAny(config);
+  }
+
+  /** Legacy Bun.Glob traversal — behaviorally unchanged for `any`. */
+  private async walkAny(config: WalkConfig): Promise<{
+    entries: WalkEntry[];
+    skipped: SkippedEntry[];
   }> {
     const entries: WalkEntry[] = [];
     const skipped: SkippedEntry[] = [];
@@ -316,6 +359,221 @@ export class FileWalker implements WalkerPort {
 
     return { entries, skipped };
   }
+
+  /**
+   * Local-mode hierarchical walk: classify each directory before descent.
+   * Does not add a per-file availability syscall — content recheck stays at
+   * the SourceContentReaderPort boundary.
+   */
+  private async walkLocal(config: WalkConfig): Promise<{
+    entries: WalkEntry[];
+    skipped: SkippedEntry[];
+  }> {
+    const entries: WalkEntry[] = [];
+    const skipped: SkippedEntry[] = [];
+
+    for (const pattern of scanPatterns(config.pattern)) {
+      const patternError = validatePattern(pattern);
+      if (patternError) {
+        throw new Error(`Invalid glob pattern: ${patternError}`);
+      }
+    }
+
+    const rootAbs = resolve(config.root);
+    const classifier =
+      config.directoryAvailability ??
+      createDirectoryAvailability(config.sourceAvailability ?? "local");
+    const configuredRootClassified = await classifier.classify(rootAbs);
+    if (isUnprovenDirectoryResult(configuredRootClassified)) {
+      pushUnprovenDirectorySkip(skipped, rootAbs, "", configuredRootClassified);
+      return { entries, skipped };
+    }
+
+    let rootReal: string;
+    try {
+      rootReal = await realpath(rootAbs);
+    } catch {
+      skipped.push({
+        absPath: rootAbs,
+        relPath: "",
+        reason: "SOURCE_AVAILABILITY_UNKNOWN",
+        unprovenPrefix: true,
+        message:
+          "Collection root could not be resolved after availability check",
+      });
+      return { entries, skipped };
+    }
+
+    if (rootReal !== rootAbs) {
+      const canonicalRootClassified = await classifier.classify(rootReal);
+      if (isUnprovenDirectoryResult(canonicalRootClassified)) {
+        pushUnprovenDirectorySkip(
+          skipped,
+          rootReal,
+          "",
+          canonicalRootClassified
+        );
+        return { entries, skipped };
+      }
+    }
+
+    const queue: Array<{ absPath: string; relPath: string }> = [
+      { absPath: rootReal, relPath: "" },
+    ];
+    let head = 0;
+
+    while (head < queue.length) {
+      const dir = queue[head] as { absPath: string; relPath: string };
+      head += 1;
+
+      let dirents;
+      try {
+        const read = classifier.readDirectory(dir.absPath, () =>
+          readdirSync(dir.absPath, { withFileTypes: true })
+        );
+        if (read.kind !== "available") {
+          pushUnprovenDirectorySkip(skipped, dir.absPath, dir.relPath, read);
+          continue;
+        }
+        dirents = read.value;
+      } catch {
+        skipped.push({
+          absPath: dir.absPath,
+          relPath: dir.relPath,
+          reason: "SOURCE_AVAILABILITY_UNKNOWN",
+          unprovenPrefix: true,
+          message: "Failed to enumerate directory after availability check",
+        });
+        continue;
+      }
+
+      dirents.sort((left, right) => left.name.localeCompare(right.name));
+      for (const dirent of dirents) {
+        const name = dirent.name;
+        if (name === "" || name === "." || name === "..") {
+          continue;
+        }
+        if (name.includes("\0")) {
+          continue;
+        }
+
+        const childAbs = join(dir.absPath, name);
+        const childRel =
+          dir.relPath === "" ? name : `${dir.relPath}/${toPosixPath(name)}`;
+
+        // No-follow: symlinks are leaf candidates so the guarded content open
+        // can refuse them with a receipt; never descend through them.
+        if (dirent.isSymbolicLink()) {
+          if (!matchesWalkPath(childRel, config)) {
+            skipped.push({
+              absPath: childAbs,
+              relPath: childRel,
+              reason: "EXCLUDED",
+            });
+            continue;
+          }
+          try {
+            const linkStat = await lstat(childAbs);
+            entries.push({
+              absPath: childAbs,
+              relPath: childRel,
+              size: linkStat.size,
+              mtime: linkStat.mtime.toISOString(),
+              ctime: (
+                linkStat.birthtime ??
+                linkStat.ctime ??
+                linkStat.mtime
+              ).toISOString(),
+            });
+          } catch {
+            skipped.push({
+              absPath: childAbs,
+              relPath: childRel,
+              reason: "SOURCE_AVAILABILITY_UNKNOWN",
+              unprovenPrefix: true,
+              message: "Symlink metadata changed during local traversal",
+            });
+          }
+          continue;
+        }
+
+        if (dirent.isDirectory()) {
+          if (matchesCollectionSubtreeExclusion(childRel, config.exclude)) {
+            skipped.push({
+              absPath: childAbs,
+              relPath: childRel,
+              reason: "EXCLUDED",
+            });
+            continue;
+          }
+          const classified = await classifier.classify(childAbs);
+          if (isUnprovenDirectoryResult(classified)) {
+            pushUnprovenDirectorySkip(skipped, childAbs, childRel, classified);
+            continue;
+          }
+          queue.push({ absPath: childAbs, relPath: childRel });
+          continue;
+        }
+
+        if (!dirent.isFile()) {
+          continue;
+        }
+
+        const safePath = await safeRelPath(rootReal, childAbs);
+        if (safePath === null) {
+          continue;
+        }
+        const { absPath, relPath } = safePath;
+
+        if (!matchesWalkPath(relPath, config)) {
+          skipped.push({
+            absPath,
+            relPath,
+            reason: "EXCLUDED",
+          });
+          continue;
+        }
+
+        const file = Bun.file(absPath);
+        let fileStat: {
+          size: number;
+          mtime: Date;
+          ctime?: Date;
+          birthtime?: Date;
+        };
+        try {
+          fileStat = await file.stat();
+        } catch {
+          continue;
+        }
+
+        if (fileStat.size > config.maxBytes) {
+          skipped.push({
+            absPath,
+            relPath,
+            reason: "TOO_LARGE",
+            size: fileStat.size,
+          });
+          continue;
+        }
+
+        entries.push({
+          absPath,
+          relPath,
+          size: fileStat.size,
+          mtime: fileStat.mtime.toISOString(),
+          ctime: (
+            fileStat.birthtime ??
+            fileStat.ctime ??
+            fileStat.mtime
+          ).toISOString(),
+        });
+      }
+    }
+
+    entries.sort((a, b) => a.relPath.localeCompare(b.relPath));
+    return { entries, skipped };
+  }
 }
 
 /**

package/src/mcp/http-egress.ts

@@ -56,6 +56,7 @@ export const MCP_HTTP_EGRESS_TOOLS = {
   gno_search: "snippet",
   gno_section: "metadata",
   gno_similar: "snippet",
+  gno_peek: "metadata",
   gno_status: "metadata",
   gno_sync: "metadata",
   gno_trace_delete: "retrieval_trace",

package/src/mcp/tools/index.ts

@@ -65,6 +65,7 @@ import {
 import { handleListJobs } from "./list-jobs";
 import { handleListTags } from "./list-tags";
 import { handleMultiGet } from "./multi-get";
+import { handlePeek, PEEK_MCP_ANNOTATIONS } from "./peek";
 import { handleQuery, handleQueryDiagnose } from "./query";
 import { handleRemoveCollection } from "./remove-collection";
 import { handleSearch } from "./search";
@@ -129,6 +130,7 @@ export const MCP_TOOL_DESCRIPTIONS = {
     "Retrieve multiple documents by refs array or glob pattern. Use after gno_search/gno_query to batch top result URIs/docids; set maxBytes and lineNumbers to control context size.",
   section:
     "Create or resolve a durable SectionTargetV1 against one indexed document. action=create needs ref plus exactly one of anchor|line; action=resolve needs ref plus target. Exact/recovered include citation (uri, anchor, title, inclusive lines, fingerprint); ambiguous/stale/missing omit citation and are not safe to navigate or cite. Read-only — does not write or persist targets. Follow navigable ranges with gno_get fromLine/lineCount.",
+  peek: "Cheap peek@1.0 snapshot: initialized flag, document/collection counts, embedding backlog, recent files, and serve liveness. Model-free — never initializes embeddings or models. Use for counts/backlog/recent/serve questions; use gno_status for full health and activation.",
   status:
     "Get index health: collection count, document count, chunk count, embedding backlog, and per-collection stats. Check first when vector/hybrid results look stale or unavailable.",
   audit:
@@ -664,6 +666,8 @@ const multiGetInputSchema = z.object({
     .describe("Include line numbers in output"),
 });
 
+const peekInputSchema = z.object({});
+
 const statusInputSchema = z.object({});
 
 const jobStatusInputSchema = z.object({
@@ -1086,6 +1090,16 @@ export function registerTools(server: McpServer, ctx: ToolContext): void {
     (args) => handleMultiGet(args, ctx)
   );
 
+  server.registerTool(
+    "gno_peek",
+    {
+      description: MCP_TOOL_DESCRIPTIONS.peek,
+      inputSchema: peekInputSchema,
+      annotations: PEEK_MCP_ANNOTATIONS,
+    },
+    (args) => handlePeek(args, ctx)
+  );
+
   server.tool(
     "gno_status",
     MCP_TOOL_DESCRIPTIONS.status,

package/src/mcp/tools/peek.ts

@@ -0,0 +1,78 @@
+/**
+ * MCP gno_peek tool — cheap peek@1.0 snapshot, model-free.
+ *
+ * @module src/mcp/tools/peek
+ */
+
+import type { PeekSnapshot } from "../../core/peek";
+import type { ToolContext } from "../server";
+
+import { buildPeekSnapshot } from "../../core/peek";
+import { runTool, type ToolResult } from "./index";
+
+type PeekInput = Record<string, never>;
+
+export const PEEK_MCP_ANNOTATIONS = {
+  readOnlyHint: true,
+  destructiveHint: false,
+  idempotentHint: true,
+  openWorldHint: false,
+} as const;
+
+function formatPeek(snapshot: PeekSnapshot): string {
+  const lines = [
+    `schema: ${snapshot.schemaVersion}`,
+    `gno: ${snapshot.gnoVersion}`,
+    `index: ${snapshot.indexName}`,
+    `initialized: ${snapshot.initialized ? "yes" : "no"}`,
+  ];
+  if (snapshot.counts) {
+    lines.push(
+      `documents: ${snapshot.counts.documents}`,
+      `collections: ${snapshot.counts.collections}`
+    );
+  }
+  if (snapshot.backlog) {
+    lines.push(
+      `backlog: ${snapshot.backlog.pending} pending, ${snapshot.backlog.failed} failed`
+    );
+  }
+  if (snapshot.lastIndexedAt) {
+    lines.push(`lastIndexedAt: ${snapshot.lastIndexedAt}`);
+  }
+  lines.push(
+    snapshot.serve.running && snapshot.serve.url
+      ? `serve: ${snapshot.serve.url}`
+      : "serve: down"
+  );
+  if (snapshot.recent.length > 0) {
+    lines.push("recent:");
+    for (const item of snapshot.recent) {
+      const label = item.title ?? item.uri;
+      lines.push(`  ${item.docid} ${label}`);
+    }
+  }
+  return lines.join("\n");
+}
+
+/**
+ * Handle gno_peek tool call.
+ *
+ * Uninitialized (no config / no store) is a success payload, not an error.
+ * Never probes serve over HTTP; the shared builder uses pid-file liveness.
+ */
+export function handlePeek(
+  _args: PeekInput,
+  ctx: ToolContext
+): Promise<ToolResult> {
+  return runTool(
+    ctx,
+    "gno_peek",
+    async () =>
+      buildPeekSnapshot({
+        configPath: ctx.actualConfigPath,
+        indexName: ctx.indexName,
+      }),
+    formatPeek
+  );
+}