@gmickel/gno 1.34.6 → 1.36.0

package/src/cli/options.ts

@@ -68,6 +68,7 @@ export const CMD = {
   multiGet: "multi-get",
   ls: "ls",
   status: "status",
+  peek: "peek",
   publishExport: "publish.export",
   collectionList: "collection.list",
   contextList: "context.list",
@@ -101,6 +102,7 @@ const FORMAT_SUPPORT: Record<CommandId, OutputFormat[]> = {
   [CMD.multiGet]: ["terminal", "json", "files", "md"],
   [CMD.ls]: ["terminal", "json", "files", "md"],
   [CMD.status]: ["terminal", "json"],
+  [CMD.peek]: ["terminal", "json"],
   [CMD.publishExport]: ["terminal", "json"],
   [CMD.collectionList]: ["terminal", "json", "md"],
   [CMD.contextList]: ["terminal", "json", "md"],

package/src/cli/program.ts

@@ -1676,6 +1676,26 @@ function wireOnboardingCommands(program: Command): void {
       );
     });
 
+  // peek - Cheap metadata snapshot for integrations
+  program
+    .command("peek")
+    .description("Show a cheap index snapshot for integrations")
+    .option("--json", "JSON output")
+    .action(async (cmdOpts: Record<string, unknown>) => {
+      const format = getFormat(cmdOpts);
+      assertFormatSupported(CMD.peek, format);
+
+      const { peek, formatPeek } = await import("./commands/peek");
+      const globals = getGlobals();
+      const snapshot = await peek({
+        configPath: globals.config,
+        indexName: globals.index,
+      });
+      process.stdout.write(
+        `${formatPeek(snapshot, { json: format === "json" })}\n`
+      );
+    });
+
   // doctor - Diagnose configuration issues
   program
     .command("doctor")

package/src/config/index.ts

@@ -60,6 +60,7 @@ export {
   type Config,
   ConfigSchema,
   DEFAULT_EGRESS_POLICY,
+  DEFAULT_SOURCE_AVAILABILITY,
   HttpGatewayConfigSchema,
   HttpGatewayLimitsSchema,
   CONTENT_TYPE_GRAPH_HINTS,
@@ -84,6 +85,9 @@ export {
   isValidLanguageHint,
   parseScope,
   resolveConfiguredEgressPolicy,
+  SOURCE_AVAILABILITY_MODES,
+  type SourceAvailabilityMode,
+  SourceAvailabilitySchema,
   type ProjectProfileBinding,
   ProjectProfileBindingSchema,
   type ScopeType,

package/src/config/types.ts

@@ -56,6 +56,12 @@ export type EgressPolicy = z.infer<typeof EgressPolicySchema>;
 /** Missing policy is always interpreted as the fail-closed local-only default. */
 export const DEFAULT_EGRESS_POLICY: EgressPolicy = "local_only";
 
+/** Source byte materialization boundary for collection indexing. */
+export const SOURCE_AVAILABILITY_MODES = ["any", "local"] as const;
+export const SourceAvailabilitySchema = z.enum(SOURCE_AVAILABILITY_MODES);
+export type SourceAvailabilityMode = z.infer<typeof SourceAvailabilitySchema>;
+export const DEFAULT_SOURCE_AVAILABILITY: SourceAvailabilityMode = "any";
+
 /** Provenance for an effective collection egress policy. */
 export const EGRESS_POLICY_SOURCES = [
   "explicit",
@@ -138,6 +144,14 @@ export const CollectionSchema = z.object({
    */
   egressPolicyRevision: z.number().int().nonnegative().optional(),
 
+  /**
+   * Source content availability policy for indexing. Distinct from egress:
+   * controls whether cloud placeholders may be materialized, not where data
+   * may leave the machine. Omitted / unset means `any` (legacy read behavior);
+   * `local` is opt-in and fails closed where the platform guard is unavailable.
+   */
+  sourceAvailability: SourceAvailabilitySchema.optional(),
+
   /** Optional per-collection model overrides */
   models: z
     .object({

package/src/core/path-rules.ts

@@ -51,3 +51,37 @@ export function matchesCollectionExclusion(
   }
   return false;
 }
+
+/**
+ * Whether a known directory is the root of a fully excluded subtree.
+ * A trailing `/**` excludes every descendant even though Bun.Glob does not
+ * match the directory root itself. Other partial glob shapes must still be
+ * traversed because they may leave eligible descendants.
+ */
+export function matchesCollectionSubtreeExclusion(
+  relPath: string,
+  excludes: readonly string[]
+): boolean {
+  const normalizedPath = relPath.replaceAll("\\", "/");
+  if (matchesCollectionExclusion(normalizedPath, excludes)) {
+    return true;
+  }
+
+  for (const rawPattern of excludes) {
+    const pattern = rawPattern.replaceAll("\\", "/");
+    if (pattern === "**") {
+      return true;
+    }
+    if (!pattern.endsWith("/**")) {
+      continue;
+    }
+    const subtreeRootPattern = pattern.slice(0, -3);
+    if (
+      subtreeRootPattern.length > 0 &&
+      new Bun.Glob(subtreeRootPattern).match(normalizedPath)
+    ) {
+      return true;
+    }
+  }
+  return false;
+}

package/src/core/peek.ts

@@ -0,0 +1,202 @@
+/**
+ * Shared peek snapshot builder for CLI and later MCP.
+ * Metadata-only: never opens the model cache, resolves model URIs, or activates.
+ *
+ * @module src/core/peek
+ */
+
+// node:path — no Bun path utils
+import { join as pathJoin } from "node:path";
+
+import type { DocumentRow, IndexStatus } from "../store/types";
+
+import { DEFAULT_INDEX_NAME, VERSION, getIndexDbPath } from "../app/constants";
+import {
+  isProcessAlive,
+  readPidFile,
+  resolveProcessPaths,
+} from "../cli/detach";
+import { CliError } from "../cli/errors";
+import { isInitialized, loadConfig } from "../config";
+import { SqliteAdapter } from "../store/sqlite/adapter";
+
+export const PEEK_SCHEMA_VERSION = "peek@1.0" as const;
+export const PEEK_RECENT_LIMIT = 10;
+
+export interface PeekCounts {
+  documents: number;
+  collections: number;
+}
+
+export interface PeekBacklog {
+  pending: number;
+  failed: number;
+}
+
+export interface PeekRecentItem {
+  docid: string;
+  uri: string;
+  title: string | null;
+  collection: string;
+  absPath: string;
+  modifiedAt: string;
+}
+
+export interface PeekServe {
+  running: boolean;
+  url: string | null;
+}
+
+export interface PeekSnapshot {
+  schemaVersion: typeof PEEK_SCHEMA_VERSION;
+  gnoVersion: string;
+  generatedAt: string;
+  initialized: boolean;
+  indexName: string;
+  counts: PeekCounts | null;
+  backlog: PeekBacklog | null;
+  lastIndexedAt: string | null;
+  recent: PeekRecentItem[];
+  serve: PeekServe;
+}
+
+export interface BuildPeekOptions {
+  configPath?: string;
+  indexName?: string;
+}
+
+function asRuntimeError(error: unknown, fallback: string): CliError {
+  if (error instanceof CliError) {
+    return error;
+  }
+  return new CliError(
+    "RUNTIME",
+    error instanceof Error ? error.message : fallback
+  );
+}
+
+async function readServeLiveness(): Promise<PeekServe> {
+  try {
+    const { pidFile } = resolveProcessPaths("serve");
+    const payload = await readPidFile(pidFile);
+    if (!payload || !isProcessAlive(payload.pid) || payload.port == null) {
+      return { running: false, url: null };
+    }
+    return {
+      running: true,
+      url: `http://localhost:${payload.port}`,
+    };
+  } catch (error) {
+    throw asRuntimeError(error, "Failed to read serve process state");
+  }
+}
+
+function mapRecent(
+  documents: DocumentRow[],
+  status: IndexStatus
+): PeekRecentItem[] {
+  const collectionPaths = new Map(
+    status.collections.map((collection) => [collection.name, collection.path])
+  );
+  return documents.slice(0, PEEK_RECENT_LIMIT).map((doc) => {
+    const sourceRelPath = doc.recordSourcePath ?? doc.relPath;
+    const collectionPath = collectionPaths.get(doc.collection) ?? "";
+    return {
+      docid: doc.docid,
+      uri: doc.uri,
+      title: doc.title,
+      collection: doc.collection,
+      absPath: pathJoin(collectionPath, sourceRelPath),
+      modifiedAt: doc.sourceMtime,
+    };
+  });
+}
+
+function emptySnapshot(
+  indexName: string,
+  generatedAt: string,
+  serve: PeekServe
+): PeekSnapshot {
+  return {
+    schemaVersion: PEEK_SCHEMA_VERSION,
+    gnoVersion: VERSION,
+    generatedAt,
+    initialized: false,
+    indexName,
+    counts: null,
+    backlog: null,
+    lastIndexedAt: null,
+    recent: [],
+    serve,
+  };
+}
+
+/**
+ * Build a peek@1.0 snapshot. Throws CliError("RUNTIME") on any subquery
+ * failure so callers never emit a half-filled payload.
+ */
+export async function buildPeekSnapshot(
+  options: BuildPeekOptions = {}
+): Promise<PeekSnapshot> {
+  const generatedAt = new Date().toISOString();
+  const requestedIndex = options.indexName ?? DEFAULT_INDEX_NAME;
+  const serve = await readServeLiveness();
+
+  const initialized = await isInitialized(options.configPath);
+  if (!initialized) {
+    return emptySnapshot(requestedIndex, generatedAt, serve);
+  }
+
+  const configResult = await loadConfig(options.configPath);
+  if (!configResult.ok) {
+    throw new CliError("RUNTIME", configResult.error.message);
+  }
+
+  const store = new SqliteAdapter();
+  const openResult = await store.open(
+    getIndexDbPath(options.indexName),
+    configResult.value.ftsTokenizer
+  );
+  if (!openResult.ok) {
+    throw new CliError("RUNTIME", openResult.error.message);
+  }
+
+  try {
+    const statusResult = await store.getStatus();
+    if (!statusResult.ok) {
+      throw new CliError("RUNTIME", statusResult.error.message);
+    }
+
+    const recentResult = await store.listDocumentsPaginated({
+      limit: PEEK_RECENT_LIMIT,
+      offset: 0,
+    });
+    if (!recentResult.ok) {
+      throw new CliError("RUNTIME", recentResult.error.message);
+    }
+
+    const status = statusResult.value;
+    return {
+      schemaVersion: PEEK_SCHEMA_VERSION,
+      gnoVersion: VERSION,
+      generatedAt,
+      initialized: true,
+      indexName: status.indexName,
+      counts: {
+        documents: status.activeDocuments,
+        collections: status.collections.length,
+      },
+      backlog: {
+        pending: status.embeddingBacklog,
+        failed: status.recentErrors,
+      },
+      lastIndexedAt: status.lastUpdatedAt,
+      recent: mapRecent(recentResult.value.documents, status),
+      serve,
+    };
+  } catch (error) {
+    throw asRuntimeError(error, "Failed to build peek snapshot");
+  } finally {
+    await store.close();
+  }
+}

package/src/ingestion/index.ts

@@ -11,6 +11,27 @@ export { defaultLanguageDetector, SimpleLanguageDetector } from "./language";
 // Sync service
 export { defaultSyncService, SyncService } from "./sync";
 export { resolveContentTypeRules, withContentTypeRules } from "./sync-options";
+// Source availability (content-boundary guard)
+export {
+  createDirectoryAvailability,
+  createSourceContentReader,
+  DEFAULT_SOURCE_AVAILABILITY,
+  findUnprovenAvailabilityPrefix,
+  isSourceAvailabilitySkip,
+  isUnprovenAbsenceCode,
+  memoizeDirectoryAvailability,
+  relPathUnderAnyPrefix,
+  resolveSourceAvailability,
+  SOURCE_AVAILABILITY_MODES,
+} from "./source-availability";
+export type {
+  DirectoryAvailabilityPort,
+  DirectoryAvailabilityResult,
+  SourceAvailabilityCode,
+  SourceAvailabilityMode,
+  SourceContentReaderPort,
+  SourceReadResult,
+} from "./source-availability";
 // Types
 export type {
   ChunkerPort,

package/src/ingestion/record-container.ts

@@ -58,6 +58,11 @@ interface RecordContainerInput {
   sourceMtime: string;
   sourceSize: number;
   store: StorePort;
+  /**
+   * Optional pre-read source bytes from the guarded content boundary.
+   * When set, record import streams from this buffer and never reopens the path.
+   */
+  sourceBytes?: Uint8Array;
 }
 
 interface AppliedRecordReconciliation {
@@ -283,6 +288,20 @@ const sourceStream = (
   },
 });
 
+const bytesStream = (
+  bytes: Uint8Array,
+  signal?: AbortSignal
+): AsyncIterable<Uint8Array> => ({
+  async *[Symbol.asyncIterator]() {
+    if (signal?.aborted) {
+      throw new Error("record adapter aborted");
+    }
+    if (bytes.byteLength > 0) {
+      yield bytes;
+    }
+  },
+});
+
 const loadPreviousStructure = async (
   store: StorePort,
   existing: DocumentRow | undefined
@@ -443,7 +462,10 @@ export async function processRecordContainer(
       collection: input.collection.name,
       mime: input.mime,
       ext: input.ext,
-      open: (signal) => sourceStream(input.entry.absPath, signal),
+      open: (signal) =>
+        input.sourceBytes
+          ? bytesStream(input.sourceBytes, signal)
+          : sourceStream(input.entry.absPath, signal),
       limits: {
         ...DEFAULT_RECORD_ADAPTER_LIMITS,
         timeoutMs: Math.min(

package/src/ingestion/source-availability/darwin-io.ts

@@ -0,0 +1,295 @@
+/**
+ * Darwin FFI backend for no-materialization I/O policy and guarded content reads.
+ * Provider-neutral; reuses the TN3150 mechanism proven in
+ * scripts/macos-file-provider-smoke.ts (IOPOL_TYPE_VFS_MATERIALIZE_DATALESS_FILES).
+ * No provider SDK; no pin/evict/download/availability mutation.
+ *
+ * @module src/ingestion/source-availability/darwin-io
+ */
+
+// bun:ffi — getiopolicy_np/setiopolicy_np/open/read require libc FFI; no Bun high-level equivalent
+import { dlopen, FFIType, ptr, toArrayBuffer } from "bun:ffi";
+// node:fs constants expose Darwin O_NOFOLLOW; Bun has no file-open flag API.
+import { constants as fsConstants } from "node:fs";
+
+/** TN3150 / File Provider constants (from smoke harness evidence). */
+export const SF_DATALESS = 0x4000_0000;
+export const IOPOL_TYPE_VFS_MATERIALIZE_DATALESS_FILES = 3;
+export const IOPOL_SCOPE_PROCESS = 0;
+export const IOPOL_MATERIALIZE_DATALESS_FILES_OFF = 1;
+/** Darwin errno for guarded dataless materialization refusal. */
+export const DARWIN_EDEADLK = 11;
+export const DARWIN_EACCES = 13;
+export const DARWIN_EPERM = 1;
+export const DARWIN_ENOENT = 2;
+export const DARWIN_EISDIR = 21;
+export const DARWIN_ELOOP = 62;
+export const DARWIN_EIO = 5;
+const OPEN_RDONLY_NOFOLLOW = fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW;
+
+export type DarwinIoPolicyPort = {
+  get: (type: number, scope: number) => number;
+  set: (type: number, scope: number, policy: number) => number;
+  readErrno: () => number;
+};
+
+export type DarwinFileIoPort = {
+  open: (absPath: string, flags: number) => number;
+  read: (fd: number, buf: Uint8Array) => number;
+  close: (fd: number) => number;
+  readErrno: () => number;
+};
+
+/**
+ * No-follow lstat of `st_flags` for SF_DATALESS directory classification.
+ * Distinct from content open/read — used only at directory boundaries.
+ */
+export type DarwinStatPort = {
+  lstatFlags: (
+    absPath: string
+  ) => { ok: true; stFlags: number } | { ok: false; errno: number };
+};
+
+export type DarwinIoBundle = {
+  policy: DarwinIoPolicyPort;
+  file: DarwinFileIoPort;
+  stat: DarwinStatPort;
+};
+
+/** Darwin `struct stat` layout used by the smoke harness (arm64/x86_64). */
+export const DARWIN_STAT_BUF_SIZE = 144;
+export const DARWIN_ST_FLAGS_OFFSET = 116;
+
+type LibSymbols = {
+  getiopolicy_np: (type: number, scope: number) => number;
+  setiopolicy_np: (type: number, scope: number, policy: number) => number;
+  open: (path: ReturnType<typeof ptr>, flags: number) => number;
+  close: (fd: number) => number;
+  read: (fd: number, buf: ReturnType<typeof ptr>, n: bigint) => bigint;
+  lstat: (path: ReturnType<typeof ptr>, buf: ReturnType<typeof ptr>) => number;
+  __error: () => ReturnType<typeof ptr> | null;
+};
+
+let cachedBundle: DarwinIoBundle | null | undefined;
+
+/** Test-only: clear FFI caches between cases. */
+export function resetDarwinIoCachesForTests(): void {
+  cachedBundle = undefined;
+}
+
+function asFn<T extends (...args: never[]) => unknown>(
+  value: unknown
+): T | null {
+  return typeof value === "function" ? (value as T) : null;
+}
+
+function cstr(value: string): Uint8Array {
+  return Buffer.from(`${value}\0`);
+}
+
+function readErrnoFrom(symbols: LibSymbols): number {
+  const p = symbols.__error();
+  if (!p) {
+    return 0;
+  }
+  return new Int32Array(toArrayBuffer(p, 0, 4))[0] ?? 0;
+}
+
+function loadLibSystem(): {
+  library: ReturnType<typeof dlopen>;
+  symbols: LibSymbols;
+} | null {
+  if (process.platform !== "darwin") {
+    return null;
+  }
+  for (const soname of ["libSystem.B.dylib", "libSystem.dylib"]) {
+    try {
+      const library = dlopen(soname, {
+        getiopolicy_np: {
+          args: [FFIType.i32, FFIType.i32],
+          returns: FFIType.i32,
+        },
+        setiopolicy_np: {
+          args: [FFIType.i32, FFIType.i32, FFIType.i32],
+          returns: FFIType.i32,
+        },
+        open: { args: [FFIType.ptr, FFIType.i32], returns: FFIType.i32 },
+        close: { args: [FFIType.i32], returns: FFIType.i32 },
+        read: {
+          args: [FFIType.i32, FFIType.ptr, FFIType.u64],
+          returns: FFIType.i64,
+        },
+        lstat: { args: [FFIType.ptr, FFIType.ptr], returns: FFIType.i32 },
+        __error: { args: [], returns: FFIType.ptr },
+      });
+      const raw = library.symbols as Record<string, unknown>;
+      const getiopolicy_np = asFn<LibSymbols["getiopolicy_np"]>(
+        raw.getiopolicy_np
+      );
+      const setiopolicy_np = asFn<LibSymbols["setiopolicy_np"]>(
+        raw.setiopolicy_np
+      );
+      const openFn = asFn<LibSymbols["open"]>(raw.open);
+      const closeFn = asFn<LibSymbols["close"]>(raw.close);
+      const readFn = asFn<LibSymbols["read"]>(raw.read);
+      const lstatFn = asFn<LibSymbols["lstat"]>(raw.lstat);
+      const errorFn = asFn<LibSymbols["__error"]>(raw.__error);
+      if (
+        !getiopolicy_np ||
+        !setiopolicy_np ||
+        !openFn ||
+        !closeFn ||
+        !readFn ||
+        !lstatFn ||
+        !errorFn
+      ) {
+        continue;
+      }
+      return {
+        library,
+        symbols: {
+          getiopolicy_np,
+          setiopolicy_np,
+          open: openFn,
+          close: closeFn,
+          read: readFn,
+          lstat: lstatFn,
+          __error: errorFn,
+        },
+      };
+    } catch {
+      // try next soname
+    }
+  }
+  return null;
+}
+
+/** Load Darwin policy + file I/O ports; null when unavailable. */
+export function loadDarwinIo(): DarwinIoBundle | null {
+  if (cachedBundle !== undefined) {
+    return cachedBundle;
+  }
+  const loaded = loadLibSystem();
+  if (!loaded) {
+    cachedBundle = null;
+    return null;
+  }
+  const { symbols } = loaded;
+  const readErrno = (): number => readErrnoFrom(symbols);
+  cachedBundle = {
+    policy: {
+      get: symbols.getiopolicy_np,
+      set: symbols.setiopolicy_np,
+      readErrno,
+    },
+    file: {
+      open: (absPath: string, flags: number): number =>
+        symbols.open(ptr(cstr(absPath)), flags),
+      read: (fd: number, buf: Uint8Array): number =>
+        Number(symbols.read(fd, ptr(buf), BigInt(buf.byteLength))),
+      close: symbols.close,
+      readErrno,
+    },
+    stat: {
+      lstatFlags: (absPath: string) => {
+        const buf = new Uint8Array(DARWIN_STAT_BUF_SIZE);
+        const rc = symbols.lstat(ptr(cstr(absPath)), ptr(buf));
+        if (rc !== 0) {
+          return { ok: false as const, errno: readErrno() };
+        }
+        const stFlags = new DataView(
+          buf.buffer,
+          buf.byteOffset,
+          buf.byteLength
+        ).getUint32(DARWIN_ST_FLAGS_OFFSET, true);
+        return { ok: true as const, stFlags };
+      },
+    },
+  };
+  // Keep library strongly referenced via symbols closure for process lifetime.
+  void loaded.library;
+  return cachedBundle;
+}
+
+/**
+ * Run `fn` under process-scoped IOPOL_MATERIALIZE_DATALESS_FILES_OFF.
+ * Always restores prior policy. Fail-closed on setup/restore failure.
+ */
+export function withNoMaterializePolicy<T>(
+  run: () => T,
+  port: DarwinIoPolicyPort
+):
+  | { ok: true; value: T }
+  | {
+      ok: false;
+      error:
+        | "policy_get_failed"
+        | "policy_set_failed"
+        | "policy_restore_failed";
+    } {
+  let prior: number;
+  try {
+    prior = port.get(
+      IOPOL_TYPE_VFS_MATERIALIZE_DATALESS_FILES,
+      IOPOL_SCOPE_PROCESS
+    );
+  } catch {
+    return { ok: false, error: "policy_get_failed" };
+  }
+  if (prior < 0) {
+    return { ok: false, error: "policy_get_failed" };
+  }
+  let setupResult: number;
+  try {
+    setupResult = port.set(
+      IOPOL_TYPE_VFS_MATERIALIZE_DATALESS_FILES,
+      IOPOL_SCOPE_PROCESS,
+      IOPOL_MATERIALIZE_DATALESS_FILES_OFF
+    );
+  } catch {
+    return { ok: false, error: "policy_set_failed" };
+  }
+  if (setupResult !== 0) {
+    return { ok: false, error: "policy_set_failed" };
+  }
+  let value: T;
+  let thrown: unknown;
+  try {
+    value = run();
+  } catch (error) {
+    thrown = error;
+  }
+  let restoreResult: number;
+  try {
+    restoreResult = port.set(
+      IOPOL_TYPE_VFS_MATERIALIZE_DATALESS_FILES,
+      IOPOL_SCOPE_PROCESS,
+      prior
+    );
+  } catch {
+    return { ok: false, error: "policy_restore_failed" };
+  }
+  if (restoreResult !== 0) {
+    return { ok: false, error: "policy_restore_failed" };
+  }
+  if (thrown !== undefined) {
+    throw thrown;
+  }
+  return { ok: true, value: value! };
+}
+
+export function guardedOpenFlags(): number {
+  return OPEN_RDONLY_NOFOLLOW;
+}
+
+export function classifyGuardedReadErrno(
+  errno: number
+): "EDEADLK" | "EACCES" | "EPERM" | "ENOENT" | "EISDIR" | "ELOOP" | "OTHER" {
+  if (errno === DARWIN_EDEADLK) return "EDEADLK";
+  if (errno === DARWIN_EACCES) return "EACCES";
+  if (errno === DARWIN_EPERM) return "EPERM";
+  if (errno === DARWIN_ENOENT) return "ENOENT";
+  if (errno === DARWIN_EISDIR) return "EISDIR";
+  if (errno === DARWIN_ELOOP) return "ELOOP";
+  return "OTHER";
+}