@gmickel/gno 1.34.6 → 1.36.0

package/src/ingestion/source-availability/readers.ts

@@ -0,0 +1,360 @@
+/**
+ * Source content readers for `any` (legacy) and `local` (Darwin-guarded) modes.
+ *
+ * @module src/ingestion/source-availability/readers
+ */
+
+import {
+  classifyGuardedReadErrno,
+  type DarwinFileIoPort,
+  type DarwinIoPolicyPort,
+  guardedOpenFlags,
+  loadDarwinIo,
+  withNoMaterializePolicy,
+} from "./darwin-io";
+import {
+  classifyDarwinFileProviderPath,
+  type DarwinFileProviderPathSupport,
+} from "./darwin-path";
+import {
+  type SourceAvailabilityMode,
+  type SourceContentReaderPort,
+  type SourceReadResult,
+  sourceAvailabilityMessage,
+} from "./types";
+
+const READ_CHUNK_BYTES = 64 * 1024;
+/** Injectable deps for unit tests; production uses loadDarwinIo(). */
+export type LocalReaderDeps = {
+  platform?: string;
+  policy?: DarwinIoPolicyPort | null;
+  file?: DarwinFileIoPort | null;
+  pathSupport?: (absPath: string) => DarwinFileProviderPathSupport;
+  /**
+   * Optional pre-open flag probe. When provided and returns a non-null
+   * outcome, the reader fails closed without materializing content.
+   * Production leaves this unset (content-boundary policy is the guard).
+   */
+  safetyProbe?: (absPath: string) => SourceReadResult | null;
+};
+
+/** Legacy path: Bun.file byte reads (current ingestion behavior). */
+export class AnySourceContentReader implements SourceContentReaderPort {
+  readonly mode: SourceAvailabilityMode = "any";
+
+  async readAll(
+    absPath: string,
+    _expectedSize?: number
+  ): Promise<SourceReadResult> {
+    try {
+      const bytes = await Bun.file(absPath).bytes();
+      return { ok: true, bytes };
+    } catch (error) {
+      return mapNodeLikeReadError(error);
+    }
+  }
+}
+
+/**
+ * Local mode reader. On Darwin, establishes no-materialization I/O policy and
+ * reads via native open/read so EDEADLK maps to cloud-placeholder skip.
+ * Non-Darwin and policy/FFI failures fail closed with distinct codes.
+ */
+export class LocalSourceContentReader implements SourceContentReaderPort {
+  readonly mode: SourceAvailabilityMode = "local";
+  private readonly platform: string;
+  private readonly policy: DarwinIoPolicyPort | null;
+  private readonly file: DarwinFileIoPort | null;
+  private readonly safetyProbe?: (absPath: string) => SourceReadResult | null;
+  private readonly pathSupport: (
+    absPath: string
+  ) => DarwinFileProviderPathSupport;
+
+  constructor(deps: LocalReaderDeps = {}) {
+    this.platform = deps.platform ?? process.platform;
+    if (deps.policy !== undefined || deps.file !== undefined) {
+      this.policy = deps.policy ?? null;
+      this.file = deps.file ?? null;
+    } else if (this.platform === "darwin") {
+      const loaded = loadDarwinIo();
+      this.policy = loaded?.policy ?? null;
+      this.file = loaded?.file ?? null;
+    } else {
+      this.policy = null;
+      this.file = null;
+    }
+    this.safetyProbe = deps.safetyProbe;
+    this.pathSupport = deps.pathSupport ?? classifyDarwinFileProviderPath;
+  }
+
+  async readAll(
+    absPath: string,
+    expectedSize?: number
+  ): Promise<SourceReadResult> {
+    if (this.platform !== "darwin") {
+      return {
+        ok: false,
+        code: "SOURCE_AVAILABILITY_UNSUPPORTED",
+        message: sourceAvailabilityMessage(
+          "SOURCE_AVAILABILITY_UNSUPPORTED",
+          `platform=${this.platform}`
+        ),
+      };
+    }
+    if (!this.policy || !this.file) {
+      return {
+        ok: false,
+        code: "SOURCE_AVAILABILITY_POLICY_FAILED",
+        message: sourceAvailabilityMessage(
+          "SOURCE_AVAILABILITY_POLICY_FAILED",
+          "darwin_io_unavailable"
+        ),
+      };
+    }
+
+    const pathSupport = this.pathSupport(absPath);
+    if (pathSupport === "unsupported") {
+      return {
+        ok: false,
+        code: "SOURCE_AVAILABILITY_UNSUPPORTED",
+        message: sourceAvailabilityMessage(
+          "SOURCE_AVAILABILITY_UNSUPPORTED",
+          "path is outside the physically evidenced macOS File Provider layouts"
+        ),
+      };
+    }
+    if (pathSupport === "unknown") {
+      return {
+        ok: false,
+        code: "SOURCE_AVAILABILITY_UNKNOWN",
+        message: sourceAvailabilityMessage(
+          "SOURCE_AVAILABILITY_UNKNOWN",
+          "path support could not be established"
+        ),
+      };
+    }
+
+    if (this.safetyProbe) {
+      const probed = this.safetyProbe(absPath);
+      if (probed !== null) {
+        return probed;
+      }
+    }
+
+    const policy = this.policy;
+    const file = this.file;
+    let wrapped: ReturnType<typeof withNoMaterializePolicy<SourceReadResult>>;
+    try {
+      wrapped = withNoMaterializePolicy(
+        () => readAllNative(absPath, file, expectedSize),
+        policy
+      );
+    } catch (error) {
+      return {
+        ok: false,
+        code: "SOURCE_AVAILABILITY_UNKNOWN",
+        message: sourceAvailabilityMessage(
+          "SOURCE_AVAILABILITY_UNKNOWN",
+          error instanceof Error ? error.message : "native_read_failed"
+        ),
+      };
+    }
+    if (!wrapped.ok) {
+      return {
+        ok: false,
+        code: "SOURCE_AVAILABILITY_POLICY_FAILED",
+        message: sourceAvailabilityMessage(
+          "SOURCE_AVAILABILITY_POLICY_FAILED",
+          wrapped.error
+        ),
+      };
+    }
+    return wrapped.value;
+  }
+}
+
+function readAllNative(
+  absPath: string,
+  file: DarwinFileIoPort,
+  expectedSize?: number
+): SourceReadResult {
+  const fd = file.open(absPath, guardedOpenFlags());
+  if (fd < 0) {
+    return mapErrno(file.readErrno(), 0);
+  }
+  try {
+    const chunks: Uint8Array[] = [];
+    let total = 0;
+    const buf = new Uint8Array(READ_CHUNK_BYTES);
+    for (;;) {
+      const n = file.read(fd, buf);
+      if (n < 0) {
+        return mapErrno(file.readErrno(), total);
+      }
+      if (n === 0) {
+        if (expectedSize !== undefined && total < expectedSize) {
+          return {
+            ok: false,
+            code: "IO_ERROR",
+            message: sourceAvailabilityMessage(
+              "IO_ERROR",
+              `short_read expected=${expectedSize} read=${total}`
+            ),
+          };
+        }
+        break;
+      }
+      chunks.push(buf.slice(0, n));
+      total += n;
+    }
+    return { ok: true, bytes: concatChunks(chunks, total) };
+  } finally {
+    file.close(fd);
+  }
+}
+
+function mapErrno(errno: number, bytesAlreadyRead: number): SourceReadResult {
+  const kind = classifyGuardedReadErrno(errno);
+  if (kind === "EDEADLK") {
+    if (bytesAlreadyRead > 0) {
+      return {
+        ok: false,
+        code: "CLOUD_PARTIAL",
+        message: sourceAvailabilityMessage("CLOUD_PARTIAL", `errno=${errno}`),
+        errno,
+      };
+    }
+    return {
+      ok: false,
+      code: "CLOUD_PLACEHOLDER",
+      message: sourceAvailabilityMessage("CLOUD_PLACEHOLDER", `errno=${errno}`),
+      errno,
+    };
+  }
+  if (kind === "EACCES" || kind === "EPERM") {
+    return {
+      ok: false,
+      code: "PERMISSION",
+      message: sourceAvailabilityMessage("PERMISSION", `errno=${errno}`),
+      errno,
+    };
+  }
+  if (kind === "ENOENT") {
+    return {
+      ok: false,
+      code: "NOT_FOUND",
+      message: sourceAvailabilityMessage("NOT_FOUND", `errno=${errno}`),
+      errno,
+    };
+  }
+  if (kind === "EISDIR") {
+    return {
+      ok: false,
+      code: "NOT_FILE",
+      message: sourceAvailabilityMessage("NOT_FILE", `errno=${errno}`),
+      errno,
+    };
+  }
+  if (kind === "ELOOP") {
+    return {
+      ok: false,
+      code: "SOURCE_AVAILABILITY_UNKNOWN",
+      message: sourceAvailabilityMessage(
+        "SOURCE_AVAILABILITY_UNKNOWN",
+        `symlink_loop errno=${errno}`
+      ),
+      errno,
+    };
+  }
+  // Unknown errno / flags / safety: fail closed, never claim safe to read.
+  return {
+    ok: false,
+    code: "SOURCE_AVAILABILITY_UNKNOWN",
+    message: sourceAvailabilityMessage(
+      "SOURCE_AVAILABILITY_UNKNOWN",
+      `errno=${errno}`
+    ),
+    errno,
+  };
+}
+
+function mapNodeLikeReadError(error: unknown): SourceReadResult {
+  const code =
+    typeof error === "object" &&
+    error !== null &&
+    "code" in error &&
+    typeof (error as { code?: unknown }).code === "string"
+      ? (error as { code: string }).code
+      : undefined;
+  if (code === "ENOENT") {
+    return {
+      ok: false,
+      code: "NOT_FOUND",
+      message: sourceAvailabilityMessage("NOT_FOUND"),
+    };
+  }
+  if (code === "EACCES" || code === "EPERM") {
+    return {
+      ok: false,
+      code: "PERMISSION",
+      message: sourceAvailabilityMessage("PERMISSION"),
+    };
+  }
+  if (code === "EISDIR") {
+    return {
+      ok: false,
+      code: "NOT_FILE",
+      message: sourceAvailabilityMessage("NOT_FILE"),
+    };
+  }
+  const detail = error instanceof Error ? error.message : "read_failed";
+  return {
+    ok: false,
+    code: "IO_ERROR",
+    message: sourceAvailabilityMessage("IO_ERROR", detail),
+  };
+}
+
+function concatChunks(chunks: Uint8Array[], total: number): Uint8Array {
+  if (chunks.length === 0) {
+    return new Uint8Array(0);
+  }
+  if (chunks.length === 1) {
+    return chunks[0] ?? new Uint8Array(0);
+  }
+  const out = new Uint8Array(total);
+  let offset = 0;
+  for (const chunk of chunks) {
+    out.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return out;
+}
+
+/** Factory: mode → reader. Local uses Darwin guard; unsupported platforms fail closed. */
+export function createSourceContentReader(
+  mode: SourceAvailabilityMode,
+  deps: LocalReaderDeps = {}
+): SourceContentReaderPort {
+  if (mode === "any") {
+    return new AnySourceContentReader();
+  }
+  return new LocalSourceContentReader(deps);
+}
+
+/** Yield one pre-read buffer as an async iterable (record-import open seam). */
+export function bytesAsAsyncIterable(
+  bytes: Uint8Array,
+  signal?: AbortSignal
+): AsyncIterable<Uint8Array> {
+  return {
+    async *[Symbol.asyncIterator]() {
+      if (signal?.aborted) {
+        throw new Error("source read aborted");
+      }
+      if (bytes.byteLength > 0) {
+        yield bytes;
+      }
+    },
+  };
+}

package/src/ingestion/source-availability/resolve.ts

@@ -0,0 +1,28 @@
+/**
+ * Resolve effective source-availability mode for a collection/index run.
+ *
+ * @module src/ingestion/source-availability/resolve
+ */
+
+import type { Collection } from "../../config/types";
+import type { SyncOptions } from "../types";
+
+import {
+  DEFAULT_SOURCE_AVAILABILITY,
+  type SourceAvailabilityMode,
+} from "./types";
+
+/**
+ * Run-level SyncOptions override wins over collection config.
+ * Missing values resolve to `any` (behaviorally unchanged default).
+ */
+export function resolveSourceAvailability(
+  collection: Pick<Collection, "sourceAvailability">,
+  options?: Pick<SyncOptions, "sourceAvailability">
+): SourceAvailabilityMode {
+  return (
+    options?.sourceAvailability ??
+    collection.sourceAvailability ??
+    DEFAULT_SOURCE_AVAILABILITY
+  );
+}

package/src/ingestion/source-availability/types.ts

@@ -0,0 +1,170 @@
+/**
+ * Platform-neutral source-availability policy contract.
+ * Controls whether source content materialization is allowed during ingestion.
+ * Distinct from collection egress/privacy policy.
+ *
+ * @module src/ingestion/source-availability/types
+ */
+
+import {
+  DEFAULT_SOURCE_AVAILABILITY,
+  SOURCE_AVAILABILITY_MODES,
+  type SourceAvailabilityMode,
+} from "../../config/types";
+
+export {
+  DEFAULT_SOURCE_AVAILABILITY,
+  SOURCE_AVAILABILITY_MODES,
+  type SourceAvailabilityMode,
+};
+
+/**
+ * Distinct fail-closed / skip outcomes for guarded source content access.
+ * CLOUD_* and DATALESS_DIRECTORY are skips (not conversion errors).
+ * Others fail closed as errors. Directory-boundary unknowns preserve indexed
+ * descendants (unproven absence) rather than proving deletion.
+ */
+export const SOURCE_AVAILABILITY_CODES = [
+  "CLOUD_PLACEHOLDER",
+  "CLOUD_PARTIAL",
+  "DATALESS_DIRECTORY",
+  "SOURCE_AVAILABILITY_UNSUPPORTED",
+  "SOURCE_AVAILABILITY_POLICY_FAILED",
+  "SOURCE_AVAILABILITY_UNKNOWN",
+  "PERMISSION",
+  "NOT_FOUND",
+  "NOT_FILE",
+  "IO_ERROR",
+] as const;
+export type SourceAvailabilityCode = (typeof SOURCE_AVAILABILITY_CODES)[number];
+
+/** Codes that must surface as skipped (not conversion/store errors). */
+export const SOURCE_AVAILABILITY_SKIP_CODES = new Set<SourceAvailabilityCode>([
+  "CLOUD_PLACEHOLDER",
+  "CLOUD_PARTIAL",
+  "DATALESS_DIRECTORY",
+]);
+
+/**
+ * Directory / prefix outcomes that must not prove descendant absence.
+ * Includes fail-closed errors where enumeration safety is unknown.
+ */
+export const SOURCE_AVAILABILITY_UNPROVEN_PREFIX_CODES =
+  new Set<SourceAvailabilityCode>([
+    "DATALESS_DIRECTORY",
+    "SOURCE_AVAILABILITY_UNSUPPORTED",
+    "SOURCE_AVAILABILITY_POLICY_FAILED",
+    "SOURCE_AVAILABILITY_UNKNOWN",
+    "PERMISSION",
+    "IO_ERROR",
+    "NOT_FOUND",
+    "NOT_FILE",
+  ]);
+
+export type SourceReadSuccess = {
+  ok: true;
+  bytes: Uint8Array;
+};
+
+export type SourceReadFailure = {
+  ok: false;
+  code: SourceAvailabilityCode;
+  message: string;
+  /** Optional low-level errno for diagnostics (never required by callers). */
+  errno?: number | null;
+};
+
+export type SourceReadResult = SourceReadSuccess | SourceReadFailure;
+
+/**
+ * Single content-boundary port: one guarded read supplies bytes for
+ * sniff/hash/conversion and record-import open paths.
+ */
+export interface SourceContentReaderPort {
+  readonly mode: SourceAvailabilityMode;
+  /**
+   * Read all source bytes under the active availability policy.
+   * Local mode rechecks only at this byte-consumption boundary.
+   */
+  readAll(absPath: string, expectedSize?: number): Promise<SourceReadResult>;
+}
+
+/** Amortized directory-boundary classification (not per discovered file). */
+export type DirectoryAvailabilityResult =
+  | { kind: "available" }
+  | {
+      kind: "dataless";
+      code: "DATALESS_DIRECTORY";
+      message: string;
+    }
+  | {
+      kind: "error";
+      code: SourceAvailabilityCode;
+      message: string;
+      errno?: number | null;
+    };
+
+export type DirectoryReadResult<T> =
+  | { kind: "available"; value: T }
+  | Exclude<DirectoryAvailabilityResult, { kind: "available" }>;
+
+export type SynchronousDirectoryRead<T> = () => T &
+  (T extends PromiseLike<unknown> ? never : unknown);
+
+export interface DirectoryAvailabilityPort {
+  readonly mode: SourceAvailabilityMode;
+  /** Classify one directory before descent or ancestor-prefix checks. */
+  classify(absPath: string): Promise<DirectoryAvailabilityResult>;
+  /** Reclassify and synchronously enumerate while the policy remains active. */
+  readDirectory<T>(
+    absPath: string,
+    read: SynchronousDirectoryRead<T>
+  ): DirectoryReadResult<T>;
+}
+
+export function isSourceAvailabilitySkip(
+  code: string | undefined
+): code is "CLOUD_PLACEHOLDER" | "CLOUD_PARTIAL" | "DATALESS_DIRECTORY" {
+  return (
+    code === "CLOUD_PLACEHOLDER" ||
+    code === "CLOUD_PARTIAL" ||
+    code === "DATALESS_DIRECTORY"
+  );
+}
+
+export function isUnprovenAbsenceCode(code: string | undefined): boolean {
+  return (
+    typeof code === "string" &&
+    SOURCE_AVAILABILITY_UNPROVEN_PREFIX_CODES.has(
+      code as SourceAvailabilityCode
+    )
+  );
+}
+
+export function sourceAvailabilityMessage(
+  code: SourceAvailabilityCode,
+  detail?: string
+): string {
+  const base: Record<SourceAvailabilityCode, string> = {
+    CLOUD_PLACEHOLDER:
+      "Source is a cloud placeholder; local mode refused materialization",
+    CLOUD_PARTIAL:
+      "Source has partial cloud content; local mode refused incomplete materialization",
+    DATALESS_DIRECTORY:
+      "Directory is dataless or availability-unknown; local mode refused descent",
+    SOURCE_AVAILABILITY_UNSUPPORTED:
+      "sourceAvailability local is not supported on this platform or filesystem",
+    SOURCE_AVAILABILITY_POLICY_FAILED:
+      "Failed to establish no-materialization I/O policy for sourceAvailability local",
+    SOURCE_AVAILABILITY_UNKNOWN:
+      "Source availability could not be determined safely; local mode fails closed",
+    PERMISSION: "Permission denied reading source file",
+    NOT_FOUND: "Source file not found",
+    NOT_FILE: "Path is not a regular file",
+    IO_ERROR: "I/O error reading source file",
+  };
+  if (detail && detail.length > 0) {
+    return `${base[code]}: ${detail}`;
+  }
+  return base[code];
+}