@hicaru/pi-rlm 0.2.2 → 0.3.1

package/src/context/source-dir.ts

@@ -0,0 +1,166 @@
+/**
+ * Directory pack: walk + per-file routing (bounded pool).
+ *
+ * Routing order is load-bearing:
+ *   sensitive (link name)?  → skipped: "sensitive"
+ *   checkPathSafety (lstat/realpath; escape / resolved-sensitive)
+ *   format != null          → documentToContextFile() ← MUST come before isBinary
+ *   else                    → textToContextFile()
+ *
+ * A .docx IS binary. Reversed, every document in the tree would be dropped as an asset —
+ * precisely the bug this change exists to fix.
+ */
+
+import { join } from "node:path";
+import { documentExtFromPath, getAnydoc, type AnydocHandle } from "./anydoc.ts";
+import { Semaphore } from "../util/concurrency.ts";
+import { documentToContextFile } from "./source-doc.ts";
+import { textToContextFile } from "./source-text.ts";
+import { checkPathSafety, enumerateFiles, isSensitivePath } from "./walk.ts";
+import {
+  MAX_SKIPPED_REPORTED,
+  type ContextFile,
+  type ResolveOpts,
+  type SkippedFile,
+  type SourceResult,
+} from "./types.ts";
+import { applyPathPrefix, contextSourceId, pathPrefixFor } from "./namespace.ts";
+
+/** Conversions are libuv-threadpool bound (default pool 4); 8 keeps the pool busy without thrash. */
+const CONVERT_CONCURRENCY = 8;
+
+export interface PackDirResult {
+  readonly files: readonly ContextFile[];
+  readonly chars: number;
+  readonly documents: number;
+  readonly converted: number;
+  readonly skipped: readonly SkippedFile[];
+}
+
+/**
+ * Walk `dir` and produce ContextFiles. Resolves the anydoc handle once per source
+ * (not per file) and threads it into the per-file router.
+ */
+export async function packDirectory(
+  dir: string,
+  pathPrefix: string,
+  signal?: AbortSignal,
+  anydoc?: AnydocHandle | null,
+): Promise<PackDirResult> {
+  const paths = await enumerateFiles(dir, signal);
+  if (paths.length === 0) {
+    return Object.freeze({
+      files: Object.freeze([]),
+      chars: 0,
+      documents: 0,
+      converted: 0,
+      skipped: Object.freeze([]),
+    });
+  }
+
+  // One handle check per source, not per file.
+  const handle = anydoc !== undefined ? anydoc : await getAnydoc();
+  const gate = new Semaphore(CONVERT_CONCURRENCY);
+  // Pre-allocated slots; compacted in one pass with a single length trim — house style.
+  const slots = new Array<ContextFile | undefined>(paths.length);
+  const drops = new Array<SkippedFile | undefined>(paths.length);
+  const didConvert = new Array<boolean>(paths.length); // true = fresh anydoc conversion
+  const isDoc = new Array<boolean>(paths.length); // true = document-type entry in payload
+
+  await Promise.all(paths.map((rel, i) => gate.run(async () => {
+    if (signal?.aborted) {
+      drops[i] = Object.freeze({ path: applyPathPrefix(rel, pathPrefix), reason: "aborted" });
+      return;
+    }
+    // Sensitive deny-list on the enumerated name — second net under .gitignore.
+    if (isSensitivePath(rel)) {
+      drops[i] = Object.freeze({ path: applyPathPrefix(rel, pathPrefix), reason: "sensitive" });
+      return;
+    }
+    const abs = join(dir, rel);
+    // Symlink safety: realpath, refuse escape of pack root, re-check sensitive on target.
+    const safety = await checkPathSafety(abs, dir);
+    if (!safety.ok) {
+      drops[i] = Object.freeze({ path: applyPathPrefix(rel, pathPrefix), reason: safety.reason });
+      return;
+    }
+    const readAbs = safety.realAbs;
+    // Document router FIRST — a .docx is binary and would be dropped by the text path.
+    // When anydoc is absent, still detect by extension so documents skip as "no-converter".
+    const format = handle?.formatFromPath(rel) ?? documentExtFromPath(rel);
+    if (format !== null) {
+      const doc = await documentToContextFile(readAbs, rel, pathPrefix, handle);
+      if (doc.ok) {
+        slots[i] = doc.value;
+        didConvert[i] = doc.converted;
+        isDoc[i] = true;
+      } else {
+        drops[i] = doc.skipped;
+      }
+      return;
+    }
+    const text = await textToContextFile(readAbs, rel, pathPrefix);
+    if (text.ok) {
+      slots[i] = text.value;
+      didConvert[i] = false;
+      isDoc[i] = false;
+    } else {
+      drops[i] = text.skipped;
+    }
+  })));
+
+  const files = new Array<ContextFile>(paths.length);
+  const skipped = new Array<SkippedFile>(paths.length);
+  let nFiles = 0;
+  let nSkip = 0;
+  let chars = 0;
+  let converted = 0;
+  let documents = 0;
+  for (let i = 0; i < paths.length; i++) {
+    const f = slots[i];
+    if (f !== undefined) {
+      files[nFiles++] = f;
+      chars += f.content.length;
+      if (didConvert[i]) converted += 1;
+      if (isDoc[i]) documents += 1;
+      continue;
+    }
+    const d = drops[i];
+    // Filter unreadable from the model-facing skip list (ENOENT on deleted-tracked paths).
+    // Cap so an asset-heavy repo cannot flood the wire / reply frame.
+    if (d !== undefined && d.reason !== "unreadable" && nSkip < MAX_SKIPPED_REPORTED) {
+      skipped[nSkip++] = d;
+    }
+  }
+  files.length = nFiles;
+  skipped.length = nSkip;
+  return Object.freeze({
+    files: Object.freeze(files),
+    chars,
+    documents,
+    converted,
+    skipped: Object.freeze(skipped),
+  });
+}
+
+/** Pack a directory into a SourceResult with derived (or explicit) namespace. */
+export async function sourceDir(
+  dir: string,
+  source: string,
+  opts: ResolveOpts,
+): Promise<SourceResult> {
+  const sourceId = contextSourceId(source, dir);
+  // Explicit pathPrefix (including "") wins; otherwise derive ctx/<id>/.
+  const pathPrefix = opts.pathPrefix !== undefined ? opts.pathPrefix : pathPrefixFor(sourceId);
+  const packed = await packDirectory(dir, pathPrefix, opts.signal);
+  return Object.freeze({
+    payload: packed.files,
+    files: packed.files.length,
+    chars: packed.chars,
+    sourceId,
+    pathPrefix,
+    documents: packed.documents,
+    converted: packed.converted,
+    skipped: packed.skipped,
+  });
+}

package/src/context/source-doc.ts

@@ -0,0 +1,71 @@
+/**
+ * Document container → anydoc → Markdown → ContextFile, with on-disk MD cache.
+ *
+ * Stamp is captured BEFORE conversion so mid-edit races cannot freeze a stale body forever.
+ * Document size is capped at MAX_DOCUMENT_BYTES (walk and single-file paths share this).
+ */
+
+import type { AnydocHandle } from "./anydoc.ts";
+import { captureStamp, readMdCache, writeMdCache } from "./md-cache.ts";
+import { makeContextFile } from "./merge.ts";
+import { applyPathPrefix } from "./namespace.ts";
+import {
+  MAX_DOCUMENT_BYTES,
+  type ContextFile,
+  type SkipReason,
+  type SkippedFile,
+} from "./types.ts";
+
+export type DocFileResult =
+  | { readonly ok: true; readonly value: ContextFile; /** true only for a real anydoc conversion */ readonly converted: boolean }
+  | { readonly ok: false; readonly skipped: SkippedFile };
+
+function skipReasonFromError(err: unknown): SkipReason {
+  if (err !== null && typeof err === "object" && "code" in err) {
+    const code: unknown = err.code;
+    // Preserve anydoc ConvertErrorCode values rather than collapsing to "convert-failed".
+    if (code === "unsupported" || code === "malformed" || code === "encrypted"
+      || code === "resourceLimit" || code === "missingPart" || code === "io") {
+      return code;
+    }
+  }
+  return "convert-failed";
+}
+
+/**
+ * Convert a document file to a ContextFile via anydoc.
+ * Cache hits do not increment `converted`. Failures degrade to skipped, never throw.
+ */
+export async function documentToContextFile(
+  absPath: string,
+  relPath: string,
+  pathPrefix: string,
+  anydoc: AnydocHandle | null,
+): Promise<DocFileResult> {
+  const path = applyPathPrefix(relPath, pathPrefix);
+  if (anydoc === null) {
+    return { ok: false, skipped: Object.freeze({ path, reason: "no-converter" }) };
+  }
+
+  const stamp = await captureStamp(absPath);
+  if (stamp === undefined) {
+    return { ok: false, skipped: Object.freeze({ path, reason: "unreadable" }) };
+  }
+  if (stamp.size > MAX_DOCUMENT_BYTES) {
+    return { ok: false, skipped: Object.freeze({ path, reason: "oversized" }) };
+  }
+
+  try {
+    const cached = await readMdCache(absPath);
+    if (cached !== undefined) {
+      // Cache hit — document is in context, but no conversion was performed.
+      return { ok: true, value: makeContextFile(path, cached), converted: false };
+    }
+    const markdown = await anydoc.toMarkdown(absPath);
+    // Pass the PRE-conversion stamp — never re-stat after convert (stale-forever bug).
+    await writeMdCache(absPath, markdown, stamp);
+    return { ok: true, value: makeContextFile(path, markdown), converted: true };
+  } catch (err: unknown) {
+    return { ok: false, skipped: Object.freeze({ path, reason: skipReasonFromError(err) }) };
+  }
+}

package/src/context/source-git.ts

@@ -0,0 +1,51 @@
+/**
+ * Shallow-clone a git URL, then pack with source-dir.
+ */
+
+import { execFile } from "node:child_process";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import { errorMessage, type Result } from "../util/errors.ts";
+import { contextSourceId, pathPrefixFor } from "./namespace.ts";
+import { packDirectory } from "./source-dir.ts";
+import type { ResolveOpts, SourceResult } from "./types.ts";
+
+const execFileP = promisify(execFile);
+
+/** Shallow-clone `url` into a temp dir, pack, then remove the clone. */
+export async function sourceGit(
+  url: string,
+  opts: ResolveOpts,
+): Promise<Result<SourceResult, string>> {
+  const dir = await mkdtemp(join(tmpdir(), "rlm-ctx-"));
+  const sourceId = contextSourceId(url);
+  const pathPrefix = opts.pathPrefix !== undefined ? opts.pathPrefix : pathPrefixFor(sourceId);
+  try {
+    await execFileP("git", ["clone", "--depth", "1", "--", url, dir], {
+      signal: opts.signal,
+      timeout: 120_000,
+      // Never stall on a credential helper for a private URL — fail fast instead.
+      env: Object.freeze({ ...process.env, GIT_TERMINAL_PROMPT: "0" }),
+    });
+    const packed = await packDirectory(dir, pathPrefix, opts.signal);
+    return {
+      ok: true,
+      value: Object.freeze({
+        payload: packed.files,
+        files: packed.files.length,
+        chars: packed.chars,
+        sourceId,
+        pathPrefix,
+        documents: packed.documents,
+        converted: packed.converted,
+        skipped: packed.skipped,
+      }),
+    };
+  } catch (err: unknown) {
+    return { ok: false, error: `git clone failed for ${url} — ${errorMessage(err)}` };
+  } finally {
+    await rm(dir, { recursive: true, force: true }).catch(() => {});
+  }
+}

package/src/context/source-text.ts

@@ -0,0 +1,45 @@
+/**
+ * Plain (non-document) file → ContextFile.
+ * The isBinary probe lives HERE — documents are routed earlier by formatFromPath.
+ */
+
+import { readFile, stat } from "node:fs/promises";
+import { makeContextFile } from "./merge.ts";
+import { applyPathPrefix } from "./namespace.ts";
+import { isBinary } from "./walk.ts";
+import { MAX_WALK_FILE_BYTES, type ContextFile, type SkippedFile } from "./types.ts";
+
+export type TextFileResult =
+  | { readonly ok: true; readonly value: ContextFile }
+  | { readonly ok: false; readonly skipped: SkippedFile };
+
+/**
+ * Read a plain-text file into a ContextFile.
+ * Skips binaries (NUL probe), oversize files, and unreadable paths.
+ */
+export async function textToContextFile(
+  absPath: string,
+  relPath: string,
+  pathPrefix: string,
+  maxBytes: number = MAX_WALK_FILE_BYTES,
+): Promise<TextFileResult> {
+  const path = applyPathPrefix(relPath, pathPrefix);
+  try {
+    const s = await stat(absPath);
+    if (!s.isFile()) {
+      return { ok: false, skipped: Object.freeze({ path, reason: "unreadable" }) };
+    }
+    if (s.size > maxBytes) {
+      return { ok: false, skipped: Object.freeze({ path, reason: "oversized" }) };
+    }
+    // Binary probe AFTER the document router has already claimed known containers —
+    // a .docx IS binary; routing order in source-dir is load-bearing.
+    if (await isBinary(absPath)) {
+      return { ok: false, skipped: Object.freeze({ path, reason: "binary" }) };
+    }
+    const content = await readFile(absPath, "utf-8");
+    return { ok: true, value: makeContextFile(path, content) };
+  } catch {
+    return { ok: false, skipped: Object.freeze({ path, reason: "unreadable" }) };
+  }
+}

package/src/context/types.ts

@@ -0,0 +1,88 @@
+/**
+ * Shared DTOs for every context producer (text, document, directory, git).
+ * All producers return the same shape; nothing over ~150 lines per file.
+ */
+
+/** One file entry in the sandbox `context` list. */
+export interface ContextFile {
+  readonly path: string;
+  readonly content: string;
+  readonly tokens: number;
+}
+
+/**
+ * Why a path was not converted into a ContextFile.
+ * Closed union — never a bare string (keeps skip reasons type-safe and visible).
+ */
+export type SkipReason =
+  | "binary"
+  | "sensitive"
+  | "oversized"
+  | "unreadable"
+  | "no-converter"
+  | "aborted"
+  /** Symlink whose realpath escapes the packed root (deny-list bypass). */
+  | "symlink-escape"
+  /** anydoc ConvertErrorCode values, preserved rather than collapsed. */
+  | "unsupported"
+  | "malformed"
+  | "encrypted"
+  | "resourceLimit"
+  | "missingPart"
+  | "io"
+  /** Fallback when an anydoc rejection carries no recognised code. */
+  | "convert-failed";
+
+/** A path that was enumerated but not converted into a ContextFile. */
+export interface SkippedFile {
+  readonly path: string;
+  readonly reason: SkipReason;
+}
+
+/**
+ * Result of resolving one source (dir / file / git URL) into a sandbox-ready payload.
+ * Always a namespaced (or un-prefixed for cwd) list of ContextFile.
+ */
+export interface SourceResult {
+  readonly payload: readonly ContextFile[];
+  readonly files: number;
+  /** Sum of raw content lengths — what the model should size batches against. */
+  readonly chars: number;
+  readonly sourceId: string;
+  readonly pathPrefix: string;
+  /**
+   * Document-type files present in the payload (fresh conversions + cache hits).
+   * Distinct from `converted` so a second session over cached PDFs is not "0 documents".
+   */
+  readonly documents: number;
+  /** Documents freshly converted to Markdown this call (cache hits do NOT count). */
+  readonly converted: number;
+  /** Paths skipped (binary, sensitive, no-converter, …). Capped; unreadable filtered. */
+  readonly skipped: readonly SkippedFile[];
+}
+
+/** Options shared by every resolve / pack path. */
+export interface ResolveOpts {
+  readonly cwd: string;
+  /**
+   * Namespace under which files land. `""` marks the primary/cwd source (un-prefixed paths
+   * so search() hits remain real paths edit/write can act on). Omit to derive `ctx/<id>/`.
+   */
+  readonly pathPrefix?: string;
+  readonly signal?: AbortSignal;
+}
+
+/** Single-file sources above this must use open() + llm_query_chunked in the REPL. */
+export const MAX_CONTEXT_FILE_BYTES = 8 * 1024 * 1024;
+
+/** Per-file text size cap when walking a directory (parity with the old repomix 1MB cap). */
+export const MAX_WALK_FILE_BYTES = 1_048_576;
+
+/** Per-document size cap during a directory walk (and single-file document path). */
+export const MAX_DOCUMENT_BYTES = 64 * 1024 * 1024;
+
+/** Cap on model-facing skipped entries so an asset-heavy repo cannot flood the wire. */
+export const MAX_SKIPPED_REPORTED = 64;
+
+/** Catch-all prefix for a raw string payload with no namespace — never an identity key. */
+export const LEGACY_UNKNOWN_PREFIX = "ctx/unknown/";

package/src/context/walk.ts

@@ -0,0 +1,250 @@
+/**
+ * Native repository walker — replaces repomix.
+ *
+ * git ls-files -co --exclude-standard -z IS gitignore semantics, not an approximation.
+ * -z because a path may legally contain a newline. Tracked-but-deleted paths are listed too;
+ * they fail the later read with ENOENT and are dropped there (reason "unreadable").
+ * Returns undefined when not a git work tree → caller falls back to walkFs.
+ *
+ * A second net beneath .gitignore: isSensitivePath() denies secrets (.env*, keys, .ssh/, .aws/)
+ * that repomix used to drop via useDefaultPatterns. Applied by packDirectory, not here —
+ * enumeration is pure listing; the router reports skipped: "sensitive".
+ */
+
+import { execFile } from "node:child_process";
+import type { Dirent } from "node:fs";
+import { lstat, open, readdir, realpath, stat } from "node:fs/promises";
+import { basename, join, relative, resolve, sep } from "node:path";
+import { promisify } from "node:util";
+import type { SkipReason } from "./types.ts";
+
+const execFileP = promisify(execFile);
+
+/** 8KB probe window for the NUL-byte binary check. */
+const BINARY_PROBE_BYTES = 8 * 1024;
+
+/**
+ * Directory names ignored by the non-git fallback walk. Dot-directories are skipped
+ * by default (see walkFs) except DOT_DIR_ALLOWED; this set covers non-dot noise.
+ * Growing array is the one walkFs exception (size unknown a priori).
+ */
+const FALLBACK_IGNORED: ReadonlySet<string> = Object.freeze(new Set([
+  "node_modules", "dist", "build", "out", "coverage",
+  "__pycache__", "venv",
+]));
+
+/**
+ * Dot-directories still walked on the non-git fallback. Git trees are unaffected
+ * (ls-files already lists .github/workflows etc. when tracked/unignored).
+ */
+const DOT_DIR_ALLOWED: ReadonlySet<string> = Object.freeze(new Set([
+  ".github",
+]));
+
+/**
+ * Deny-list beneath .gitignore. Secrets that must never enter context (and therefore never
+ * reach a third-party sub-LLM API). Checked against every relative path in packDirectory.
+ */
+const SENSITIVE_BASENAME = Object.freeze([
+  /^\.env$/i,
+  /^\.env\..+/i,
+  /^id_rsa/i,
+  /^id_dsa/i,
+  /^id_ecdsa/i,
+  /^id_ed25519/i,
+  /\.pem$/i,
+  /\.key$/i,
+  /\.p12$/i,
+  /\.pfx$/i,
+  /\.ppk$/i,
+  /^\.npmrc$/i,
+  /^\.pypirc$/i,
+  /^\.netrc$/i,
+  /^netrc$/i,
+  /^\.git-credentials$/i,
+]);
+
+const SENSITIVE_DIR_SEGMENTS: ReadonlySet<string> = Object.freeze(new Set([
+  ".ssh", ".aws", ".gnupg", ".kube", ".docker",
+]));
+
+/**
+ * True when a cwd-relative path must not enter context.
+ * Matches basenames (.env*, *.pem, id_rsa*, …) and any path under .ssh/ .aws/ etc.
+ */
+export function isSensitivePath(relPath: string): boolean {
+  // Match on the path as-is. git ls-files -z already emits forward-slashed paths;
+  // walkFs normalises to POSIX. Do NOT rewrite backslashes — a POSIX filename may contain `\`.
+  const segments = relPath.split("/");
+  for (let i = 0; i < segments.length; i++) {
+    const seg = segments[i];
+    if (seg !== undefined && SENSITIVE_DIR_SEGMENTS.has(seg)) return true;
+  }
+  const base = basename(relPath);
+  for (let i = 0; i < SENSITIVE_BASENAME.length; i++) {
+    if (SENSITIVE_BASENAME[i].test(base)) return true;
+  }
+  return false;
+}
+
+/** Absolute path with no trailing slash (except root). */
+function absKey(path: string): string {
+  const r = resolve(path);
+  return r.length > 1 && (r.endsWith("/") || r.endsWith("\\")) ? r.slice(0, -1) : r;
+}
+
+/** True when `fileAbs` is `rootAbs` or a descendant (prefix + separator). */
+export function isInsideRoot(fileAbs: string, rootAbs: string): boolean {
+  const root = absKey(rootAbs);
+  const file = absKey(fileAbs);
+  if (file === root) return true;
+  const prefix = root.endsWith(sep) ? root : root + sep;
+  return file.startsWith(prefix);
+}
+
+export type PathSafety =
+  | { readonly ok: true; readonly realAbs: string }
+  | { readonly ok: false; readonly reason: Extract<SkipReason, "sensitive" | "symlink-escape" | "unreadable"> };
+
+/**
+ * lstat first; for symlinks, realpath and refuse targets that escape `packRoot` or land on
+ * a sensitive path. isSensitivePath on the link name alone is not enough — `notes.txt →
+ * /tmp/prod.env` would otherwise leak secrets past the deny-list.
+ */
+export async function checkPathSafety(absPath: string, packRoot: string): Promise<PathSafety> {
+  try {
+    const rootReal = await realpath(packRoot).catch(() => absKey(packRoot));
+    const lst = await lstat(absPath);
+    if (lst.isSymbolicLink()) {
+      let realAbs: string;
+      try {
+        realAbs = await realpath(absPath);
+      } catch {
+        return { ok: false, reason: "unreadable" };
+      }
+      if (!isInsideRoot(realAbs, rootReal)) {
+        return { ok: false, reason: "symlink-escape" };
+      }
+      // Sensitive check on the resolved path (relative to pack root) AND its basename.
+      const relFromRoot = relative(rootReal, realAbs).split(sep).join("/");
+      if (isSensitivePath(relFromRoot) || isSensitivePath(basename(realAbs))) {
+        return { ok: false, reason: "sensitive" };
+      }
+      return { ok: true, realAbs };
+    }
+    // Non-symlink: still resolve for a stable absolute path.
+    const realAbs = await realpath(absPath).catch(() => absKey(absPath));
+    return { ok: true, realAbs };
+  } catch {
+    return { ok: false, reason: "unreadable" };
+  }
+}
+
+/**
+ * List tracked + untracked, non-ignored files via git. Returns undefined when `cwd` is not a
+ * git work tree (or git is unavailable) so the caller can fall back to walkFs.
+ *
+ * Paths are returned raw from git (forward-slashed). Sensitive paths are NOT filtered here —
+ * packDirectory reports them as skipped: "sensitive" so the drop is visible.
+ */
+export async function gitFiles(cwd: string, signal?: AbortSignal): Promise<readonly string[] | undefined> {
+  try {
+    const { stdout } = await execFileP(
+      "git",
+      ["ls-files", "-co", "--exclude-standard", "-z"],
+      { cwd, signal, maxBuffer: 64 * 1024 * 1024, encoding: "buffer" },
+    );
+    if (stdout.length === 0) return Object.freeze([]);
+    // Split on NUL; drop the trailing empty segment git always emits after the last path.
+    // git ls-files -z emits raw unescaped paths, already forward-slashed — never rewrite `\`.
+    const parts = stdout.toString("utf-8").split("\0");
+    const out = new Array<string>(parts.length);
+    let n = 0;
+    for (let i = 0; i < parts.length; i++) {
+      const p = parts[i];
+      if (p !== undefined && p !== "") out[n++] = p;
+    }
+    out.length = n;
+    return Object.freeze(out);
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Stack-based recursive walk. Skips:
+ *  - FALLBACK_IGNORED non-dot dirs (node_modules, dist, …)
+ *  - Dot-directories except DOT_DIR_ALLOWED (.github) — .ssh/.aws/.git stay off the walk
+ * Returns cwd-relative POSIX paths. Sensitive *files* and escaping symlinks are reported
+ * by packDirectory after checkPathSafety.
+ */
+export async function walkFs(root: string, signal?: AbortSignal): Promise<readonly string[]> {
+  const files: string[] = [];
+  const stack: string[] = [root];
+  while (stack.length > 0) {
+    if (signal?.aborted) break;
+    const dir = stack.pop();
+    if (dir === undefined) break;
+    let entries: Dirent[];
+    try {
+      // Explicit encoding keeps Dirent.name as string under @types/node ≥20.
+      entries = await readdir(dir, { withFileTypes: true, encoding: "utf8" });
+    } catch {
+      continue;
+    }
+    for (const entry of entries) {
+      const name = entry.name;
+      if (name === "." || name === "..") continue;
+      const abs = join(dir, name);
+      if (entry.isDirectory()) {
+        if (FALLBACK_IGNORED.has(name)) continue;
+        // Dot-dirs blocked except carve-outs (.github/workflows is often the analysis target).
+        if (name.startsWith(".") && !DOT_DIR_ALLOWED.has(name)) continue;
+        stack.push(abs);
+        continue;
+      }
+      if (!entry.isFile() && !entry.isSymbolicLink()) continue;
+      const rel = relative(root, abs).split(sep).join("/");
+      if (rel !== "" && !rel.startsWith("..")) files.push(rel);
+    }
+  }
+  return Object.freeze(files);
+}
+
+/**
+ * Open once, read 8KB, return true if any NUL byte is present.
+ * Failures (unreadable, gone) return false — the later text read will surface ENOENT.
+ */
+export async function isBinary(absPath: string): Promise<boolean> {
+  let fh: Awaited<ReturnType<typeof open>> | undefined;
+  try {
+    fh = await open(absPath, "r");
+    const buf = Buffer.allocUnsafe(BINARY_PROBE_BYTES);
+    const { bytesRead } = await fh.read(buf, 0, BINARY_PROBE_BYTES, 0);
+    return buf.subarray(0, bytesRead).includes(0);
+  } catch {
+    return false;
+  } finally {
+    await fh?.close().catch(() => {});
+  }
+}
+
+/**
+ * Enumerate files under `root`: git ls-files when possible, walkFs otherwise.
+ * Paths are cwd-relative POSIX.
+ */
+export async function enumerateFiles(root: string, signal?: AbortSignal): Promise<readonly string[]> {
+  const fromGit = await gitFiles(root, signal);
+  if (fromGit !== undefined) return fromGit;
+  return await walkFs(root, signal);
+}
+
+/** True when a path exists and is a regular file (or symlink to one). */
+export async function isRegularFile(absPath: string): Promise<boolean> {
+  try {
+    const s = await stat(absPath);
+    return s.isFile();
+  } catch {
+    return false;
+  }
+}