@untangle-code/scanner 0.0.0-stage → 0.1.0

package/src/discovery.ts

@@ -0,0 +1,362 @@
+/**
+ * Recursive discovery (spec.md §6.1; architecture.md §6.4, §23.1).
+ *
+ * Walks the repository tree through `FileSystemPort` only (never
+ * `node:fs`), applies the ignore engine, resolves and guards symlinks, and
+ * hands each surviving path to bounded-concurrency classification/hashing.
+ */
+import {
+  classifyTextFormat,
+  assertWithinWorkspace,
+  isUntangleError,
+  type FileStatV1,
+  type FileSystemPort,
+} from "@untangle-code/domain";
+
+import { classifyBinarySample, detectMinified } from "./binary-classifier.js";
+import { mapConcurrent } from "./concurrency.js";
+import { createPortContentHasher } from "./hashing.js";
+import type { IgnoreEngine } from "./ignore-engine.js";
+import { createIgnoreEngine } from "./ignore-engine.js";
+import { detectLanguage } from "./language-classifier.js";
+import { portOptions } from "./port-options.js";
+import type {
+  ContentHasher,
+  ScannedFileV1,
+  ScanOptions,
+  ScanSkipV1,
+  TextEncodingGuess,
+} from "./types.js";
+
+interface WalkDir {
+  relativePath: string;
+  absolutePath: string;
+}
+
+interface FileCandidate {
+  relativePath: string;
+  absolutePath: string;
+  /** Already-known stat (symlink targets are stat'd during resolution); avoids a redundant call. */
+  stat?: FileStatV1;
+}
+
+interface WalkContext {
+  fs: FileSystemPort;
+  ignoreEngine: IgnoreEngine;
+  roots: readonly string[];
+  visitedRealDirs: Set<string>;
+  gitIgnoreFileName: string;
+  untangleIgnoreFileName: string;
+  onSkip?: ((skip: ScanSkipV1) => void) | undefined;
+  signal?: AbortSignal | undefined;
+}
+
+function skip(
+  context: WalkContext,
+  relativePath: string,
+  reason: ScanSkipV1["reason"],
+  detail?: string,
+): void {
+  context.onSkip?.(
+    detail === undefined ? { relativePath, reason } : { relativePath, reason, detail },
+  );
+}
+
+async function* walkDirectory(context: WalkContext, dir: WalkDir): AsyncGenerator<FileCandidate> {
+  context.signal?.throwIfAborted();
+  const entries = await context.fs.readDir(dir.absolutePath, portOptions(context.signal));
+
+  let pushedGitignore = false;
+  let pushedUntangleIgnore = false;
+  for (const entry of entries) {
+    if (entry.kind === "file" && entry.name === context.gitIgnoreFileName) {
+      const content = await context.fs.readTextFile(entry.path, portOptions(context.signal));
+      context.ignoreEngine.pushGitignoreLayer(dir.relativePath, content);
+      pushedGitignore = true;
+    } else if (entry.kind === "file" && entry.name === context.untangleIgnoreFileName) {
+      const content = await context.fs.readTextFile(entry.path, portOptions(context.signal));
+      context.ignoreEngine.pushUntangleIgnoreLayer(dir.relativePath, content);
+      pushedUntangleIgnore = true;
+    }
+  }
+
+  try {
+    for (const entry of entries) {
+      context.signal?.throwIfAborted();
+      const childRelative =
+        dir.relativePath === "" ? entry.name : `${dir.relativePath}/${entry.name}`;
+
+      if (entry.kind === "directory") {
+        const decision = context.ignoreEngine.decide(childRelative, true);
+        if (decision.ignored) {
+          skip(context, childRelative, decision.reason ?? "default-exclude");
+          continue;
+        }
+        if (context.visitedRealDirs.has(entry.path)) {
+          skip(context, childRelative, "symlink-cycle", "directory real path already visited");
+          continue;
+        }
+        context.visitedRealDirs.add(entry.path);
+        yield* walkDirectory(context, { relativePath: childRelative, absolutePath: entry.path });
+        continue;
+      }
+
+      if (entry.kind === "symlink") {
+        yield* handleSymlink(context, entry.path, childRelative);
+        continue;
+      }
+
+      if (entry.kind === "file") {
+        const decision = context.ignoreEngine.decide(childRelative, false);
+        if (decision.ignored) {
+          skip(context, childRelative, decision.reason ?? "default-exclude");
+          continue;
+        }
+        yield { relativePath: childRelative, absolutePath: entry.path };
+        continue;
+      }
+
+      skip(
+        context,
+        childRelative,
+        "unreadable",
+        `unsupported directory entry kind "${entry.kind}"`,
+      );
+    }
+  } finally {
+    if (pushedUntangleIgnore) {
+      context.ignoreEngine.popUntangleIgnoreLayer();
+    }
+    if (pushedGitignore) {
+      context.ignoreEngine.popGitignoreLayer();
+    }
+  }
+}
+
+async function* handleSymlink(
+  context: WalkContext,
+  entryAbsolutePath: string,
+  childRelative: string,
+): AsyncGenerator<FileCandidate> {
+  let real: string;
+  try {
+    real = await context.fs.realPath(entryAbsolutePath, portOptions(context.signal));
+  } catch {
+    skip(context, childRelative, "unreadable", "symlink target could not be resolved");
+    return;
+  }
+
+  try {
+    assertWithinWorkspace(real, context.roots);
+  } catch (error) {
+    if (isUntangleError(error) && error.code === "PATH_OUTSIDE_WORKSPACE") {
+      skip(context, childRelative, "symlink-escape", real);
+      return;
+    }
+    throw error;
+  }
+
+  const targetStat = await context.fs.stat(real, portOptions(context.signal));
+  if (targetStat === null) {
+    skip(context, childRelative, "unreadable", "dangling symlink");
+    return;
+  }
+
+  if (targetStat.kind === "directory") {
+    const decision = context.ignoreEngine.decide(childRelative, true);
+    if (decision.ignored) {
+      skip(context, childRelative, decision.reason ?? "default-exclude");
+      return;
+    }
+    if (context.visitedRealDirs.has(real)) {
+      skip(context, childRelative, "symlink-cycle", real);
+      return;
+    }
+    context.visitedRealDirs.add(real);
+    yield* walkDirectory(context, { relativePath: childRelative, absolutePath: real });
+    return;
+  }
+
+  if (targetStat.kind === "file") {
+    const decision = context.ignoreEngine.decide(childRelative, false);
+    if (decision.ignored) {
+      skip(context, childRelative, decision.reason ?? "default-exclude");
+      return;
+    }
+    yield { relativePath: childRelative, absolutePath: real, stat: targetStat };
+    return;
+  }
+
+  skip(
+    context,
+    childRelative,
+    "unreadable",
+    `symlink target has unsupported kind "${targetStat.kind}"`,
+  );
+}
+
+/**
+ * Decodes a content sample as text using the *detected* encoding
+ * (WP-64 fix, spec.md E4: silent data loss is worse than a slightly slower
+ * decode). Before this fix, the sample was always decoded as UTF-8
+ * regardless of `classifyBinarySample`'s detected `encoding`, which produced
+ * mojibake for BOM-less UTF-16 files (their bytes are ~50% NUL under a
+ * UTF-8 reading) and, worse, fed that mojibake into `detectLanguage`'s
+ * shebang sniff for extensionless scripts — silently misclassifying a
+ * legitimate `#!/usr/bin/env python3` UTF-16 script as unknown/no language.
+ *
+ * UTF-16 (LE/BE, whether BOM-confirmed or detected BOM-less by
+ * `detectUtf16WithoutBom`) decodes under its own `TextDecoder` label so the
+ * BOM — if present — is consumed correctly. Every other guess (`utf8`,
+ * `ascii`, `unknown`) falls back to UTF-8 non-fatal decoding, which is a
+ * safe superset of ASCII and the correct default per spec.md.
+ */
+function decodeSampleText(sample: Uint8Array, encoding: TextEncodingGuess): string {
+  try {
+    if (encoding === "utf16le" || encoding === "utf16be") {
+      return new TextDecoder(encoding === "utf16le" ? "utf-16le" : "utf-16be", {
+        fatal: false,
+      }).decode(sample);
+    }
+    return new TextDecoder("utf-8", { fatal: false }).decode(sample);
+  } catch {
+    return "";
+  }
+}
+
+interface ProcessConfig {
+  fs: FileSystemPort;
+  maxFileSizeBytes: number;
+  sampleBytes: number;
+  nonPrintableThreshold: number;
+  hasher: ContentHasher;
+  onSkip?: ((skip: ScanSkipV1) => void) | undefined;
+  signal?: AbortSignal | undefined;
+}
+
+async function processCandidate(
+  config: ProcessConfig,
+  candidate: FileCandidate,
+): Promise<ScannedFileV1 | null> {
+  const stat =
+    candidate.stat ?? (await config.fs.stat(candidate.absolutePath, portOptions(config.signal)));
+  if (stat === null) {
+    config.onSkip?.({
+      relativePath: candidate.relativePath,
+      reason: "unreadable",
+      detail: "vanished before stat",
+    });
+    return null;
+  }
+  if (stat.sizeBytes > config.maxFileSizeBytes) {
+    config.onSkip?.({ relativePath: candidate.relativePath, reason: "size-limit" });
+    return null;
+  }
+
+  const bytes = await config.fs.readFile(candidate.absolutePath, portOptions(config.signal));
+  const sample = bytes.subarray(0, Math.min(bytes.length, config.sampleBytes));
+  const binary = classifyBinarySample(sample, {
+    nonPrintableThreshold: config.nonPrintableThreshold,
+  });
+  if (binary.isBinary) {
+    config.onSkip?.({ relativePath: candidate.relativePath, reason: "binary" });
+    return null;
+  }
+
+  const sampleText = decodeSampleText(sample, binary.encoding);
+  const language = detectLanguage(candidate.relativePath, sampleText);
+  // Die Textklasse steht eine Zeile weiter oben schon fest und kostet nichts:
+  // sie ist rein pfadbasiert. Ohne sie entschied die Einzelzeilenregel auch
+  // ueber Prosa — und ob sie zuschlug, haing daran, ob die erste lange Zeile
+  // zufaellig in die 8-KB-Stichprobe fiel.
+  const isMinified = detectMinified(sample, undefined, classifyTextFormat(candidate.relativePath));
+  const contentHash = await config.hasher.hashFile(
+    candidate.absolutePath,
+    stat.sizeBytes,
+    portOptions(config.signal),
+  );
+
+  return {
+    relativePath: candidate.relativePath,
+    absolutePath: candidate.absolutePath,
+    sizeBytes: stat.sizeBytes,
+    mtimeEpochMs: stat.mtimeEpochMs,
+    contentHash,
+    languageId: language.languageId,
+    languageDetectedBy: language.detectedBy,
+    isMinified,
+    encoding: binary.encoding,
+    hasBom: binary.hasBom,
+  };
+}
+
+const DEFAULT_MAX_FILE_SIZE_BYTES = 5 * 1024 * 1024;
+const DEFAULT_SAMPLE_BYTES = 8192;
+const DEFAULT_CONCURRENCY = 8;
+const DEFAULT_NON_PRINTABLE_THRESHOLD = 0.3;
+
+/**
+ * Discovers files under `options.repoRoot`, applying ignore rules and
+ * symlink safety, and yields a `ScannedFileV1` for every file accepted for
+ * downstream analysis. Excluded/skipped paths are reported via `onSkip`,
+ * never yielded. Respects `options.signal` throughout.
+ */
+export async function* discoverFiles(
+  options: ScanOptions,
+): AsyncGenerator<ScannedFileV1, void, void> {
+  const maxFileSizeBytes = options.maxFileSizeBytes ?? DEFAULT_MAX_FILE_SIZE_BYTES;
+  const sampleBytes = options.sampleBytes ?? DEFAULT_SAMPLE_BYTES;
+  const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
+  const nonPrintableThreshold = options.nonPrintableThreshold ?? DEFAULT_NON_PRINTABLE_THRESHOLD;
+  const gitIgnoreFileName = options.ignore?.gitIgnoreFileName ?? ".gitignore";
+  const untangleIgnoreFileName = options.ignore?.untangleIgnoreFileName ?? ".untangleignore";
+
+  options.signal?.throwIfAborted();
+  // Step 2 (architecture.md §23.1): lexical check before any I/O.
+  assertWithinWorkspace(options.repoRoot, options.roots);
+
+  // Step 1+3: canonically resolve the roots themselves too, not just the
+  // repo root and symlink targets encountered later — a root path can
+  // itself contain an unresolved symlink component (e.g. macOS's
+  // `/var` -> `/private/var`), and comparing a realpath-resolved candidate
+  // against a *literal* root would then spuriously look like an escape.
+  const resolvedRoots = await Promise.all(
+    options.roots.map((root) => options.fs.realPath(root, portOptions(options.signal))),
+  );
+  const rootReal = await options.fs.realPath(options.repoRoot, portOptions(options.signal));
+  assertWithinWorkspace(rootReal, resolvedRoots);
+
+  const context: WalkContext = {
+    fs: options.fs,
+    ignoreEngine: createIgnoreEngine(options.ignore),
+    roots: resolvedRoots,
+    visitedRealDirs: new Set<string>([rootReal]),
+    gitIgnoreFileName,
+    untangleIgnoreFileName,
+    onSkip: options.onSkip,
+    signal: options.signal,
+  };
+
+  const candidates = walkDirectory(context, { relativePath: "", absolutePath: rootReal });
+
+  const processConfig: ProcessConfig = {
+    fs: options.fs,
+    maxFileSizeBytes,
+    sampleBytes,
+    nonPrintableThreshold,
+    hasher: options.hasher ?? createPortContentHasher(options.fs),
+    onSkip: options.onSkip,
+    signal: options.signal,
+  };
+
+  for await (const result of mapConcurrent(
+    candidates,
+    concurrency,
+    (candidate) => processCandidate(processConfig, candidate),
+    options.signal,
+  )) {
+    if (result !== null) {
+      yield result;
+    }
+  }
+}

package/src/gitignore-pattern.ts

@@ -0,0 +1,170 @@
+/**
+ * A small, self-contained gitignore-pattern compiler (spec.md §6.2:
+ * "Gitignore-Semantik selbst implementieren oder ueber die ignore-Bibliothek").
+ * Supports negation (`!pattern`), directory-only patterns (`pattern/`),
+ * root anchors (`/pattern`), `**`, `*`, `?` and `[...]` character classes.
+ *
+ * Deliberately self-implemented (no dependency) to keep full control over
+ * how multiple layers compose (see `ignore-engine.ts`), which the spec's
+ * multi-source priority model (§6.2) needs more of than a single
+ * `.gitignore` parser typically exposes.
+ */
+
+export interface CompiledPattern {
+  regex: RegExp;
+  negate: boolean;
+  dirOnly: boolean;
+  raw: string;
+}
+
+function escapeRegexLiteral(char: string): string {
+  return /[.+^${}()|[\]\\]/.test(char) ? `\\${char}` : char;
+}
+
+function escapeBaseDir(baseDir: string): string {
+  return baseDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/** Translates one gitignore glob (no leading `/`, no trailing `/`) into a regex source string. */
+function globToRegexSource(pattern: string): string {
+  let out = "";
+  for (let i = 0; i < pattern.length; i += 1) {
+    const ch = pattern[i];
+    if (ch === "*") {
+      if (pattern[i + 1] === "*") {
+        if (pattern[i + 2] === "/") {
+          out += "(?:.*/)?";
+          i += 2;
+        } else if (i + 2 >= pattern.length) {
+          out += ".*";
+          i += 1;
+        } else {
+          // "**" not followed by "/" and not at the end: treat as a plain "*".
+          out += "[^/]*";
+          i += 1;
+        }
+      } else {
+        out += "[^/]*";
+      }
+    } else if (ch === "?") {
+      out += "[^/]";
+    } else if (ch === "[") {
+      let j = i + 1;
+      let cls = "[";
+      if (pattern[j] === "!" || pattern[j] === "^") {
+        cls += "^";
+        j += 1;
+      }
+      while (j < pattern.length && pattern[j] !== "]") {
+        const c = pattern[j];
+        cls += c === "\\" ? "\\\\" : c;
+        j += 1;
+      }
+      cls += "]";
+      out += cls;
+      i = j;
+    } else if (ch === "\\" && i + 1 < pattern.length) {
+      out += escapeRegexLiteral(pattern[i + 1] as string);
+      i += 1;
+    } else {
+      out += escapeRegexLiteral(ch as string);
+    }
+  }
+  return out;
+}
+
+/**
+ * Compiles one gitignore-style line, scoped to `baseDir` (the "/"-relative
+ * directory the source `.gitignore`/`.untangleignore` file lives in, `""`
+ * for the workspace/repo root). Returns `null` for blank lines and comments.
+ */
+export function compilePattern(rawLine: string, baseDir: string): CompiledPattern | null {
+  const trimmed = rawLine.replace(/\s+$/, "");
+  if (trimmed.length === 0) {
+    return null;
+  }
+  if (trimmed.startsWith("#")) {
+    return null;
+  }
+
+  let pattern = trimmed;
+  let negate = false;
+  if (pattern.startsWith("!")) {
+    negate = true;
+    pattern = pattern.slice(1);
+  } else if (pattern.startsWith("\\!") || pattern.startsWith("\\#")) {
+    pattern = pattern.slice(1);
+  }
+
+  let dirOnly = false;
+  if (pattern.endsWith("/")) {
+    dirOnly = true;
+    pattern = pattern.slice(0, -1);
+  }
+  if (pattern.length === 0) {
+    return null;
+  }
+
+  let anchored = pattern.includes("/");
+  if (pattern.startsWith("/")) {
+    pattern = pattern.slice(1);
+    anchored = true;
+  }
+
+  const base = escapeBaseDir(baseDir);
+  const basePrefix = baseDir.length > 0 ? `${base}/` : "";
+  const body = globToRegexSource(pattern);
+
+  const source = anchored
+    ? `^${basePrefix}${body}(?:/.*)?$`
+    : `^${basePrefix}(?:.*/)?${body}(?:/.*)?$`;
+
+  return { regex: new RegExp(source), negate, dirOnly, raw: rawLine };
+}
+
+/** Compiles every line of a `.gitignore`/`.untangleignore`-style file. */
+export function compilePatterns(content: string, baseDir: string): CompiledPattern[] {
+  const patterns: CompiledPattern[] = [];
+  for (const line of content.split(/\r\n|\r|\n/)) {
+    const compiled = compilePattern(line, baseDir);
+    if (compiled !== null) {
+      patterns.push(compiled);
+    }
+  }
+  return patterns;
+}
+
+/** Compiles a flat list of plain glob strings (e.g. config include/exclude), all rooted at "". */
+export function compileGlobList(patterns: readonly string[]): CompiledPattern[] {
+  const compiled: CompiledPattern[] = [];
+  for (const pattern of patterns) {
+    const result = compilePattern(pattern, "");
+    if (result !== null) {
+      compiled.push(result);
+    }
+  }
+  return compiled;
+}
+
+/**
+ * Evaluates one layer's patterns against a path (standard gitignore
+ * semantics: later patterns override earlier ones within the same layer).
+ * Returns `undefined` when no pattern in the layer applies at all — that is
+ * the signal for callers to defer to a lower-priority layer's decision.
+ */
+export function evaluateLayer(
+  patterns: readonly CompiledPattern[],
+  relativePath: string,
+  isDirectory: boolean,
+): boolean | undefined {
+  let result: boolean | undefined;
+  for (const pattern of patterns) {
+    if (pattern.dirOnly && !isDirectory) {
+      continue;
+    }
+    if (pattern.regex.test(relativePath)) {
+      result = !pattern.negate;
+    }
+  }
+  return result;
+}

package/src/hashing.ts

@@ -0,0 +1,37 @@
+/**
+ * Content-hash pipeline (spec.md §12.1): SHA-256 of the raw file bytes is
+ * the sole source of truth for "unchanged" decisions; path/size/mtime are
+ * fast hints only.
+ *
+ * `node:crypto` is used directly here — the "no direct node I/O in the
+ * core" rule (WP-05 order) is about the *file system* (`node:fs`), so the
+ * scanner reads bytes exclusively through `FileSystemPort`. Hashing pure
+ * in-memory bytes with `node:crypto` is not I/O.
+ */
+import { createHash } from "node:crypto";
+
+import type { FileSystemPort, PortCallOptions } from "@untangle-code/domain";
+
+import type { ContentHasher } from "./types.js";
+
+/** SHA-256 hex digest of raw bytes already held in memory. */
+export function hashBytes(bytes: Uint8Array): string {
+  return createHash("sha256").update(bytes).digest("hex");
+}
+
+/**
+ * Default `ContentHasher`: reads the whole file through `FileSystemPort`
+ * and hashes the resulting buffer. This is the only option available
+ * against the abstract port (it exposes no range/stream read), so it is
+ * what every non-Node adapter (in-memory test doubles included) gets by
+ * default. Real streaming is available from `./node-fs` for the Node
+ * adapter (see README, "Entscheidungen").
+ */
+export function createPortContentHasher(fs: FileSystemPort): ContentHasher {
+  return {
+    async hashFile(absolutePath: string, _sizeBytes: number, options?: PortCallOptions) {
+      const bytes = await fs.readFile(absolutePath, options);
+      return hashBytes(bytes);
+    },
+  };
+}