@untangle-code/scanner 0.0.0-stage → 0.1.0

package/src/ignore-engine.ts

@@ -0,0 +1,157 @@
+/**
+ * Ignore engine implementing the priority order of spec.md §6.2:
+ *
+ *   1. security/system excludes         — never overridable
+ *   2. explicit config excludes         — `RepositoryConfig.exclude`
+ *   3. `.untangleignore`
+ *   4. `.gitignore`
+ *   5. default excludes                 — spec.md §6.3
+ *   6. config includes override (5) only, never (1)-(4)
+ *
+ * Implementation model: layers are applied from lowest to highest priority
+ * (5 → 6 → 4 → 3 → 2 → 1); a layer only changes the running decision when
+ * it actually has an opinion (some pattern in it matched), otherwise the
+ * decision made by a lower-priority layer is kept. This makes "higher
+ * priority" mean exactly "wins whenever it has an opinion", matching the
+ * spec text without requiring gitignore-style whole-tree renegotiation.
+ *
+ * See README, "Entscheidungen" for the two points spec.md leaves implicit
+ * and that this engine fixes: (a) which of the §6.3 default-exclude names
+ * are promoted to the non-overridable "security/system" tier, and (b) that
+ * config includes are read as overriding *only* the default-exclude layer,
+ * per the literal wording of §6.2 item 6.
+ */
+import {
+  compileGlobList,
+  compilePatterns,
+  evaluateLayer,
+  type CompiledPattern,
+} from "./gitignore-pattern.js";
+import type { ExclusionReason, IgnoreConfig, IgnoreDecision } from "./types.js";
+
+/**
+ * Non-overridable system/VCS directories (spec.md §6.2 item 1). A subset of
+ * §6.3's list is promoted here rather than left in the overridable default
+ * tier — see README, "Entscheidungen" (decision 1).
+ */
+const SECURITY_EXCLUDE_NAMES = [".git", ".svn", ".hg"];
+
+/** Overridable default excludes (spec.md §6.3), minus the security/system tier above. */
+const DEFAULT_EXCLUDE_NAMES = [
+  "node_modules",
+  "vendor",
+  "dist",
+  "build",
+  "out",
+  "coverage",
+  ".cache",
+  ".next",
+  ".nuxt",
+  "target",
+  ".gradle",
+  ".venv",
+  "venv",
+  "__pycache__",
+];
+
+/** "generated assets" (spec.md §6.3) — common generated/minified/output file patterns. */
+const DEFAULT_GENERATED_ASSET_PATTERNS = ["**/*.min.js", "**/*.min.css", "**/*.map"];
+
+interface LayerFrame {
+  size: number;
+}
+
+/**
+ * Stateful ignore evaluator. `pushGitignoreLayer`/`popGitignoreLayer` (and
+ * the `untangleignore` equivalents) let a directory walker maintain, as a
+ * stack, exactly the nested-ignore-file patterns that apply along the
+ * current root-to-here path — deeper files are pushed later and therefore
+ * win ties within their own layer, matching git's "nearer file wins"
+ * behavior.
+ */
+export class IgnoreEngine {
+  private readonly securityPatterns: CompiledPattern[];
+  private readonly defaultPatterns: CompiledPattern[];
+  private readonly configIncludePatterns: CompiledPattern[];
+  private readonly configExcludePatterns: CompiledPattern[];
+
+  private readonly gitignoreStack: CompiledPattern[] = [];
+  private readonly gitignoreFrames: LayerFrame[] = [];
+  private readonly untangleignoreStack: CompiledPattern[] = [];
+  private readonly untangleignoreFrames: LayerFrame[] = [];
+
+  public constructor(config?: IgnoreConfig) {
+    this.securityPatterns = compileGlobList(SECURITY_EXCLUDE_NAMES);
+    this.defaultPatterns = compileGlobList([
+      ...DEFAULT_EXCLUDE_NAMES,
+      ...DEFAULT_GENERATED_ASSET_PATTERNS,
+      ...(config?.additionalDefaultExcludes ?? []),
+    ]);
+    this.configIncludePatterns = compileGlobList(config?.configInclude ?? []);
+    this.configExcludePatterns = compileGlobList(config?.configExclude ?? []);
+  }
+
+  public pushGitignoreLayer(baseDir: string, content: string): void {
+    const patterns = compilePatterns(content, baseDir);
+    this.gitignoreFrames.push({ size: patterns.length });
+    this.gitignoreStack.push(...patterns);
+  }
+
+  public popGitignoreLayer(): void {
+    const frame = this.gitignoreFrames.pop();
+    if (frame !== undefined) {
+      this.gitignoreStack.length -= frame.size;
+    }
+  }
+
+  public pushUntangleIgnoreLayer(baseDir: string, content: string): void {
+    const patterns = compilePatterns(content, baseDir);
+    this.untangleignoreFrames.push({ size: patterns.length });
+    this.untangleignoreStack.push(...patterns);
+  }
+
+  public popUntangleIgnoreLayer(): void {
+    const frame = this.untangleignoreFrames.pop();
+    if (frame !== undefined) {
+      this.untangleignoreStack.length -= frame.size;
+    }
+  }
+
+  public decide(relativePath: string, isDirectory: boolean): IgnoreDecision {
+    let ignored = evaluateLayer(this.defaultPatterns, relativePath, isDirectory) ?? false;
+    let reason: ExclusionReason | undefined = ignored ? "default-exclude" : undefined;
+
+    if (evaluateLayer(this.configIncludePatterns, relativePath, isDirectory) === true) {
+      ignored = false;
+      reason = undefined;
+    }
+
+    const gitDecision = evaluateLayer(this.gitignoreStack, relativePath, isDirectory);
+    if (gitDecision !== undefined) {
+      ignored = gitDecision;
+      reason = gitDecision ? "gitignore" : undefined;
+    }
+
+    const untangleDecision = evaluateLayer(this.untangleignoreStack, relativePath, isDirectory);
+    if (untangleDecision !== undefined) {
+      ignored = untangleDecision;
+      reason = untangleDecision ? "untangleignore" : undefined;
+    }
+
+    if (evaluateLayer(this.configExcludePatterns, relativePath, isDirectory) === true) {
+      ignored = true;
+      reason = "config-exclude";
+    }
+
+    if (evaluateLayer(this.securityPatterns, relativePath, isDirectory) === true) {
+      ignored = true;
+      reason = "security";
+    }
+
+    return reason === undefined ? { ignored } : { ignored, reason };
+  }
+}
+
+export function createIgnoreEngine(config?: IgnoreConfig): IgnoreEngine {
+  return new IgnoreEngine(config);
+}

package/src/index.ts

@@ -0,0 +1,48 @@
+/**
+ * @untangle-code/scanner — recursive discovery, ignore rules, symlink
+ * safety, binary/language classification and the content-hash pipeline
+ * (spec.md §6.1–§6.3, §12.1; architecture.md §6.4, §23.1).
+ *
+ * Pure core: reads only through `FileSystemPort` (domain), never
+ * `node:fs`. A concrete Node adapter is published separately from
+ * `@untangle-code/scanner/node-fs` (see that module's docs) — importing it
+ * here would make this entry point Node-specific and untestable against
+ * arbitrary `FileSystemPort` implementations (e.g. an in-memory one).
+ */
+
+// Scanner-local contracts (not domain contracts — see README, "Entscheidungen").
+export * from "./types.js";
+
+// Discovery entry point.
+export { discoverFiles } from "./discovery.js";
+
+// Ignore engine (spec.md §6.2 priority layering).
+export { IgnoreEngine, createIgnoreEngine } from "./ignore-engine.js";
+export {
+  compilePattern,
+  compilePatterns,
+  compileGlobList,
+  evaluateLayer,
+  type CompiledPattern,
+} from "./gitignore-pattern.js";
+
+// Content classification, usable standalone (e.g. for re-classifying a
+// buffer already in memory, without going through discovery).
+export {
+  classifyBinarySample,
+  detectMinified,
+  type MinifiedHeuristics,
+} from "./binary-classifier.js";
+export {
+  detectLanguage,
+  detectLanguageByExtension,
+  detectLanguageFromContent,
+  type LanguageDetectionResult,
+} from "./language-classifier.js";
+
+// Hash pipeline (spec.md §12.1).
+export { hashBytes, createPortContentHasher } from "./hashing.js";
+
+// Bounded-concurrency primitives (architecture.md §13.1–§13.2).
+export { Semaphore } from "./semaphore.js";
+export { mapConcurrent } from "./concurrency.js";

package/src/language-classifier.ts

@@ -0,0 +1,137 @@
+/**
+ * Coarse language classification (spec.md §6.1, §7.2): an extension map for
+ * the normal case, plus a shebang/marker content sniff for extensionless
+ * files. This is a fast pre-classification for the scanner's own output —
+ * the authoritative language/adapter dispatch belongs to parser-registry
+ * (architecture.md §6.5), which is free to disagree.
+ */
+import type { LanguageDetectionSource } from "./types.js";
+
+// MVP + Public Beta languages (spec.md §7.2) plus common structural/text
+// formats the scanner needs to tell apart from opaque binaries.
+const EXTENSION_LANGUAGE_MAP: Readonly<Record<string, string>> = {
+  ts: "typescript",
+  mts: "typescript",
+  cts: "typescript",
+  tsx: "tsx",
+  js: "javascript",
+  mjs: "javascript",
+  cjs: "javascript",
+  jsx: "jsx",
+  py: "python",
+  pyi: "python",
+  pyw: "python",
+  go: "go",
+  java: "java",
+  kt: "kotlin",
+  kts: "kotlin",
+  cs: "csharp",
+  rs: "rust",
+  php: "php",
+  phtml: "php",
+  rb: "ruby",
+  // WP-86: die Parser-Registry fuehrt diese Endungen als Ruby beziehungsweise
+  // PHP und Swift. Fehlten sie hier, meldete der Scanner "unknown" fuer eine
+  // Datei, die der Adapter danach doch tief analysiert — zwei Aussagen ueber
+  // dieselbe Datei, von denen eine falsch ist.
+  rake: "ruby",
+  gemspec: "ruby",
+  swift: "swift",
+  c: "c",
+  h: "c",
+  cpp: "cpp",
+  cc: "cpp",
+  cxx: "cpp",
+  hpp: "cpp",
+  hh: "cpp",
+  hxx: "cpp",
+  json: "json",
+  jsonc: "json",
+  yaml: "yaml",
+  yml: "yaml",
+  toml: "toml",
+  md: "markdown",
+  mdx: "markdown",
+  css: "css",
+  scss: "scss",
+  less: "less",
+  html: "html",
+  htm: "html",
+  vue: "vue",
+  svelte: "svelte",
+  sh: "shell",
+  bash: "shell",
+  zsh: "shell",
+  ksh: "shell",
+  sql: "sql",
+  graphql: "graphql",
+  gql: "graphql",
+  proto: "protobuf",
+  xml: "xml",
+};
+
+/** Extension-based language guess; `undefined` for extensionless or unknown extensions. */
+export function detectLanguageByExtension(relativePath: string): string | undefined {
+  const base = relativePath.slice(relativePath.lastIndexOf("/") + 1);
+  const dot = base.lastIndexOf(".");
+  // dot <= 0 covers "no extension" and dotfiles like ".gitignore".
+  if (dot <= 0) {
+    return undefined;
+  }
+  const ext = base.slice(dot + 1).toLowerCase();
+  return EXTENSION_LANGUAGE_MAP[ext];
+}
+
+interface ContentMarker {
+  pattern: RegExp;
+  languageId: string;
+}
+
+const SHEBANG_MARKERS: readonly ContentMarker[] = [
+  { pattern: /^#!.*\bnode\b/, languageId: "javascript" },
+  { pattern: /^#!.*\bdeno\b/, languageId: "typescript" },
+  { pattern: /^#!.*\bpython[0-9.]*\b/, languageId: "python" },
+  { pattern: /^#!.*\b(?:bash|sh|zsh|dash|ksh)\b/, languageId: "shell" },
+  { pattern: /^#!.*\bruby\b/, languageId: "ruby" },
+  { pattern: /^#!.*\bperl\b/, languageId: "perl" },
+];
+
+const CONTENT_MARKERS: readonly ContentMarker[] = [
+  { pattern: /^<\?php/, languageId: "php" },
+  { pattern: /^<\?xml/, languageId: "xml" },
+  { pattern: /^\s*<!DOCTYPE html/i, languageId: "html" },
+];
+
+/** Content-based language guess for extensionless files (shebang or leading markers). */
+export function detectLanguageFromContent(text: string): string | undefined {
+  const head = text.slice(0, 256);
+  for (const marker of SHEBANG_MARKERS) {
+    if (marker.pattern.test(head)) {
+      return marker.languageId;
+    }
+  }
+  for (const marker of CONTENT_MARKERS) {
+    if (marker.pattern.test(head)) {
+      return marker.languageId;
+    }
+  }
+  return undefined;
+}
+
+export interface LanguageDetectionResult {
+  languageId: string;
+  detectedBy: LanguageDetectionSource;
+}
+
+/** Combines extension and content sniffing; `languageId` is "unknown" if neither matches. */
+export function detectLanguage(relativePath: string, sampleText: string): LanguageDetectionResult {
+  const byExtension = detectLanguageByExtension(relativePath);
+  if (byExtension !== undefined) {
+    return { languageId: byExtension, detectedBy: "extension" };
+  }
+  const byContent = detectLanguageFromContent(sampleText);
+  if (byContent !== undefined) {
+    return { languageId: byContent, detectedBy: "content" };
+  }
+  return { languageId: "unknown", detectedBy: "unknown" };
+}

package/src/node-fs/index.ts

@@ -0,0 +1,9 @@
+/**
+ * `@untangle-code/scanner/node-fs` — the Node.js-specific half of the
+ * scanner: a `FileSystemPort` adapter and a genuinely streaming
+ * `ContentHasher`. Kept out of the package's main entry point so that
+ * entry point stays testable against any `FileSystemPort` (e.g. an
+ * in-memory one) without pulling in `node:fs` (see README, "Entscheidungen").
+ */
+export { NodeFileSystem } from "./node-file-system.js";
+export { createNodeStreamingHasher } from "./node-streaming-hasher.js";

package/src/node-fs/node-file-system.ts

@@ -0,0 +1,137 @@
+/**
+ * Node.js implementation of `FileSystemPort` (domain). This is the only
+ * file in the package allowed to import `node:fs` — the core (`../index.ts`)
+ * stays testable against any port implementation, this adapter is what
+ * production code actually wires up.
+ */
+import * as nodeFs from "node:fs/promises";
+import * as nodePath from "node:path";
+
+import type {
+  DirEntryV1,
+  FileStatV1,
+  FileSystemPort,
+  PortCallOptions,
+} from "@untangle-code/domain";
+
+interface NodeStatsLike {
+  isDirectory(): boolean;
+  isFile(): boolean;
+  isSymbolicLink(): boolean;
+  size: number;
+  mtimeMs: number;
+}
+
+function kindOf(stats: NodeStatsLike): FileStatV1["kind"] {
+  if (stats.isSymbolicLink()) {
+    return "symlink";
+  }
+  if (stats.isDirectory()) {
+    return "directory";
+  }
+  if (stats.isFile()) {
+    return "file";
+  }
+  return "other";
+}
+
+function toFileStat(path: string, stats: NodeStatsLike): FileStatV1 {
+  return {
+    path,
+    sizeBytes: stats.size,
+    mtimeEpochMs: Math.trunc(stats.mtimeMs),
+    kind: kindOf(stats),
+  };
+}
+
+function isNotFoundError(error: unknown): boolean {
+  return (
+    typeof error === "object" &&
+    error !== null &&
+    "code" in error &&
+    (error as { code?: unknown }).code === "ENOENT"
+  );
+}
+
+/** Production `FileSystemPort` adapter backed by `node:fs/promises`. */
+export class NodeFileSystem implements FileSystemPort {
+  public async stat(path: string, options?: PortCallOptions): Promise<FileStatV1 | null> {
+    options?.signal?.throwIfAborted();
+    try {
+      const stats = await nodeFs.stat(path);
+      return toFileStat(path, stats);
+    } catch (error) {
+      if (isNotFoundError(error)) {
+        return null;
+      }
+      throw error;
+    }
+  }
+
+  public async lstat(path: string, options?: PortCallOptions): Promise<FileStatV1 | null> {
+    options?.signal?.throwIfAborted();
+    try {
+      const stats = await nodeFs.lstat(path);
+      return toFileStat(path, stats);
+    } catch (error) {
+      if (isNotFoundError(error)) {
+        return null;
+      }
+      throw error;
+    }
+  }
+
+  public async readFile(path: string, options?: PortCallOptions): Promise<Uint8Array> {
+    const buffer = await nodeFs.readFile(path, { signal: options?.signal });
+    return new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+  }
+
+  public async readTextFile(path: string, options?: PortCallOptions): Promise<string> {
+    return await nodeFs.readFile(path, { encoding: "utf8", signal: options?.signal });
+  }
+
+  public async readDir(path: string, options?: PortCallOptions): Promise<DirEntryV1[]> {
+    options?.signal?.throwIfAborted();
+    const entries = await nodeFs.readdir(path, { withFileTypes: true });
+    return entries.map((entry) => {
+      const kind: DirEntryV1["kind"] = entry.isSymbolicLink()
+        ? "symlink"
+        : entry.isDirectory()
+          ? "directory"
+          : entry.isFile()
+            ? "file"
+            : "other";
+      return { name: entry.name, path: nodePath.join(path, entry.name), kind };
+    });
+  }
+
+  public async realPath(path: string, options?: PortCallOptions): Promise<string> {
+    options?.signal?.throwIfAborted();
+    return await nodeFs.realpath(path);
+  }
+
+  public async writeFile(path: string, data: Uint8Array, options?: PortCallOptions): Promise<void> {
+    await nodeFs.writeFile(path, data, { signal: options?.signal });
+  }
+
+  public async mkdir(
+    path: string,
+    options?: PortCallOptions & { recursive?: boolean },
+  ): Promise<void> {
+    options?.signal?.throwIfAborted();
+    await nodeFs.mkdir(path, { recursive: options?.recursive ?? false });
+  }
+
+  public async rename(from: string, to: string, options?: PortCallOptions): Promise<void> {
+    options?.signal?.throwIfAborted();
+    await nodeFs.rename(from, to);
+  }
+
+  public async remove(
+    path: string,
+    options?: PortCallOptions & { recursive?: boolean },
+  ): Promise<void> {
+    options?.signal?.throwIfAborted();
+    await nodeFs.rm(path, { recursive: options?.recursive ?? false, force: true });
+  }
+}

package/src/node-fs/node-streaming-hasher.ts

@@ -0,0 +1,35 @@
+/**
+ * Streaming SHA-256 content hasher for the Node adapter.
+ *
+ * `FileSystemPort.readFile` (domain) only exposes whole-buffer reads, so
+ * the port-based default hasher (`createPortContentHasher`, in the pure
+ * core) must materialize a file's bytes to hash them. This implementation
+ * hashes straight off `fs.createReadStream` instead, without ever holding
+ * the whole file in memory — the genuine "streaming hashing for large
+ * files" the WP-05 order asks for (see README, "Entscheidungen").
+ */
+import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+
+import type { ContentHasher } from "../types.js";
+
+/** `ContentHasher` that streams the file instead of buffering it whole. */
+export function createNodeStreamingHasher(): ContentHasher {
+  return {
+    hashFile(absolutePath, _sizeBytes, options) {
+      return new Promise<string>((resolve, reject) => {
+        const hash = createHash("sha256");
+        const stream = createReadStream(absolutePath, { signal: options?.signal });
+        stream.on("data", (chunk: string | Buffer) => {
+          hash.update(typeof chunk === "string" ? Buffer.from(chunk) : chunk);
+        });
+        stream.on("error", (error) => {
+          reject(error);
+        });
+        stream.on("end", () => {
+          resolve(hash.digest("hex"));
+        });
+      });
+    },
+  };
+}

package/src/port-options.ts

@@ -0,0 +1,10 @@
+/**
+ * Builds a `PortCallOptions` from an optional `AbortSignal` without ever
+ * setting `signal: undefined` explicitly — required because domain's
+ * `PortCallOptions` is checked under `exactOptionalPropertyTypes`.
+ */
+import type { PortCallOptions } from "@untangle-code/domain";
+
+export function portOptions(signal: AbortSignal | undefined): PortCallOptions | undefined {
+  return signal === undefined ? undefined : { signal };
+}

package/src/semaphore.ts

@@ -0,0 +1,45 @@
+/**
+ * A minimal counting semaphore used to bound I/O parallelism during
+ * discovery (spec.md §6.1; architecture.md §13.1 "bounded queues").
+ *
+ * Deliberately hand-rolled per WP-05 order ("ein einfacher, eigenen
+ * Semaphor/Pool") instead of pulling in a scheduling dependency.
+ */
+export class Semaphore {
+  private available: number;
+  private readonly waiters: Array<() => void> = [];
+
+  public constructor(concurrency: number) {
+    if (!Number.isInteger(concurrency) || concurrency < 1) {
+      throw new RangeError(`Semaphore concurrency must be a positive integer, got ${concurrency}`);
+    }
+    this.available = concurrency;
+  }
+
+  /** Resolves once a slot is free; returns the matching release function. */
+  public acquire(): Promise<() => void> {
+    if (this.available > 0) {
+      this.available -= 1;
+      return Promise.resolve(() => this.release());
+    }
+    return new Promise<() => void>((resolve) => {
+      this.waiters.push(() => {
+        this.available -= 1;
+        resolve(() => this.release());
+      });
+    });
+  }
+
+  private release(): void {
+    this.available += 1;
+    const next = this.waiters.shift();
+    if (next !== undefined) {
+      next();
+    }
+  }
+
+  /** Number of currently free slots — exposed for tests only. */
+  public get availableCount(): number {
+    return this.available;
+  }
+}