@euconform/core 1.2.0 → 1.3.0

package/src/scanner/detectors/shared.ts

@@ -0,0 +1,45 @@
+import type { ConfidenceLevel } from "../../evidence/types";
+
+/** Source-code extensions used by ai-provider, ai-framework, rag, compliance detectors */
+export const SOURCE_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".jsx"]);
+
+/** Extended set including config formats, used by framework and local-inference detectors */
+export const EXTENDED_EXTENSIONS = new Set([
+  ".ts",
+  ".tsx",
+  ".js",
+  ".jsx",
+  ".json",
+  ".yaml",
+  ".yml",
+]);
+
+/** Standardized snippet truncation length */
+export const MAX_SNIPPET_LENGTH = 200;
+
+/** Numeric ranking for confidence levels (higher = more confident) */
+export const CONFIDENCE_RANK: Record<ConfidenceLevel, number> = {
+  high: 2,
+  medium: 1,
+  low: 0,
+};
+
+/** Return the higher of two confidence levels */
+export function higherConfidence(a: ConfidenceLevel, b: ConfidenceLevel): ConfidenceLevel {
+  return CONFIDENCE_RANK[a] >= CONFIDENCE_RANK[b] ? a : b;
+}
+
+/** Escape special regex characters in a string */
+export function escapeRegex(str: string): string {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/** Check whether an extension is a standard source file */
+export function isSourceFile(ext: string): boolean {
+  return SOURCE_EXTENSIONS.has(ext);
+}
+
+/** Check whether an extension is in the extended (source + config) set */
+export function isExtendedSourceFile(ext: string): boolean {
+  return EXTENDED_EXTENSIONS.has(ext);
+}

package/src/scanner/discovery.ts

@@ -0,0 +1,290 @@
+/**
+ * File discovery for the EuConform scanner.
+ *
+ * Walks a target directory, respects .gitignore, filters by extension
+ * and named files, and returns file contents for pattern matching.
+ */
+
+import { readFile, readdir, stat } from "node:fs/promises";
+import { basename, extname, join, resolve } from "node:path";
+import ignore from "ignore";
+import type { FileProvenance, ScanFile, ScanOptions, ScanScope } from "../evidence/types";
+
+const DEFAULT_EXTENSIONS = [".ts", ".tsx", ".js", ".jsx", ".json", ".md", ".yaml", ".yml"];
+
+/** Named files to always include regardless of extension */
+const NAMED_FILE_PATTERNS = [
+  /^package\.json$/,
+  /^pnpm-lock\.yaml$/,
+  /^package-lock\.json$/,
+  /^yarn\.lock$/,
+  /^\.env\.example$/,
+  /^\.env\.sample$/,
+  /^next\.config\./,
+  /^vite\.config\./,
+  /^docker-compose/,
+  /^Dockerfile/,
+  /^tsconfig.*\.json$/,
+];
+
+const ALWAYS_SKIP_DIRS = [
+  "node_modules",
+  ".git",
+  "dist",
+  ".next",
+  "build",
+  "coverage",
+  ".turbo",
+  ".cache",
+  ".euconform",
+];
+
+const DEFAULT_MAX_FILE_SIZE = 1_048_576; // 1 MB
+const DEFAULT_MAX_FILES = 10_000;
+const DEFAULT_SCOPE: ScanScope = "production";
+
+const PRODUCTION_SKIP_PROVENANCE = new Set<FileProvenance>([
+  "test",
+  "fixture",
+  "example",
+  "generated",
+]);
+
+const TEST_SEGMENTS = new Set(["test", "tests", "__test__", "__tests__", "spec", "__spec__"]);
+const FIXTURE_SEGMENTS = new Set([
+  "fixture",
+  "fixtures",
+  "__fixture__",
+  "__fixtures__",
+  "mocks",
+  "__mocks__",
+  "mock-data",
+]);
+const EXAMPLE_SEGMENTS = new Set(["example", "examples", "sample", "samples", "demo", "demos"]);
+const GENERATED_SEGMENTS = new Set(["generated", "__generated__", ".generated", "gen"]);
+const TOOLING_SEGMENTS = new Set([".github", "scripts", "tools", "tooling", "bin"]);
+
+const TEST_FILE_PATTERN = /(?:^|\.)(?:test|spec)\.[^.]+$/i;
+const GENERATED_FILE_PATTERN = /(?:^|\.)(?:generated|min)\.[^.]+$/i;
+
+const BINARY_EXTENSIONS = new Set([
+  ".png",
+  ".jpg",
+  ".jpeg",
+  ".gif",
+  ".ico",
+  ".svg",
+  ".woff",
+  ".woff2",
+  ".ttf",
+  ".eot",
+  ".zip",
+  ".tar",
+  ".gz",
+  ".br",
+  ".pdf",
+  ".doc",
+  ".docx",
+  ".mp3",
+  ".mp4",
+  ".webm",
+  ".ogg",
+  ".wasm",
+  ".node",
+  ".dylib",
+  ".so",
+  ".lock", // bun.lockb is binary
+]);
+
+function isBinaryExtension(ext: string): boolean {
+  return BINARY_EXTENSIONS.has(ext);
+}
+
+function isNamedFile(fileName: string): boolean {
+  return NAMED_FILE_PATTERNS.some((pattern) => pattern.test(fileName));
+}
+
+function shouldIncludeFile(fileName: string, ext: string, extensions: string[]): boolean {
+  if (isNamedFile(fileName)) return true;
+  if (isBinaryExtension(ext)) return false;
+  return extensions.includes(ext);
+}
+
+export function classifyFileProvenance(relativePath: string): FileProvenance {
+  const normalized = relativePath.replace(/\\/g, "/");
+  const rawSegments = normalized.split("/").filter(Boolean);
+  const segments = rawSegments.map((segment) => segment.toLowerCase());
+  const rawFileName = rawSegments[rawSegments.length - 1] ?? "";
+  const fileName = segments[segments.length - 1] ?? "";
+  const ext = extname(fileName);
+  const isRootFile = segments.length === 1;
+
+  if (segments.some((segment) => FIXTURE_SEGMENTS.has(segment))) return "fixture";
+  if (segments.some((segment) => TEST_SEGMENTS.has(segment)) || TEST_FILE_PATTERN.test(fileName)) {
+    return "test";
+  }
+  if (segments.some((segment) => EXAMPLE_SEGMENTS.has(segment))) return "example";
+  if (
+    segments.some((segment) => GENERATED_SEGMENTS.has(segment)) ||
+    GENERATED_FILE_PATTERN.test(fileName)
+  ) {
+    return "generated";
+  }
+  if (segments.some((segment) => TOOLING_SEGMENTS.has(segment))) return "tooling";
+
+  if (fileName.startsWith("readme") && isRootFile) return "root-docs";
+  if (ext === ".md") return "docs";
+
+  if (
+    isNamedFile(rawFileName) ||
+    ext === ".yaml" ||
+    ext === ".yml" ||
+    fileName.startsWith(".env") ||
+    /\.config\.[^.]+$/i.test(rawFileName)
+  ) {
+    return "config";
+  }
+
+  return "runtime";
+}
+
+async function loadGitignore(targetPath: string): Promise<ReturnType<typeof ignore>> {
+  const ig = ignore();
+  try {
+    const content = await readFile(join(targetPath, ".gitignore"), "utf-8");
+    ig.add(content);
+  } catch {
+    // no .gitignore — that's fine
+  }
+  // Always ignore these regardless
+  ig.add(ALWAYS_SKIP_DIRS.map((d) => `${d}/`));
+  return ig;
+}
+
+export interface DiscoveryResult {
+  files: ScanFile[];
+  skippedCount: number;
+}
+
+interface FileFilterOptions {
+  scope: ScanScope;
+  extensions: string[];
+  maxSize: number;
+}
+
+function shouldProcessEntry(
+  entry: string,
+  targetPath: string,
+  filterOpts: FileFilterOptions
+): { absolutePath: string; ext: string; provenance: FileProvenance } | "skip" | "ignore" {
+  const absolutePath = join(targetPath, entry);
+  const fileName = basename(entry);
+  const ext = extname(fileName).toLowerCase();
+  const provenance = classifyFileProvenance(entry);
+
+  if (filterOpts.scope === "production" && PRODUCTION_SKIP_PROVENANCE.has(provenance)) {
+    return "skip";
+  }
+  if (!shouldIncludeFile(fileName, ext, filterOpts.extensions)) return "ignore";
+
+  return { absolutePath, ext, provenance };
+}
+
+async function tryReadFile(
+  absolutePath: string,
+  entry: string,
+  ext: string,
+  provenance: FileProvenance,
+  maxSize: number
+): Promise<ScanFile | "skip" | "ignore"> {
+  try {
+    const fileStat = await stat(absolutePath);
+    if (!fileStat.isFile() || fileStat.size === 0) return "ignore";
+    if (fileStat.size > maxSize) return "skip";
+
+    const content = await readFile(absolutePath, "utf-8");
+    return {
+      relativePath: entry,
+      absolutePath,
+      extension: ext,
+      content,
+      sizeBytes: fileStat.size,
+      provenance,
+    };
+  } catch {
+    return "skip";
+  }
+}
+
+function resolveOptions(options: ScanOptions) {
+  return {
+    targetPath: resolve(options.targetPath),
+    scope: options.scope ?? DEFAULT_SCOPE,
+    extensions: options.extensions ?? DEFAULT_EXTENSIONS,
+    maxSize: options.maxFileSizeBytes ?? DEFAULT_MAX_FILE_SIZE,
+    maxFiles: options.maxFiles ?? DEFAULT_MAX_FILES,
+    excludeGlobs: options.excludeGlobs,
+  };
+}
+
+async function readDirectoryEntries(targetPath: string): Promise<string[]> {
+  try {
+    return (await readdir(targetPath, { recursive: true, encoding: "utf-8" })) as string[];
+  } catch (err) {
+    throw new Error(`Cannot read directory: ${targetPath}: ${err}`);
+  }
+}
+
+function processEntry(
+  entry: string,
+  ig: ReturnType<typeof ignore>,
+  targetPath: string,
+  filterOpts: FileFilterOptions
+): "skip" | "ignore" | { absolutePath: string; ext: string; provenance: FileProvenance } {
+  if (ig.ignores(entry)) return "ignore";
+  return shouldProcessEntry(entry, targetPath, filterOpts);
+}
+
+export async function discoverFiles(options: ScanOptions): Promise<DiscoveryResult> {
+  const { targetPath, scope, extensions, maxSize, maxFiles, excludeGlobs } =
+    resolveOptions(options);
+
+  const ig = await loadGitignore(targetPath);
+  if (excludeGlobs?.length) ig.add(excludeGlobs);
+
+  const entries = await readDirectoryEntries(targetPath);
+  const filterOpts: FileFilterOptions = { scope, extensions, maxSize };
+  const files: ScanFile[] = [];
+  let skippedCount = 0;
+
+  for (const entry of entries) {
+    if (files.length >= maxFiles) {
+      skippedCount++;
+      continue;
+    }
+
+    const processed = processEntry(entry, ig, targetPath, filterOpts);
+    if (processed === "skip") {
+      skippedCount++;
+      continue;
+    }
+    if (processed === "ignore") continue;
+
+    const result = await tryReadFile(
+      processed.absolutePath,
+      entry,
+      processed.ext,
+      processed.provenance,
+      maxSize
+    );
+    if (result === "skip") {
+      skippedCount++;
+      continue;
+    }
+    if (result === "ignore") continue;
+
+    files.push(result);
+  }
+
+  return { files, skippedCount };
+}

package/src/scanner/index.ts

@@ -0,0 +1,9 @@
+/**
+ * @euconform/core/scanner — Local repository scanner
+ *
+ * Scans a project directory for AI components, compliance signals,
+ * and regulatory hints. Produces evidence, not verdicts.
+ */
+
+export { scanRepository } from "./aggregator";
+export { discoverFiles } from "./discovery";

package/tests/evidence/bundle.test.ts

@@ -0,0 +1,128 @@
+import { describe, expect, it } from "vitest";
+import { buildBundleManifest } from "../../src/evidence/bundle";
+
+const REPORT_CONTENT = JSON.stringify({
+  schemaVersion: "euconform.report.v1",
+  generatedAt: "2026-01-01T00:00:00Z",
+});
+
+const AIBOM_CONTENT = JSON.stringify({
+  schemaVersion: "euconform.aibom.v1",
+  generatedAt: "2026-01-01T00:00:00Z",
+});
+
+const CI_CONTENT = JSON.stringify({
+  schemaVersion: "euconform.ci.v1",
+  generatedAt: "2026-01-01T00:00:00Z",
+});
+
+const SUMMARY_CONTENT = "# EuConform Scan Report\n\nSummary here.";
+
+const BASE_OPTS = {
+  tool: { name: "euconform", version: "1.0.0" },
+  target: { name: "test-project", rootPath: "/test" },
+  generatedAt: "2026-01-01T00:00:00Z",
+};
+
+describe("buildBundleManifest", () => {
+  it("produces a valid bundle with all artifacts", () => {
+    const bundle = buildBundleManifest({
+      ...BASE_OPTS,
+      report: { content: REPORT_CONTENT, fileName: "euconform.report.json" },
+      aibom: { content: AIBOM_CONTENT, fileName: "euconform.aibom.json" },
+      ci: { content: CI_CONTENT, fileName: "euconform.ci.json" },
+      summary: { content: SUMMARY_CONTENT, fileName: "euconform.summary.md" },
+    });
+
+    expect(bundle.schemaVersion).toBe("euconform.bundle.v1");
+    expect(bundle.generatedAt).toBe("2026-01-01T00:00:00Z");
+    expect(bundle.tool).toEqual({ name: "euconform", version: "1.0.0" });
+    expect(bundle.target).toEqual({ name: "test-project", rootPath: "/test" });
+    expect(bundle.artifacts).toHaveLength(4);
+  });
+
+  it("sets report as required and others as optional", () => {
+    const bundle = buildBundleManifest({
+      ...BASE_OPTS,
+      report: { content: REPORT_CONTENT, fileName: "euconform.report.json" },
+      aibom: { content: AIBOM_CONTENT, fileName: "euconform.aibom.json" },
+    });
+
+    const report = bundle.artifacts.find((a) => a.role === "report");
+    const aibom = bundle.artifacts.find((a) => a.role === "aibom");
+    expect(report?.required).toBe(true);
+    expect(aibom?.required).toBe(false);
+  });
+
+  it("produces valid SHA-256 hex hashes", () => {
+    const bundle = buildBundleManifest({
+      ...BASE_OPTS,
+      report: { content: REPORT_CONTENT, fileName: "euconform.report.json" },
+    });
+
+    for (const artifact of bundle.artifacts) {
+      expect(artifact.sha256).toMatch(/^[a-f0-9]{64}$/);
+    }
+  });
+
+  it("extracts schemaVersion from JSON artifacts", () => {
+    const bundle = buildBundleManifest({
+      ...BASE_OPTS,
+      report: { content: REPORT_CONTENT, fileName: "euconform.report.json" },
+      aibom: { content: AIBOM_CONTENT, fileName: "euconform.aibom.json" },
+      ci: { content: CI_CONTENT, fileName: "euconform.ci.json" },
+    });
+
+    expect(bundle.artifacts.find((a) => a.role === "report")?.schemaVersion).toBe(
+      "euconform.report.v1"
+    );
+    expect(bundle.artifacts.find((a) => a.role === "aibom")?.schemaVersion).toBe(
+      "euconform.aibom.v1"
+    );
+    expect(bundle.artifacts.find((a) => a.role === "ci")?.schemaVersion).toBe("euconform.ci.v1");
+  });
+
+  it("sets mimeType for non-JSON artifacts", () => {
+    const bundle = buildBundleManifest({
+      ...BASE_OPTS,
+      report: { content: REPORT_CONTENT, fileName: "euconform.report.json" },
+      summary: { content: SUMMARY_CONTENT, fileName: "euconform.summary.md" },
+    });
+
+    const summary = bundle.artifacts.find((a) => a.role === "summary");
+    expect(summary?.mimeType).toBe("text/markdown");
+    expect(summary?.schemaVersion).toBeUndefined();
+
+    const report = bundle.artifacts.find((a) => a.role === "report");
+    expect(report?.mimeType).toBeUndefined();
+  });
+
+  it("excludes missing optional artifacts", () => {
+    const bundle = buildBundleManifest({
+      ...BASE_OPTS,
+      report: { content: REPORT_CONTENT, fileName: "euconform.report.json" },
+    });
+
+    expect(bundle.artifacts).toHaveLength(1);
+    expect(bundle.artifacts[0]?.role).toBe("report");
+  });
+
+  it("produces different hashes for different content", () => {
+    const bundle1 = buildBundleManifest({
+      ...BASE_OPTS,
+      report: {
+        content: '{"schemaVersion":"euconform.report.v1","a":1}',
+        fileName: "euconform.report.json",
+      },
+    });
+    const bundle2 = buildBundleManifest({
+      ...BASE_OPTS,
+      report: {
+        content: '{"schemaVersion":"euconform.report.v1","a":2}',
+        fileName: "euconform.report.json",
+      },
+    });
+
+    expect(bundle1.artifacts[0]?.sha256).not.toBe(bundle2.artifacts[0]?.sha256);
+  });
+});