@euconform/core 1.2.0 → 1.4.0

package/dist/scanner/discovery.js

@@ -0,0 +1,235 @@
+/**
+ * File discovery for the EuConform scanner.
+ *
+ * Walks a target directory, respects .gitignore, filters by extension
+ * and named files, and returns file contents for pattern matching.
+ */
+import { readFile, readdir, stat } from "node:fs/promises";
+import { basename, extname, join, resolve } from "node:path";
+import ignore from "ignore";
+const DEFAULT_EXTENSIONS = [".ts", ".tsx", ".js", ".jsx", ".json", ".md", ".yaml", ".yml"];
+/** Named files to always include regardless of extension */
+const NAMED_FILE_PATTERNS = [
+    /^package\.json$/,
+    /^pnpm-lock\.yaml$/,
+    /^package-lock\.json$/,
+    /^yarn\.lock$/,
+    /^\.env\.example$/,
+    /^\.env\.sample$/,
+    /^next\.config\./,
+    /^vite\.config\./,
+    /^docker-compose/,
+    /^Dockerfile/,
+    /^tsconfig.*\.json$/,
+];
+const ALWAYS_SKIP_DIRS = [
+    "node_modules",
+    ".git",
+    "dist",
+    ".next",
+    "build",
+    "coverage",
+    ".turbo",
+    ".cache",
+    ".euconform",
+];
+const DEFAULT_MAX_FILE_SIZE = 1_048_576; // 1 MB
+const DEFAULT_MAX_FILES = 10_000;
+const DEFAULT_SCOPE = "production";
+const PRODUCTION_SKIP_PROVENANCE = new Set([
+    "test",
+    "fixture",
+    "example",
+    "generated",
+]);
+const TEST_SEGMENTS = new Set(["test", "tests", "__test__", "__tests__", "spec", "__spec__"]);
+const FIXTURE_SEGMENTS = new Set([
+    "fixture",
+    "fixtures",
+    "__fixture__",
+    "__fixtures__",
+    "mocks",
+    "__mocks__",
+    "mock-data",
+]);
+const EXAMPLE_SEGMENTS = new Set(["example", "examples", "sample", "samples", "demo", "demos"]);
+const GENERATED_SEGMENTS = new Set(["generated", "__generated__", ".generated", "gen"]);
+const TOOLING_SEGMENTS = new Set([".github", "scripts", "tools", "tooling", "bin"]);
+const TEST_FILE_PATTERN = /(?:^|\.)(?:test|spec)\.[^.]+$/i;
+const GENERATED_FILE_PATTERN = /(?:^|\.)(?:generated|min)\.[^.]+$/i;
+const BINARY_EXTENSIONS = new Set([
+    ".png",
+    ".jpg",
+    ".jpeg",
+    ".gif",
+    ".ico",
+    ".svg",
+    ".woff",
+    ".woff2",
+    ".ttf",
+    ".eot",
+    ".zip",
+    ".tar",
+    ".gz",
+    ".br",
+    ".pdf",
+    ".doc",
+    ".docx",
+    ".mp3",
+    ".mp4",
+    ".webm",
+    ".ogg",
+    ".wasm",
+    ".node",
+    ".dylib",
+    ".so",
+    ".lock", // bun.lockb is binary
+]);
+function isBinaryExtension(ext) {
+    return BINARY_EXTENSIONS.has(ext);
+}
+function isNamedFile(fileName) {
+    return NAMED_FILE_PATTERNS.some((pattern) => pattern.test(fileName));
+}
+function shouldIncludeFile(fileName, ext, extensions) {
+    if (isNamedFile(fileName))
+        return true;
+    if (isBinaryExtension(ext))
+        return false;
+    return extensions.includes(ext);
+}
+export function classifyFileProvenance(relativePath) {
+    const normalized = relativePath.replace(/\\/g, "/");
+    const rawSegments = normalized.split("/").filter(Boolean);
+    const segments = rawSegments.map((segment) => segment.toLowerCase());
+    const rawFileName = rawSegments[rawSegments.length - 1] ?? "";
+    const fileName = segments[segments.length - 1] ?? "";
+    const ext = extname(fileName);
+    const isRootFile = segments.length === 1;
+    if (segments.some((segment) => FIXTURE_SEGMENTS.has(segment)))
+        return "fixture";
+    if (segments.some((segment) => TEST_SEGMENTS.has(segment)) || TEST_FILE_PATTERN.test(fileName)) {
+        return "test";
+    }
+    if (segments.some((segment) => EXAMPLE_SEGMENTS.has(segment)))
+        return "example";
+    if (segments.some((segment) => GENERATED_SEGMENTS.has(segment)) ||
+        GENERATED_FILE_PATTERN.test(fileName)) {
+        return "generated";
+    }
+    if (segments.some((segment) => TOOLING_SEGMENTS.has(segment)))
+        return "tooling";
+    if (fileName.startsWith("readme") && isRootFile)
+        return "root-docs";
+    if (ext === ".md")
+        return "docs";
+    if (isNamedFile(rawFileName) ||
+        ext === ".yaml" ||
+        ext === ".yml" ||
+        fileName.startsWith(".env") ||
+        /\.config\.[^.]+$/i.test(rawFileName)) {
+        return "config";
+    }
+    return "runtime";
+}
+async function loadGitignore(targetPath) {
+    const ig = ignore();
+    try {
+        const content = await readFile(join(targetPath, ".gitignore"), "utf-8");
+        ig.add(content);
+    }
+    catch {
+        // no .gitignore — that's fine
+    }
+    // Always ignore these regardless
+    ig.add(ALWAYS_SKIP_DIRS.map((d) => `${d}/`));
+    return ig;
+}
+function shouldProcessEntry(entry, targetPath, filterOpts) {
+    const absolutePath = join(targetPath, entry);
+    const fileName = basename(entry);
+    const ext = extname(fileName).toLowerCase();
+    const provenance = classifyFileProvenance(entry);
+    if (filterOpts.scope === "production" && PRODUCTION_SKIP_PROVENANCE.has(provenance)) {
+        return "skip";
+    }
+    if (!shouldIncludeFile(fileName, ext, filterOpts.extensions))
+        return "ignore";
+    return { absolutePath, ext, provenance };
+}
+async function tryReadFile(absolutePath, entry, ext, provenance, maxSize) {
+    try {
+        const fileStat = await stat(absolutePath);
+        if (!fileStat.isFile() || fileStat.size === 0)
+            return "ignore";
+        if (fileStat.size > maxSize)
+            return "skip";
+        const content = await readFile(absolutePath, "utf-8");
+        return {
+            relativePath: entry,
+            absolutePath,
+            extension: ext,
+            content,
+            sizeBytes: fileStat.size,
+            provenance,
+        };
+    }
+    catch {
+        return "skip";
+    }
+}
+function resolveOptions(options) {
+    return {
+        targetPath: resolve(options.targetPath),
+        scope: options.scope ?? DEFAULT_SCOPE,
+        extensions: options.extensions ?? DEFAULT_EXTENSIONS,
+        maxSize: options.maxFileSizeBytes ?? DEFAULT_MAX_FILE_SIZE,
+        maxFiles: options.maxFiles ?? DEFAULT_MAX_FILES,
+        excludeGlobs: options.excludeGlobs,
+    };
+}
+async function readDirectoryEntries(targetPath) {
+    try {
+        return (await readdir(targetPath, { recursive: true, encoding: "utf-8" }));
+    }
+    catch (err) {
+        throw new Error(`Cannot read directory: ${targetPath}: ${err}`);
+    }
+}
+function processEntry(entry, ig, targetPath, filterOpts) {
+    if (ig.ignores(entry))
+        return "ignore";
+    return shouldProcessEntry(entry, targetPath, filterOpts);
+}
+export async function discoverFiles(options) {
+    const { targetPath, scope, extensions, maxSize, maxFiles, excludeGlobs } = resolveOptions(options);
+    const ig = await loadGitignore(targetPath);
+    if (excludeGlobs?.length)
+        ig.add(excludeGlobs);
+    const entries = await readDirectoryEntries(targetPath);
+    const filterOpts = { scope, extensions, maxSize };
+    const files = [];
+    let skippedCount = 0;
+    for (const entry of entries) {
+        if (files.length >= maxFiles) {
+            skippedCount++;
+            continue;
+        }
+        const processed = processEntry(entry, ig, targetPath, filterOpts);
+        if (processed === "skip") {
+            skippedCount++;
+            continue;
+        }
+        if (processed === "ignore")
+            continue;
+        const result = await tryReadFile(processed.absolutePath, entry, processed.ext, processed.provenance, maxSize);
+        if (result === "skip") {
+            skippedCount++;
+            continue;
+        }
+        if (result === "ignore")
+            continue;
+        files.push(result);
+    }
+    return { files, skippedCount };
+}

package/dist/scanner/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * @euconform/core/scanner — Local repository scanner
+ *
+ * Scans a project directory for AI components, compliance signals,
+ * and regulatory hints. Produces evidence, not verdicts.
+ */
+export { scanRepository } from "./aggregator";
+export { discoverFiles } from "./discovery";
+//# sourceMappingURL=index.d.ts.map

package/dist/scanner/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/scanner/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC"}

package/dist/scanner/index.js

@@ -0,0 +1,8 @@
+/**
+ * @euconform/core/scanner — Local repository scanner
+ *
+ * Scans a project directory for AI components, compliance signals,
+ * and regulatory hints. Produces evidence, not verdicts.
+ */
+export { scanRepository } from "./aggregator";
+export { discoverFiles } from "./discovery";

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "@euconform/core",
-  "version": "1.2.0",
+  "version": "1.4.0",
   "license": "MIT OR EUPL-1.2",
   "author": "Benedikt Hiepler",
   "description": "Core engine for EU AI Act compliance, risk classification, and fairness metrics.",
   "type": "module",
+  "sideEffects": false,
   "keywords": [
     "eu-ai-act",
     "compliance",
@@ -26,7 +27,10 @@
     ".": "./src/index.ts",
     "./risk-engine": "./src/risk-engine.ts",
     "./fairness-metrics": "./src/fairness-metrics.ts",
-    "./types": "./src/types.ts"
+    "./types": "./src/types.ts",
+    "./scanner": "./src/scanner/index.ts",
+    "./evidence": "./src/evidence/index.ts",
+    "./datasets": "./src/datasets/crows-pairs.ts"
   },
   "devDependencies": {
     "@types/node": "^22.15.30",
@@ -36,7 +40,8 @@
     "@euconform/typescript-config": "0.0.0"
   },
   "dependencies": {
-    "@xenova/transformers": "^2.17.2"
+    "@xenova/transformers": "^2.17.2",
+    "ignore": "^7.0.3"
   },
   "scripts": {
     "build": "tsc",

package/src/datasets/crows-pairs.ts

@@ -0,0 +1,64 @@
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import type { CrowsPairsEntry } from "../types";
+
+interface CrowsPairsDataset {
+  metadata: {
+    name: string;
+    description: string;
+    version: string;
+    license: string;
+    source: string;
+  };
+  pairs: Array<{
+    id: number;
+    stereotype: string;
+    antiStereotype: string;
+    biasType: string;
+  }>;
+}
+
+function findDatasetPath(language: "en" | "de"): string {
+  const fileName = `crows-pairs-${language}.json`;
+
+  // Strategy 1: Relative to this source file (works in unbundled tsc builds)
+  const currentDir = dirname(fileURLToPath(import.meta.url));
+  const fromSource = resolve(currentDir, `../../assets/datasets/${fileName}`);
+  if (existsSync(fromSource)) return fromSource;
+
+  // Strategy 2: Relative to process.cwd() (works in monorepo context)
+  const fromCwd = resolve(process.cwd(), `packages/core/assets/datasets/${fileName}`);
+  if (existsSync(fromCwd)) return fromCwd;
+
+  // Strategy 3: Walk up from the entry script to find the monorepo root
+  if (process.argv[1]) {
+    let dir = dirname(resolve(process.argv[1]));
+    for (let i = 0; i < 5; i++) {
+      const candidate = resolve(dir, `packages/core/assets/datasets/${fileName}`);
+      if (existsSync(candidate)) return candidate;
+      dir = dirname(dir);
+    }
+  }
+
+  throw new Error(
+    `CrowS-Pairs dataset (${language}) not found. Ensure packages/core/assets/datasets/ exists.`
+  );
+}
+
+/**
+ * Load CrowS-Pairs dataset from bundled assets.
+ * Converts the structured JSON format to CrowsPairsEntry[] for the bias engine.
+ */
+export function loadCrowsPairsDataset(language: "en" | "de"): CrowsPairsEntry[] {
+  const filePath = findDatasetPath(language);
+  const raw = readFileSync(filePath, "utf-8");
+  const dataset: CrowsPairsDataset = JSON.parse(raw);
+
+  return dataset.pairs.map((pair) => ({
+    id: pair.id,
+    sent_more: pair.stereotype,
+    sent_less: pair.antiStereotype,
+    bias_type: pair.biasType,
+  }));
+}

package/src/evidence/bundle.ts

@@ -0,0 +1,65 @@
+import { sha256Hex } from "./hash";
+import type { BundleArtifactRef, BundleArtifactRole, ScanBundle } from "./types";
+
+interface ArtifactInput {
+  content: string;
+  fileName: string;
+}
+
+interface BuildBundleOptions {
+  report: ArtifactInput;
+  aibom?: ArtifactInput;
+  ci?: ArtifactInput;
+  summary?: ArtifactInput;
+  tool: { name: string; version: string };
+  target: { name: string; rootPath: string };
+  generatedAt: string;
+}
+
+function extractSchemaVersion(content: string): string | undefined {
+  try {
+    const parsed = JSON.parse(content) as { schemaVersion?: string };
+    return typeof parsed.schemaVersion === "string" ? parsed.schemaVersion : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function makeArtifactRef(
+  role: BundleArtifactRole,
+  input: ArtifactInput,
+  required: boolean
+): BundleArtifactRef {
+  const isJson = input.fileName.endsWith(".json");
+  const ref: BundleArtifactRef = {
+    role,
+    fileName: input.fileName,
+    sha256: sha256Hex(input.content),
+    required,
+  };
+
+  if (isJson) {
+    const sv = extractSchemaVersion(input.content);
+    if (sv) ref.schemaVersion = sv;
+  } else {
+    ref.mimeType = "text/markdown";
+  }
+
+  return ref;
+}
+
+export function buildBundleManifest(opts: BuildBundleOptions): ScanBundle {
+  const artifacts: BundleArtifactRef[] = [makeArtifactRef("report", opts.report, true)];
+
+  if (opts.aibom) artifacts.push(makeArtifactRef("aibom", opts.aibom, false));
+  if (opts.ci) artifacts.push(makeArtifactRef("ci", opts.ci, false));
+  if (opts.summary) artifacts.push(makeArtifactRef("summary", opts.summary, false));
+
+  return {
+    schemaVersion: "euconform.bundle.v1",
+    generatedAt: opts.generatedAt,
+    tool: opts.tool,
+    target: opts.target,
+    artifacts,
+  };
+}

package/src/evidence/compliance-evaluation.ts

@@ -0,0 +1,114 @@
+import type { DetectedSignal, FileProvenance, SignalCategory, SignalEvidence } from "./types";
+
+export const IMPLEMENTATION_PROVENANCE = new Set<FileProvenance>(["runtime", "config"]);
+const SUPPORTING_PROVENANCE = new Set<FileProvenance>(["root-docs", "docs"]);
+const USER_FACING_SEGMENTS = new Set([
+  "app",
+  "pages",
+  "routes",
+  "route",
+  "components",
+  "ui",
+  "views",
+  "view",
+  "screens",
+  "screen",
+  "templates",
+  "template",
+  "public",
+  "content",
+  "locales",
+  "messages",
+  "copy",
+  "emails",
+]);
+const USER_FACING_BASENAME_PATTERN =
+  /(?:page|layout|route|template|loading|error|view|screen|dialog|modal|form|widget|banner|notice)\.[^.]+$/i;
+
+function normalizedProvenance(provenance?: FileProvenance): FileProvenance {
+  return provenance ?? "runtime";
+}
+
+function isInternalAnalysisPath(relativePath: string): boolean {
+  const normalized = relativePath.replace(/\\/g, "/").toLowerCase();
+  return normalized.includes("/scanner/") || normalized.includes("/detectors/");
+}
+
+export function isUserFacingPath(relativePath: string): boolean {
+  const normalized = relativePath.replace(/\\/g, "/").toLowerCase();
+  const segments = normalized.split("/").filter(Boolean);
+  const basename = segments[segments.length - 1] ?? "";
+
+  return (
+    segments.some((segment) => USER_FACING_SEGMENTS.has(segment)) ||
+    USER_FACING_BASENAME_PATTERN.test(basename)
+  );
+}
+
+export function isImplementationEvidence(
+  category: SignalCategory,
+  evidence: SignalEvidence
+): boolean {
+  const provenance = normalizedProvenance(evidence.provenance);
+  if (!IMPLEMENTATION_PROVENANCE.has(provenance)) return false;
+  if (isInternalAnalysisPath(evidence.file)) return false;
+
+  if (category === "compliance-disclosure") {
+    return isUserFacingPath(evidence.file);
+  }
+
+  return true;
+}
+
+export function isSupportingEvidence(category: SignalCategory, evidence: SignalEvidence): boolean {
+  const provenance = normalizedProvenance(evidence.provenance);
+  if (!SUPPORTING_PROVENANCE.has(provenance)) return false;
+
+  if (category === "compliance-disclosure") return true;
+  if (category.startsWith("compliance-")) return true;
+
+  return false;
+}
+
+function withFilteredEvidence(
+  signal: DetectedSignal,
+  matcher: (evidence: SignalEvidence) => boolean
+): DetectedSignal | null {
+  const evidence = signal.evidence.filter(matcher);
+  if (evidence.length === 0) return null;
+  return { ...signal, evidence };
+}
+
+export interface EvaluatedSignalSet {
+  implementationSignals: DetectedSignal[];
+  supportingSignals: DetectedSignal[];
+}
+
+export function evaluateSignalsForCategories(
+  signals: DetectedSignal[],
+  categories: SignalCategory[]
+): EvaluatedSignalSet {
+  const relevant = signals.filter((signal) => categories.includes(signal.category));
+
+  return {
+    implementationSignals: relevant
+      .map((signal) =>
+        withFilteredEvidence(signal, (evidence) =>
+          isImplementationEvidence(signal.category, evidence)
+        )
+      )
+      .filter((signal): signal is DetectedSignal => signal !== null),
+    supportingSignals: relevant
+      .map((signal) =>
+        withFilteredEvidence(signal, (evidence) => isSupportingEvidence(signal.category, evidence))
+      )
+      .filter((signal): signal is DetectedSignal => signal !== null),
+  };
+}
+
+export function hasImplementationSignals(
+  signals: DetectedSignal[],
+  categories: SignalCategory[]
+): boolean {
+  return evaluateSignalsForCategories(signals, categories).implementationSignals.length > 0;
+}

package/src/evidence/hash.ts

@@ -0,0 +1,5 @@
+import { createHash } from "node:crypto";
+
+export function sha256Hex(content: string): string {
+  return createHash("sha256").update(content, "utf8").digest("hex");
+}

package/src/evidence/index.ts

@@ -0,0 +1,51 @@
+/**
+ * @euconform/core/evidence — Evidence model and output generation
+ *
+ * This module defines the evidence types produced by the scanner
+ * and transforms raw scan results into structured output artifacts.
+ */
+
+export type {
+  AiBillOfMaterials,
+  AppType,
+  AssessmentHint,
+  BomComponent,
+  BomComponentKind,
+  BundleArtifactRef,
+  BundleArtifactRole,
+  CiReport,
+  ComplianceSignalGroup,
+  ConfidenceLevel,
+  DetectedSignal,
+  DetectorContext,
+  FailOnLevel,
+  FileProvenance,
+  GapCounts,
+  OpenQuestion,
+  RepoProfile,
+  ScanBundle,
+  ScanFile,
+  ScanGap,
+  ScanMeta,
+  ScanOptions,
+  ScanOutput,
+  ScanReport,
+  ScanResult,
+  ScanScope,
+  SignalCategory,
+  SignalDetector,
+  SignalEvidence,
+} from "./types";
+
+export { generateScanOutput } from "./output";
+export { generateSummaryMarkdown } from "./markdown";
+export { buildBundleManifest } from "./bundle";
+export { sha256Hex } from "./hash";
+export {
+  getSchemaMajorVersion,
+  validateAiBillOfMaterials,
+  validateCiReport,
+  validateEcefJsonDocument,
+  validateScanBundle,
+  validateScanReport,
+} from "./validate";