@euconform/core 1.2.0 → 1.4.0

package/src/scanner/detectors/framework-detector.ts

@@ -0,0 +1,163 @@
+import type {
+  ConfidenceLevel,
+  DetectedSignal,
+  DetectorContext,
+  SignalEvidence,
+} from "../../evidence/types";
+import { EXTENDED_EXTENSIONS } from "./shared";
+
+interface FrameworkRule {
+  name: string;
+  category: "framework" | "runtime";
+  /** Match against file name (basename) */
+  configPattern?: RegExp;
+  /** Match against file content (line-level) */
+  importPattern?: RegExp;
+  configConfidence: ConfidenceLevel;
+  importConfidence: ConfidenceLevel;
+}
+
+const rules: FrameworkRule[] = [
+  // Frameworks
+  {
+    name: "nextjs",
+    category: "framework",
+    configPattern: /^next\.config\./,
+    importPattern: /\bfrom\s+['"]next[\/'"]|\brequire\(\s*['"]next['"]\)/,
+    configConfidence: "high",
+    importConfidence: "medium",
+  },
+  {
+    name: "react",
+    category: "framework",
+    importPattern: /\bfrom\s+['"]react['"]|\brequire\(\s*['"]react['"]\)/,
+    configConfidence: "high",
+    importConfidence: "medium",
+  },
+  {
+    name: "express",
+    category: "framework",
+    importPattern: /\bfrom\s+['"]express['"]|\brequire\(\s*['"]express['"]\)/,
+    configConfidence: "high",
+    importConfidence: "medium",
+  },
+  {
+    name: "fastify",
+    category: "framework",
+    importPattern: /\bfrom\s+['"]fastify['"]|\brequire\(\s*['"]fastify['"]\)/,
+    configConfidence: "high",
+    importConfidence: "medium",
+  },
+  {
+    name: "nestjs",
+    category: "framework",
+    importPattern: /\bfrom\s+['"]@nestjs\/|\brequire\(\s*['"]@nestjs\//,
+    configConfidence: "high",
+    importConfidence: "medium",
+  },
+  {
+    name: "hono",
+    category: "framework",
+    importPattern: /\bfrom\s+['"]hono['"]|\brequire\(\s*['"]hono['"]\)/,
+    configConfidence: "high",
+    importConfidence: "medium",
+  },
+  {
+    name: "elysia",
+    category: "framework",
+    importPattern: /\bfrom\s+['"]elysia['"]|\brequire\(\s*['"]elysia['"]\)/,
+    configConfidence: "high",
+    importConfidence: "medium",
+  },
+  {
+    name: "vite",
+    category: "framework",
+    configPattern: /^vite\.config\./,
+    importPattern: /\bfrom\s+['"]vite['"]|\brequire\(\s*['"]vite['"]\)/,
+    configConfidence: "high",
+    importConfidence: "medium",
+  },
+  // Runtimes
+  {
+    name: "nodejs",
+    category: "runtime",
+    importPattern:
+      /\bfrom\s+['"]node:(?:fs|http|path|crypto|stream)['"]|\bfrom\s+['"](?:fs|http|net|dgram)['"]|\brequire\(\s*['"](?:fs|http|path|crypto)['"]\)/,
+    configConfidence: "high",
+    importConfidence: "medium",
+  },
+  {
+    name: "bun",
+    category: "runtime",
+    importPattern: /\bBun\.serve\b|\bBun\.file\b|\bBun\.write\b/,
+    configConfidence: "high",
+    importConfidence: "medium",
+  },
+  {
+    name: "deno",
+    category: "runtime",
+    importPattern: /\bDeno\.serve\b|\bDeno\.readFile\b|\bDeno\.env\b/,
+    configConfidence: "high",
+    importConfidence: "medium",
+  },
+];
+
+const CONFIG_NAME_PATTERN = /^(next|vite)\.config\./;
+
+function shouldProcess(file: { extension: string; relativePath: string }): boolean {
+  if (EXTENDED_EXTENSIONS.has(file.extension)) return true;
+  const basename = file.relativePath.split("/").pop() ?? "";
+  return CONFIG_NAME_PATTERN.test(basename);
+}
+
+function findImportEvidence(lines: string[], pattern: RegExp, filePath: string): SignalEvidence[] {
+  const evidence: SignalEvidence[] = [];
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (!line) continue;
+    if (pattern.test(line)) {
+      evidence.push({ file: filePath, line: i + 1, snippet: line.trim().slice(0, 120) });
+      if (evidence.length >= 3) break;
+    }
+  }
+  return evidence;
+}
+
+export function detectFrameworks(ctx: DetectorContext): DetectedSignal[] {
+  const { file } = ctx;
+  if (!shouldProcess(file)) return [];
+
+  const basename = file.relativePath.split("/").pop() ?? "";
+  const seen = new Set<string>();
+  const signals: DetectedSignal[] = [];
+
+  for (const rule of rules) {
+    if (rule.configPattern?.test(basename) && !seen.has(rule.name)) {
+      seen.add(rule.name);
+      signals.push({
+        id: `${rule.category}-${rule.name}`,
+        name: rule.name,
+        category: rule.category,
+        confidence: rule.configConfidence,
+        evidence: [
+          { file: file.relativePath, line: 1, snippet: `Config file detected: ${basename}` },
+        ],
+      });
+    }
+
+    if (!rule.importPattern) continue;
+    const evidence = findImportEvidence(ctx.lines, rule.importPattern, file.relativePath);
+    if (evidence.length > 0 && !seen.has(rule.name)) {
+      seen.add(rule.name);
+      signals.push({
+        id: `${rule.category}-${rule.name}`,
+        name: rule.name,
+        category: rule.category,
+        confidence: rule.importConfidence,
+        evidence,
+      });
+    }
+  }
+
+  return signals;
+}

package/src/scanner/detectors/index.ts

@@ -0,0 +1,40 @@
+/**
+ * Signal detector registry.
+ *
+ * Runs all registered detectors against a file and returns combined signals.
+ */
+
+import type { DetectedSignal, DetectorContext, SignalDetector } from "../../evidence/types";
+import { detectAiFrameworkSignals } from "./ai-framework-detector";
+import { detectAiProviderSignals } from "./ai-provider-detector";
+import { detectCompliance } from "./compliance-detector";
+import { detectConfig } from "./config-detector";
+import { detectFrameworks } from "./framework-detector";
+import { detectLocalInferenceSignals } from "./local-inference-detector";
+import { detectMarkdown } from "./markdown-detector";
+import { detectPackageJsonSignals } from "./package-json-detector";
+import { detectRagSignals } from "./rag-detector";
+
+const ALL_DETECTORS: SignalDetector[] = [
+  detectPackageJsonSignals,
+  detectAiProviderSignals,
+  detectAiFrameworkSignals,
+  detectLocalInferenceSignals,
+  detectRagSignals,
+  detectFrameworks,
+  detectCompliance,
+  detectConfig,
+  detectMarkdown,
+];
+
+export function runAllDetectors(ctx: DetectorContext): DetectedSignal[] {
+  const signals: DetectedSignal[] = [];
+  for (const detector of ALL_DETECTORS) {
+    try {
+      signals.push(...detector(ctx));
+    } catch {
+      // Individual detector failure should not break the scan
+    }
+  }
+  return signals;
+}

package/src/scanner/detectors/local-inference-detector.ts

@@ -0,0 +1,124 @@
+import type {
+  ConfidenceLevel,
+  DetectedSignal,
+  DetectorContext,
+  SignalEvidence,
+} from "../../evidence/types";
+import { EXTENDED_EXTENSIONS } from "./shared";
+
+// ---------------------------------------------------------------------------
+// Detection rules
+// ---------------------------------------------------------------------------
+
+interface DetectionRule {
+  id: string;
+  name: string;
+  pattern: RegExp;
+  confidence: ConfidenceLevel;
+  /** File extensions this rule applies to. Empty = all allowed extensions. */
+  extensions?: string[];
+}
+
+const RULES: DetectionRule[] = [
+  // Ollama API URL
+  {
+    id: "local-inference-ollama-api",
+    name: "Ollama API (localhost:11434)",
+    pattern: /localhost:11434/,
+    confidence: "high",
+  },
+  // Ollama imports
+  {
+    id: "local-inference-ollama-import",
+    name: "Ollama SDK import",
+    pattern:
+      /(?:from\s+['"](?:ollama|@ollama\/ollama)['"]|require\s*\(\s*['"](?:ollama|@ollama\/ollama)['"])/,
+    confidence: "medium",
+    extensions: [".ts", ".tsx", ".js", ".jsx"],
+  },
+  // Transformers.js browser usage (pipeline / env)
+  {
+    id: "local-inference-transformers-browser",
+    name: "Transformers.js (browser inference)",
+    pattern: /(?:from\s+['"]@xenova\/transformers['"]|\.pipeline\s*\(|env\.allowLocalModels)/,
+    confidence: "medium",
+    extensions: [".ts", ".tsx", ".js", ".jsx"],
+  },
+  // ONNX Runtime
+  {
+    id: "local-inference-onnx",
+    name: "ONNX Runtime",
+    pattern:
+      /(?:from\s+['"]onnxruntime[^'"]*['"]|require\s*\(\s*['"]onnxruntime[^'"]*['"]|onnxruntime)/,
+    confidence: "medium",
+  },
+  // llama.cpp references
+  {
+    id: "local-inference-llamacpp",
+    name: "llama.cpp",
+    pattern: /llama\.cpp|llama-cpp|llamacpp/,
+    confidence: "medium",
+  },
+  // Docker Compose ollama service
+  {
+    id: "local-inference-ollama-docker",
+    name: "Ollama Docker service",
+    pattern: /ollama\/ollama|image:\s*ollama/,
+    confidence: "high",
+    extensions: [".yaml", ".yml", ".json"],
+  },
+];
+
+// ---------------------------------------------------------------------------
+// Detector
+// ---------------------------------------------------------------------------
+
+function matchRuleToLine(
+  rule: (typeof RULES)[number],
+  line: string,
+  lineNum: number,
+  filePath: string,
+  signalMap: Map<string, DetectedSignal>
+): void {
+  if (!rule.pattern.test(line)) return;
+
+  const evidence: SignalEvidence = {
+    file: filePath,
+    line: lineNum,
+    snippet: line.trim().slice(0, 200),
+  };
+
+  const existing = signalMap.get(rule.id);
+  if (existing) {
+    existing.evidence.push(evidence);
+    if (rule.confidence === "high") existing.confidence = "high";
+  } else {
+    signalMap.set(rule.id, {
+      id: rule.id,
+      name: rule.name,
+      category: "local-inference",
+      confidence: rule.confidence,
+      evidence: [evidence],
+    });
+  }
+}
+
+export function detectLocalInferenceSignals(ctx: DetectorContext): DetectedSignal[] {
+  const { file } = ctx;
+  if (!EXTENDED_EXTENSIONS.has(file.extension)) return [];
+
+  const lines = ctx.lines;
+  const signalMap = new Map<string, DetectedSignal>();
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (!line) continue;
+
+    for (const rule of RULES) {
+      if (rule.extensions && !rule.extensions.includes(file.extension)) continue;
+      matchRuleToLine(rule, line, i + 1, file.relativePath, signalMap);
+    }
+  }
+
+  return Array.from(signalMap.values());
+}

package/src/scanner/detectors/markdown-detector.ts

@@ -0,0 +1,176 @@
+import type {
+  ConfidenceLevel,
+  DetectedSignal,
+  DetectorContext,
+  SignalCategory,
+  SignalEvidence,
+} from "../../evidence/types";
+
+// ---------------------------------------------------------------------------
+// Rules
+// ---------------------------------------------------------------------------
+
+interface MarkdownRule {
+  id: string;
+  name: string;
+  category: SignalCategory;
+  /** Match against individual lines */
+  pattern: RegExp;
+  confidence: ConfidenceLevel;
+  /** When true, only matches inside heading-level sections (## ...) */
+  sectionHeading?: boolean;
+}
+
+const rules: MarkdownRule[] = [
+  // --- compliance-disclosure ---
+  {
+    id: "compliance-disclosure-readme",
+    name: "AI disclosure in README",
+    category: "compliance-disclosure",
+    pattern:
+      /\b(?:artificial\s+intelligence|machine\s+learning|AI[- ]powered|AI[- ]generated|uses?\s+AI|deep\s+learning|neural\s+network|large\s+language\s+model|LLM)\b/i,
+    confidence: "low",
+  },
+  {
+    id: "compliance-disclosure-transparency",
+    name: "Transparency notice",
+    category: "compliance-disclosure",
+    pattern: /\b(?:transparency\s+notice|AI\s+disclosure|generated\s+by\s+AI)\b/i,
+    confidence: "medium",
+  },
+
+  // --- Model card ---
+  {
+    id: "compliance-disclosure-model-card",
+    name: "Model card documentation",
+    category: "compliance-disclosure",
+    pattern: /^#{1,3}\s+Model\s+Card\b/i,
+    confidence: "medium",
+    sectionHeading: true,
+  },
+
+  // --- compliance-data ---
+  {
+    id: "compliance-data-docs",
+    name: "Data governance documentation",
+    category: "compliance-data",
+    pattern: /^#{1,3}\s+(?:Data\s+(?:Sources?|Governance|Quality|Lineage)|Training\s+Data)\b/i,
+    confidence: "medium",
+    sectionHeading: true,
+  },
+  {
+    id: "compliance-data-reference",
+    name: "Data source references",
+    category: "compliance-data",
+    pattern: /\b(?:training\s+data|data\s+sources?|dataset\s+documentation)\b/i,
+    confidence: "low",
+  },
+
+  // --- Privacy / GDPR ---
+  {
+    id: "compliance-data-privacy",
+    name: "Privacy policy reference",
+    category: "compliance-data",
+    pattern: /\b(?:privacy\s+policy|GDPR|data\s+protection|DSGVO)\b/i,
+    confidence: "medium",
+  },
+
+  // --- License / AI-specific ---
+  {
+    id: "compliance-disclosure-license",
+    name: "AI-relevant license terms",
+    category: "compliance-disclosure",
+    pattern:
+      /\b(?:model\s+license|AI\s+license|responsible\s+AI|acceptable\s+use\s+policy|RAIL\s+license)\b/i,
+    confidence: "medium",
+  },
+];
+
+// ---------------------------------------------------------------------------
+// File filtering
+// ---------------------------------------------------------------------------
+
+/** Files that should always be skipped unless they contain AI-specific content */
+const SKIP_BASENAMES = new Set([
+  "changelog.md",
+  "changes.md",
+  "history.md",
+  "contributing.md",
+  "code_of_conduct.md",
+]);
+
+/** Only process these docs-level paths */
+function isRelevantMarkdown(relativePath: string): boolean {
+  const lower = relativePath.toLowerCase();
+  const basename = lower.split("/").pop() ?? "";
+
+  // Always process README files at any level
+  if (basename.startsWith("readme")) return true;
+
+  // Process files inside docs/ directories
+  if (lower.includes("/docs/") || lower.startsWith("docs/")) {
+    return !SKIP_BASENAMES.has(basename);
+  }
+
+  // For other markdown files, skip known non-relevant ones
+  if (SKIP_BASENAMES.has(basename)) return false;
+
+  return true;
+}
+
+// ---------------------------------------------------------------------------
+// Detector
+// ---------------------------------------------------------------------------
+
+function gatherMarkdownEvidence(
+  rule: (typeof rules)[number],
+  lines: string[],
+  filePath: string
+): SignalEvidence[] {
+  const evidence: SignalEvidence[] = [];
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (!line) continue;
+    if (rule.sectionHeading && !line.startsWith("#")) continue;
+    if (rule.pattern.test(line)) {
+      evidence.push({ file: filePath, line: i + 1, snippet: line.trim().slice(0, 120) });
+      if (evidence.length >= 5) break;
+    }
+  }
+  return evidence;
+}
+
+function isSkippableGenericContent(basename: string, evidence: SignalEvidence[]): boolean {
+  if (!SKIP_BASENAMES.has(basename)) return false;
+  return !evidence.some((e) => /\bAI\b|\bML\b|\bmodel\b|\binference\b/i.test(e.snippet));
+}
+
+export function detectMarkdown(ctx: DetectorContext): DetectedSignal[] {
+  const { file } = ctx;
+  if (file.extension !== ".md") return [];
+  if (!isRelevantMarkdown(file.relativePath)) return [];
+
+  const signalMap = new Map<string, DetectedSignal>();
+  const basename = file.relativePath.toLowerCase().split("/").pop() ?? "";
+
+  for (const rule of rules) {
+    const evidence = gatherMarkdownEvidence(rule, ctx.lines, file.relativePath);
+    if (evidence.length === 0) continue;
+    if (isSkippableGenericContent(basename, evidence)) continue;
+
+    const existing = signalMap.get(rule.id);
+    if (existing) {
+      existing.evidence.push(...evidence);
+    } else {
+      signalMap.set(rule.id, {
+        id: rule.id,
+        name: rule.name,
+        category: rule.category,
+        confidence: rule.confidence,
+        evidence,
+      });
+    }
+  }
+
+  return Array.from(signalMap.values());
+}

package/src/scanner/detectors/package-json-detector.ts

@@ -0,0 +1,202 @@
+import { basename } from "node:path";
+import type { DetectedSignal, DetectorContext, SignalCategory } from "../../evidence/types";
+import { escapeRegex } from "./shared";
+
+// ---------------------------------------------------------------------------
+// Known package mappings
+// ---------------------------------------------------------------------------
+
+interface PackageInfo {
+  category: SignalCategory;
+  label: string;
+}
+
+const AI_PACKAGES: Record<string, PackageInfo> = {
+  // Providers
+  openai: { category: "ai-provider", label: "OpenAI SDK" },
+  "@anthropic-ai/sdk": { category: "ai-provider", label: "Anthropic SDK" },
+  "cohere-ai": { category: "ai-provider", label: "Cohere SDK" },
+  "@google/generative-ai": { category: "ai-provider", label: "Google Generative AI" },
+  "@mistralai/mistralai": { category: "ai-provider", label: "Mistral AI SDK" },
+
+  // Frameworks
+  langchain: { category: "ai-framework", label: "LangChain" },
+  "@langchain/core": { category: "ai-framework", label: "LangChain Core" },
+  llamaindex: { category: "ai-framework", label: "LlamaIndex" },
+  "@xenova/transformers": { category: "ai-framework", label: "Transformers.js" },
+  ai: { category: "ai-framework", label: "Vercel AI SDK" },
+  "@huggingface/inference": { category: "ai-framework", label: "HuggingFace Inference" },
+
+  // Local inference
+  ollama: { category: "local-inference", label: "Ollama" },
+  "@ollama/ollama": { category: "local-inference", label: "Ollama SDK" },
+
+  // RAG / Vector
+  chromadb: { category: "rag", label: "ChromaDB" },
+  "@pinecone-database/pinecone": { category: "rag", label: "Pinecone" },
+  "weaviate-client": { category: "rag", label: "Weaviate" },
+  "@qdrant/js-client-rest": { category: "rag", label: "Qdrant" },
+  pgvector: { category: "rag", label: "pgvector" },
+
+  // Bias / Eval
+  "@euconform/core": { category: "compliance-bias", label: "EuConform Core" },
+};
+
+const FRAMEWORK_PACKAGES: Record<string, PackageInfo> = {
+  next: { category: "framework", label: "Next.js" },
+  react: { category: "framework", label: "React" },
+  express: { category: "framework", label: "Express" },
+  fastify: { category: "framework", label: "Fastify" },
+  "@nestjs/core": { category: "framework", label: "NestJS" },
+  hono: { category: "framework", label: "Hono" },
+  elysia: { category: "framework", label: "Elysia" },
+};
+
+const ALL_PACKAGES: Record<string, PackageInfo> = {
+  ...AI_PACKAGES,
+  ...FRAMEWORK_PACKAGES,
+};
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+const PACKAGE_JSON_NAMES = new Set([
+  "package.json",
+  "pnpm-lock.yaml",
+  "package-lock.json",
+  "yarn.lock",
+]);
+
+function isPackageJsonFile(relativePath: string): boolean {
+  return PACKAGE_JSON_NAMES.has(basename(relativePath));
+}
+
+// ---------------------------------------------------------------------------
+// Detector
+// ---------------------------------------------------------------------------
+
+export function detectPackageJsonSignals(ctx: DetectorContext): DetectedSignal[] {
+  const { file } = ctx;
+  if (!isPackageJsonFile(file.relativePath)) return [];
+
+  const fileName = basename(file.relativePath);
+
+  if (fileName === "package.json") {
+    return detectFromPackageJson(file.relativePath, file.content);
+  }
+
+  // Lock files — scan for known package name strings
+  return detectFromLockFile(file.relativePath, file.content);
+}
+
+// ---------------------------------------------------------------------------
+// package.json detection
+// ---------------------------------------------------------------------------
+
+function detectFromPackageJson(filePath: string, content: string): DetectedSignal[] {
+  let parsed: Record<string, unknown>;
+  try {
+    parsed = JSON.parse(content) as Record<string, unknown>;
+  } catch {
+    return [];
+  }
+
+  const signals: DetectedSignal[] = [];
+  const deps = {
+    ...(parsed.dependencies as Record<string, string> | undefined),
+    ...(parsed.devDependencies as Record<string, string> | undefined),
+  };
+
+  for (const [pkg, info] of Object.entries(ALL_PACKAGES)) {
+    if (pkg in deps) {
+      const version = deps[pkg] ?? "unknown";
+      signals.push({
+        id: `${info.category}-${pkg.replace(/[@/]/g, "").replace(/\s/g, "-")}`,
+        name: info.label,
+        category: info.category,
+        confidence: "high",
+        evidence: [
+          {
+            file: filePath,
+            snippet: `"${pkg}": "${version}"`,
+          },
+        ],
+      });
+    }
+  }
+
+  // Monorepo signals
+  if (parsed.workspaces) {
+    signals.push({
+      id: "runtime-monorepo-workspaces",
+      name: "Monorepo (workspaces)",
+      category: "runtime",
+      confidence: "high",
+      evidence: [
+        {
+          file: filePath,
+          snippet: `"workspaces": ${JSON.stringify(parsed.workspaces).slice(0, 120)}`,
+        },
+      ],
+    });
+  }
+
+  const devDeps = parsed.devDependencies as Record<string, string> | undefined;
+  if (devDeps && "turbo" in devDeps) {
+    signals.push({
+      id: "runtime-monorepo-turbo",
+      name: "Monorepo (Turborepo)",
+      category: "runtime",
+      confidence: "high",
+      evidence: [
+        {
+          file: filePath,
+          snippet: `"turbo": "${devDeps.turbo}"`,
+        },
+      ],
+    });
+  }
+
+  return signals;
+}
+
+// ---------------------------------------------------------------------------
+// Lock-file detection (string search, no full parse)
+// ---------------------------------------------------------------------------
+
+// Pre-compiled lock-file patterns — avoids RegExp construction on every call
+const LOCK_FILE_PATTERNS: Map<string, RegExp> = new Map(
+  Object.keys(AI_PACKAGES).map((pkg) => [pkg, new RegExp(`["'/]${escapeRegex(pkg)}["'/@:]`)])
+);
+
+function detectFromLockFile(filePath: string, content: string): DetectedSignal[] {
+  const signals: DetectedSignal[] = [];
+  const seen = new Set<string>();
+
+  for (const [pkg, info] of Object.entries(AI_PACKAGES)) {
+    if (seen.has(pkg)) continue;
+
+    // Use boundary-aware pattern to avoid false positives on short names
+    // Lock files quote package names or use them as path segments
+    const pattern = LOCK_FILE_PATTERNS.get(pkg);
+    if (!pattern) continue;
+    if (pattern.test(content)) {
+      seen.add(pkg);
+      signals.push({
+        id: `${info.category}-${pkg.replace(/[@/]/g, "").replace(/\s/g, "-")}`,
+        name: info.label,
+        category: info.category,
+        confidence: "medium",
+        evidence: [
+          {
+            file: filePath,
+            snippet: `Found "${pkg}" in lock file`,
+          },
+        ],
+      });
+    }
+  }
+
+  return signals;
+}