@euconform/core 1.4.0 → 1.6.0

package/src/sbom/cyclonedx-import.ts

@@ -0,0 +1,198 @@
+import { basename, extname } from "node:path";
+import type { AiBillOfMaterials, AibomMetadata, BomComponent } from "../evidence/types";
+import type { ValidationResult } from "../validation/schema-validator";
+import { validate } from "../validation/schema-validator";
+import { mapComponent } from "./component-mapper";
+import type { CycloneDxBom } from "./types";
+import { SUPPORTED_SPEC_VERSIONS } from "./types";
+
+export interface ImportWarning {
+  component: string;
+  message: string;
+}
+
+export interface ImportSourceInfo {
+  bomFormat: "CycloneDX";
+  specVersion: string;
+  projectNameSource: "metadata.component.name" | "sourcePath" | "fallback";
+  importTool?: string;
+  originalTimestamp?: string;
+}
+
+export interface ImportSummary {
+  totalComponents: number;
+  filteredByScope: number;
+  aiRelevant: number;
+  skipped: number;
+  duplicatesRemoved: number;
+  byKind: Record<string, number>;
+  warnings: ImportWarning[];
+  source: ImportSourceInfo;
+}
+
+export interface ImportOptions {
+  scope?: "all" | "production";
+  sourcePath?: string;
+}
+
+export interface CycloneDxImportResult {
+  aibom: AiBillOfMaterials;
+  summary: ImportSummary;
+  validation: ValidationResult;
+}
+
+export function importCycloneDx(sbom: unknown, options?: ImportOptions): CycloneDxImportResult {
+  const bom = parseBom(sbom);
+  const scope = options?.scope ?? "all";
+
+  const allComponents = bom.components ?? [];
+  const warnings: ImportWarning[] = [];
+  const totalComponents = allComponents.length;
+
+  const inScope =
+    scope === "production"
+      ? allComponents.filter((c) => c.scope !== "optional" && c.scope !== "excluded")
+      : allComponents;
+  const filteredByScope = totalComponents - inScope.length;
+
+  const mapped: BomComponent[] = [];
+  for (const comp of inScope) {
+    const mapping = mapComponent(comp);
+    if (!mapping) {
+      if (!comp.name || comp.name.trim() === "") {
+        warnings.push({ component: "(empty)", message: "Skipped: missing or empty name" });
+      }
+      continue;
+    }
+
+    const version = comp.version || mapping.purlVersion;
+    if (!version) {
+      warnings.push({ component: comp.name, message: "Missing version field" });
+    }
+
+    mapped.push({
+      id: version ? `${mapping.kind}:${comp.name}:${version}` : `${mapping.kind}:${comp.name}`,
+      kind: mapping.kind,
+      name: comp.name,
+      ...(version ? { version } : {}),
+      source: "sbom-import",
+    });
+  }
+
+  const seen = new Set<string>();
+  let duplicatesRemoved = 0;
+  const deduped: BomComponent[] = [];
+  for (const comp of mapped) {
+    if (seen.has(comp.id)) {
+      duplicatesRemoved++;
+      continue;
+    }
+    seen.add(comp.id);
+    deduped.push(comp);
+  }
+
+  const byKind: Record<string, number> = {};
+  for (const comp of deduped) {
+    byKind[comp.kind] = (byKind[comp.kind] ?? 0) + 1;
+  }
+
+  const { projectName, projectNameSource } = resolveProjectName(bom, options?.sourcePath);
+  const importTool = extractToolString(bom);
+  const originalTimestamp = bom.metadata?.timestamp;
+
+  const aibom: AiBillOfMaterials = {
+    schemaVersion: "euconform.aibom.v1.1",
+    generatedAt: new Date().toISOString(),
+    project: { name: projectName, rootPath: "." },
+    components: deduped,
+    complianceCapabilities: {
+      biasEvaluation: false,
+      jsonExport: false,
+      pdfExport: false,
+      loggingInfrastructure: false,
+      humanReviewFlow: false,
+      incidentHandling: false,
+    },
+    metadata: buildMetadata(importTool, originalTimestamp),
+  };
+
+  const validation = validate(aibom);
+
+  return {
+    aibom,
+    summary: {
+      totalComponents,
+      filteredByScope,
+      aiRelevant: deduped.length,
+      skipped: inScope.length - mapped.length,
+      duplicatesRemoved,
+      byKind,
+      warnings,
+      source: buildSourceInfo(bom.specVersion, projectNameSource, importTool, originalTimestamp),
+    },
+    validation,
+  };
+}
+
+function resolveProjectName(
+  bom: CycloneDxBom,
+  sourcePath?: string
+): { projectName: string; projectNameSource: ImportSourceInfo["projectNameSource"] } {
+  const metaName = bom.metadata?.component?.name;
+  if (metaName) {
+    return { projectName: metaName, projectNameSource: "metadata.component.name" };
+  }
+
+  if (sourcePath) {
+    const name = basename(sourcePath, extname(sourcePath));
+    return { projectName: name, projectNameSource: "sourcePath" };
+  }
+
+  return { projectName: "sbom-import", projectNameSource: "fallback" };
+}
+
+function extractToolString(bom: CycloneDxBom): string | undefined {
+  const tool = bom.metadata?.tools?.[0];
+  if (!tool?.name) return undefined;
+  return tool.version ? `${tool.name} ${tool.version}` : tool.name;
+}
+
+function buildMetadata(importTool?: string, originalTimestamp?: string): AibomMetadata {
+  const meta: AibomMetadata = { importSource: "cyclonedx" };
+  if (importTool) meta.importTool = importTool;
+  if (originalTimestamp) meta.originalTimestamp = originalTimestamp;
+  return meta;
+}
+
+function buildSourceInfo(
+  specVersion: string,
+  projectNameSource: ImportSourceInfo["projectNameSource"],
+  importTool?: string,
+  originalTimestamp?: string
+): ImportSourceInfo {
+  const info: ImportSourceInfo = { bomFormat: "CycloneDX", specVersion, projectNameSource };
+  if (importTool) info.importTool = importTool;
+  if (originalTimestamp) info.originalTimestamp = originalTimestamp;
+  return info;
+}
+
+function parseBom(sbom: unknown): CycloneDxBom {
+  if (typeof sbom !== "object" || sbom === null) {
+    throw new Error("Input must be a JSON object");
+  }
+
+  const record = sbom as Record<string, unknown>;
+
+  if (record.bomFormat !== "CycloneDX") {
+    throw new Error(`Invalid bomFormat: expected "CycloneDX", got "${String(record.bomFormat)}"`);
+  }
+
+  const specVersion = String(record.specVersion ?? "");
+  if (!SUPPORTED_SPEC_VERSIONS.includes(specVersion as (typeof SUPPORTED_SPEC_VERSIONS)[number])) {
+    throw new Error(
+      `Unsupported specVersion "${specVersion}". Supported: ${SUPPORTED_SPEC_VERSIONS.join(", ")}`
+    );
+  }
+
+  return sbom as CycloneDxBom;
+}

package/src/sbom/index.ts

@@ -0,0 +1,12 @@
+export {
+  importCycloneDx,
+  type CycloneDxImportResult,
+  type ImportSummary,
+  type ImportWarning,
+  type ImportOptions,
+  type ImportSourceInfo,
+} from "./cyclonedx-import";
+export { mapComponent, type ComponentMapping } from "./component-mapper";
+export { lookupKnownPackage, KNOWN_AI_PACKAGES, KNOWN_AI_SCOPES } from "./known-packages";
+export { parsePurl, type ParsedPurl } from "./purl";
+export type { CycloneDxBom, CycloneDxComponent } from "./types";

package/src/sbom/known-packages.ts

@@ -0,0 +1,142 @@
+import type { BomComponentKind } from "../evidence/types";
+
+export const KNOWN_AI_PACKAGES: Record<string, BomComponentKind> = {
+  // Models / model runtimes
+  "llama.cpp": "model",
+  "llama-cpp-python": "model",
+  "whisper.cpp": "model",
+  ggml: "model",
+  ctransformers: "model",
+
+  // AI Frameworks
+  transformers: "ai-framework",
+  torch: "ai-framework",
+  pytorch: "ai-framework",
+  tensorflow: "ai-framework",
+  "tensorflow-gpu": "ai-framework",
+  jax: "ai-framework",
+  keras: "ai-framework",
+  "pytorch-lightning": "ai-framework",
+  "scikit-learn": "ai-framework",
+  sklearn: "ai-framework",
+  xgboost: "ai-framework",
+  lightgbm: "ai-framework",
+  spacy: "ai-framework",
+  "huggingface-hub": "ai-framework",
+  diffusers: "ai-framework",
+  accelerate: "ai-framework",
+  peft: "ai-framework",
+  trl: "ai-framework",
+  onnxruntime: "ai-framework",
+  "onnxruntime-gpu": "ai-framework",
+  mlflow: "ai-framework",
+
+  // Embeddings
+  "sentence-transformers": "embedding",
+  fastembed: "embedding",
+  "openai-embeddings": "embedding",
+  "cohere-embed": "embedding",
+
+  // Inference Providers
+  openai: "inference-provider",
+  anthropic: "inference-provider",
+  ollama: "inference-provider",
+  vllm: "inference-provider",
+  "together-ai": "inference-provider",
+  replicate: "inference-provider",
+  groq: "inference-provider",
+  mistralai: "inference-provider",
+  cohere: "inference-provider",
+  "google-generativeai": "inference-provider",
+  ai21: "inference-provider",
+
+  // Vector Stores
+  pinecone: "vector-store",
+  "pinecone-client": "vector-store",
+  weaviate: "vector-store",
+  "weaviate-client": "vector-store",
+  qdrant: "vector-store",
+  "qdrant-client": "vector-store",
+  milvus: "vector-store",
+  pymilvus: "vector-store",
+  chromadb: "vector-store",
+  "chromadb-client": "vector-store",
+  pgvector: "vector-store",
+  faiss: "vector-store",
+  "faiss-cpu": "vector-store",
+  "faiss-gpu": "vector-store",
+
+  // Datasets
+  datasets: "dataset",
+
+  // Tools / Orchestration
+  langchain: "tool",
+  "langchain-core": "tool",
+  "langchain-community": "tool",
+  llamaindex: "tool",
+  "llama-index": "tool",
+  "semantic-kernel": "tool",
+  autogen: "tool",
+  crewai: "tool",
+  dspy: "tool",
+  "haystack-ai": "tool",
+  guidance: "tool",
+  guardrails: "tool",
+  "guardrails-ai": "tool",
+  lmql: "tool",
+  outlines: "tool",
+
+  // Scoped packages (npm)
+  "@langchain/core": "tool",
+  "@langchain/openai": "tool",
+  "@langchain/community": "tool",
+  "@langchain/anthropic": "tool",
+  "@huggingface/transformers": "ai-framework",
+  "@huggingface/inference": "inference-provider",
+  "@google/generative-ai": "inference-provider",
+  "@anthropic-ai/sdk": "inference-provider",
+  "@pinecone-database/pinecone": "vector-store",
+  "@qdrant/js-client-rest": "vector-store",
+};
+
+export const KNOWN_AI_SCOPES: Record<string, BomComponentKind> = {
+  "@langchain": "tool",
+  "@huggingface": "ai-framework",
+  "@tensorflow": "ai-framework",
+  "@anthropic-ai": "inference-provider",
+  "@pinecone-database": "vector-store",
+  "@qdrant": "vector-store",
+};
+
+/**
+ * Look up a package name in the known AI packages registry.
+ * For scoped packages, tries:
+ *   1. Exact match (e.g. @langchain/core)
+ *   2. Scope match via KNOWN_AI_SCOPES (e.g. @langchain -> tool)
+ *   3. Strip scope, check bare name (e.g. @huggingface/transformers -> transformers)
+ * Returns the BomComponentKind if found, null otherwise.
+ */
+export function lookupKnownPackage(name: string): BomComponentKind | null {
+  const lower = name.toLowerCase();
+
+  // 1. Exact match
+  const exact = KNOWN_AI_PACKAGES[lower];
+  if (exact) return exact;
+
+  // Scoped package handling
+  if (lower.startsWith("@") && lower.includes("/")) {
+    const scopeEnd = lower.indexOf("/");
+    const scope = lower.slice(0, scopeEnd);
+    const bareName = lower.slice(scopeEnd + 1);
+
+    // 2. Known AI scope
+    const scopeKind = KNOWN_AI_SCOPES[scope];
+    if (scopeKind) return scopeKind;
+
+    // 3. Strip scope, check bare name
+    const bareKind = KNOWN_AI_PACKAGES[bareName];
+    if (bareKind) return bareKind;
+  }
+
+  return null;
+}

package/src/sbom/purl.ts

@@ -0,0 +1,66 @@
+export interface ParsedPurl {
+  scheme: "pkg";
+  type: string;
+  namespace?: string;
+  name: string;
+  version?: string;
+}
+
+/**
+ * Parse a Package URL (purl) string into its components.
+ * Follows the purl spec: pkg:type/namespace/name@version?qualifiers#subpath
+ * Returns null for invalid or non-pkg URLs.
+ */
+export function parsePurl(purl: string): ParsedPurl | null {
+  if (!purl || !purl.startsWith("pkg:")) return null;
+
+  // Strip qualifiers and subpath
+  let rest = purl.slice(4); // remove "pkg:"
+  const hashIdx = rest.indexOf("#");
+  if (hashIdx !== -1) rest = rest.slice(0, hashIdx);
+  const queryIdx = rest.indexOf("?");
+  if (queryIdx !== -1) rest = rest.slice(0, queryIdx);
+
+  // Split type from the rest: type/...
+  const slashIdx = rest.indexOf("/");
+  if (slashIdx <= 0) return null;
+
+  const type = rest.slice(0, slashIdx);
+  let remainder = rest.slice(slashIdx + 1);
+
+  try {
+    remainder = decodeURIComponent(remainder);
+  } catch {
+    return null;
+  }
+
+  // Extract version
+  let version: string | undefined;
+  const atIdx = remainder.lastIndexOf("@");
+  if (atIdx !== -1) {
+    version = remainder.slice(atIdx + 1);
+    remainder = remainder.slice(0, atIdx);
+  }
+
+  // Split namespace/name (last segment is name, rest is namespace)
+  const lastSlash = remainder.lastIndexOf("/");
+  let namespace: string | undefined;
+  let name: string;
+
+  if (lastSlash !== -1) {
+    namespace = remainder.slice(0, lastSlash);
+    name = remainder.slice(lastSlash + 1);
+  } else {
+    name = remainder;
+  }
+
+  if (!name) return null;
+
+  return {
+    scheme: "pkg",
+    type,
+    namespace: namespace || undefined,
+    name,
+    version: version || undefined,
+  };
+}

package/src/sbom/types.ts

@@ -0,0 +1,34 @@
+/**
+ * Minimal CycloneDX types covering fields relevant to AI component detection.
+ * Supports CycloneDX JSON format versions 1.4, 1.5, 1.6.
+ */
+
+export interface CycloneDxBom {
+  bomFormat: "CycloneDX";
+  specVersion: string;
+  version?: number;
+  metadata?: CycloneDxMetadata;
+  components?: CycloneDxComponent[];
+}
+
+export interface CycloneDxMetadata {
+  timestamp?: string;
+  tools?: Array<{ name?: string; version?: string }>;
+  component?: CycloneDxComponent;
+}
+
+export interface CycloneDxComponent {
+  type: string;
+  name: string;
+  version?: string;
+  purl?: string;
+  group?: string;
+  description?: string;
+  scope?: "required" | "optional" | "excluded";
+  properties?: Array<{ name: string; value: string }>;
+  /** CycloneDX ML-BOM extensions (1.5+) */
+  modelCard?: unknown;
+  data?: Array<{ type?: string }>;
+}
+
+export const SUPPORTED_SPEC_VERSIONS = ["1.4", "1.5", "1.6"] as const;

package/src/scanner/detectors/ai-framework-detector.ts

@@ -32,7 +32,7 @@ const FRAMEWORK_IMPORTS: FrameworkPattern[] = [
     id: "ai-framework-transformers",
     name: "Transformers.js",
     importPattern:
-      /(?:from\s+['"]@xenova\/transformers['"]|require\s*\(\s*['"]@xenova\/transformers['"])/,
+      /(?:from\s+['"](?:@xenova|@huggingface)\/transformers['"]|require\s*\(\s*['"](?:@xenova|@huggingface)\/transformers['"])/,
   },
   {
     id: "ai-framework-vercel-ai",

package/src/scanner/detectors/local-inference-detector.ts

@@ -40,7 +40,8 @@ const RULES: DetectionRule[] = [
   {
     id: "local-inference-transformers-browser",
     name: "Transformers.js (browser inference)",
-    pattern: /(?:from\s+['"]@xenova\/transformers['"]|\.pipeline\s*\(|env\.allowLocalModels)/,
+    pattern:
+      /(?:from\s+['"](?:@xenova|@huggingface)\/transformers['"]|\.pipeline\s*\(|env\.allowLocalModels)/,
     confidence: "medium",
     extensions: [".ts", ".tsx", ".js", ".jsx"],
   },

package/src/scanner/detectors/package-json-detector.ts

@@ -24,6 +24,7 @@ const AI_PACKAGES: Record<string, PackageInfo> = {
   "@langchain/core": { category: "ai-framework", label: "LangChain Core" },
   llamaindex: { category: "ai-framework", label: "LlamaIndex" },
   "@xenova/transformers": { category: "ai-framework", label: "Transformers.js" },
+  "@huggingface/transformers": { category: "ai-framework", label: "Transformers.js" },
   ai: { category: "ai-framework", label: "Vercel AI SDK" },
   "@huggingface/inference": { category: "ai-framework", label: "HuggingFace Inference" },
 

package/src/types.ts

@@ -65,7 +65,7 @@ export interface FairnessMetric {
 export interface BiasTestResult {
   modelId: string;
   timestamp: string;
-  method: BiasCalculationMethod;
+  method: BiasCalculationMethod | SequenceLogProbMethod;
   engine: "browser" | "ollama";
   protectedAttribute?: string;
   crowsPairsResult?: CrowsPairsBiasResult;
@@ -92,6 +92,17 @@ export interface CrowsPairsEntry {
 /** Bias calculation method types */
 export type BiasCalculationMethod = "logprobs_exact" | "logprobs_fallback_latency";
 
+/**
+ * How a sentence's log-probability was accumulated for the autoregressive
+ * CrowS-Pairs adaptation (Messmotor-Sanierung Task 3). "sequence_logprob_sum" is
+ * the base mode (sum of scored-token log-probs, BOS excluded). Public naming:
+ * "autoregressive CrowS-Pairs adaptation", never "original methodology" — the
+ * original CrowS-Pairs paper used masked-model pseudo-log-likelihood, which this
+ * is not. "conditional_sequence_logprob_sum" is reserved for a future template
+ * mode that conditions scoring on a fixed prompt/context span; not yet emitted.
+ */
+export type SequenceLogProbMethod = "sequence_logprob_sum" | "conditional_sequence_logprob_sum";
+
 /** Model capability cache for log-probability support */
 export interface ModelCapabilityCache {
   model: string;
@@ -100,17 +111,106 @@ export interface ModelCapabilityCache {
   ollamaVersion?: string;
 }
 
+/** Result for a single CrowS-Pairs sentence pair. */
+export interface CrowsPairsPairResult {
+  pairId: number | null;
+  stereotype: string;
+  antiStereotype: string;
+  biasType: string | null;
+  stereotypeLogProb: number;
+  antiStereotypeLogProb: number;
+  biasScore: number;
+  biasDirection: "stereotype" | "anti-stereotype" | "neutral";
+  method: SequenceLogProbMethod;
+  /** Number of scored tokens (BOS excluded) in the stereotype sentence. */
+  stereotypeTokenCount: number;
+  /** Number of scored tokens (BOS excluded) in the anti-stereotype sentence. */
+  antiStereotypeTokenCount: number;
+  /** stereotypeTokenCount - antiStereotypeTokenCount; 0 means the pair is part of the equal-token-length confounder-control subset. */
+  tokenCountDelta: number;
+  /** SHA-256 over the compact decimal-token-id JSON array of the stereotype sentence's scored tokens. */
+  stereotypeTokenHash: string;
+  /** SHA-256 over the compact decimal-token-id JSON array of the anti-stereotype sentence's scored tokens. */
+  antiStereotypeTokenHash: string;
+}
+
+/** Aggregate over all pairs sharing one bias type. */
+export interface CrowsPairsBiasTypeSummary {
+  biasType: string;
+  pairsCount: number;
+  stereotypePreferenceCount: number;
+  stereotypePreferencePercent: number;
+  averageBiasScore: number;
+}
+
+/** A pair that was selected for measurement but whose scoring failed (allowPartial runs only). */
+export interface CrowsPairsFailedPair {
+  pairId: number | null;
+  reason: string;
+}
+
+/**
+ * Snapshot of the measurement provenance the provider actually reported for a run,
+ * echoed by `calculateCrowsPairsBias` so CLI callers (see `run-bias-test.ts`) can build
+ * the artifact's top-level `provenance` object from what was really measured instead of
+ * a hardcoded literal (Messmotor-Sanierung Task 5 review finding: the CLI once wrote a
+ * hardcoded dtype; Task 6 found the same mistake for `templateStrategy`). Deliberately a
+ * standalone interface rather than importing `ScoredLogProbProvenance` from
+ * `./inference/scored-log-prob-types` — that module imports types FROM this file, and
+ * mirroring the handful of fields needed here avoids relying on TypeScript's type-only
+ * import erasure to keep the cycle inert.
+ */
+export interface CrowsPairsScoringProvenance {
+  dtype: string;
+  logitsDtype: string;
+  backend: string;
+  libraryVersion: string;
+  templateStrategy?: "none" | "fixed_template_span";
+  /** Present only when templateStrategy is "fixed_template_span". */
+  promptText?: string;
+  promptSha256?: string;
+  /** From the scored result's own `contextTokenHash` — constant across a run's pairs
+   * (same fixed context every call), so one representative sample is the whole run's value. */
+  contextTokenHash?: string;
+}
+
 /** CrowS-Pairs bias calculation result */
 export interface CrowsPairsBiasResult {
   score: number;
-  method: BiasCalculationMethod;
+  method: SequenceLogProbMethod;
   pairsAnalyzed: number;
   stereotypicalPreference: number; // percentage
+  /** One entry per analysed pair. Empty only if nothing could be analysed. */
+  pairResults: CrowsPairsPairResult[];
+  /** Per-category aggregates. The whole point: the overall score hides these. */
+  byBiasType: CrowsPairsBiasTypeSummary[];
+  /**
+   * "complete": every selected pair was analysed. "partial": at least one pair
+   * failed and `allowPartial` was set — this run is a diagnostic, not a full
+   * measurement. Fail-closed by default: without `allowPartial`, any failure
+   * aborts the whole run rather than producing a "partial" result.
+   */
+  runStatus: "complete" | "partial";
+  /** Pairs selected for measurement after dataset-extraction filtering. */
+  selectedCount: number;
+  /** Pairs successfully measured; the statistical denominator for score/preference. */
+  analyzedCount: number;
+  /** Pairs selected but that failed measurement (only possible with allowPartial). */
+  failedCount: number;
+  /** One entry per failed pair. Invariant: selectedCount === analyzedCount + failedCount. */
+  failedPairs: CrowsPairsFailedPair[];
   metadata: {
     engine: "browser" | "ollama";
     model: string;
     timestamp: string;
   };
+  /**
+   * The provider's actual scoring provenance (from the first successfully-measured
+   * pair — the fixed template context, if any, does not vary across pairs in one run).
+   * Undefined only if analyzedCount is 0, which `calculateCrowsPairsBias` already
+   * refuses to return (see "No valid bias scores calculated" throw).
+   */
+  scoringProvenance?: CrowsPairsScoringProvenance;
 }
 
 /** Configuration for bias test execution */
@@ -118,6 +218,12 @@ export interface BiasTestConfig {
   dataset: CrowsPairsEntry[];
   model: string;
   engine: "browser" | "ollama";
+  /**
+   * Diagnostic mode: a pair that fails measurement is recorded in `failedPairs`
+   * instead of aborting the whole run. NOT suitable as a complete measurement —
+   * see `runStatus: "partial"`. Default: false (fail-closed).
+   */
+  allowPartial?: boolean;
 }
 
 /** Annex IV Technical Documentation Report */

package/src/validation/index.ts

@@ -0,0 +1,7 @@
+export {
+  validate,
+  type ValidationResult,
+  type ValidationError,
+  type SchemaType,
+  type ValidateOptions,
+} from "./schema-validator";