@euconform/core 1.4.0 → 1.6.0

package/src/evidence/types.ts

@@ -5,6 +5,9 @@
  * Classification is done by the wizard from human input, not by the scanner.
  */
 
+import type { CrowsPairsFailedPair, CrowsPairsPairResult, SequenceLogProbMethod } from "../types";
+import type { BiasStatisticsV2 } from "./bias-statistics";
+
 // ---------------------------------------------------------------------------
 // Signal types
 // ---------------------------------------------------------------------------
@@ -232,11 +235,17 @@ export interface BomComponent {
   kind: BomComponentKind;
   name: string;
   version?: string;
-  source: "package.json" | "lock-file" | "code" | "config" | "docs";
+  source: "package.json" | "lock-file" | "code" | "config" | "docs" | "sbom-import";
+}
+
+export interface AibomMetadata {
+  importSource: "cyclonedx" | "spdx" | "manual";
+  importTool?: string;
+  originalTimestamp?: string;
 }
 
 export interface AiBillOfMaterials {
-  schemaVersion: "euconform.aibom.v1";
+  schemaVersion: "euconform.aibom.v1" | "euconform.aibom.v1.1";
   generatedAt: string;
   project: {
     name: string;
@@ -251,6 +260,7 @@ export interface AiBillOfMaterials {
     humanReviewFlow: boolean;
     incidentHandling: boolean;
   };
+  metadata?: AibomMetadata;
 }
 
 // ---------------------------------------------------------------------------
@@ -305,11 +315,143 @@ export interface ScanOutput {
   summaryMarkdown: string;
 }
 
+// ---------------------------------------------------------------------------
+// Bias report (euconform.bias.v1)
+// ---------------------------------------------------------------------------
+
+export interface BiasReport {
+  schemaVersion: "euconform.bias.v1";
+  generatedAt: string;
+  model: string;
+  dataset: string;
+  result: {
+    score: number;
+    method: string;
+    pairsAnalyzed: number;
+    stereotypicalPreference: number;
+  };
+  /** Absent in runs written before sampling was recorded; null fields mean the full dataset. */
+  sampling?: {
+    datasetSize: number;
+    maxPairs: number | null;
+    seed: number | null;
+  };
+  limitations?: string[];
+  /** Provenance of a user-supplied question catalog. Absent when a bundled dataset was used. */
+  catalog?: {
+    name: string;
+    version: string;
+    sha256: string;
+    pairCount: number;
+  };
+  /** Per-category aggregates over pairResults. Optional so pre-existing artifacts stay valid. */
+  byBiasType?: {
+    biasType: string;
+    pairsCount: number;
+    stereotypePreferenceCount: number;
+    stereotypePreferencePercent: number;
+    averageBiasScore: number;
+  }[];
+  /** One entry per analysed pair, enabling third-party recomputation of the score. */
+  pairResults?: {
+    pairId: number | null;
+    stereotype: string;
+    antiStereotype: string;
+    biasType: string | null;
+    stereotypeLogProb: number;
+    antiStereotypeLogProb: number;
+    biasScore: number;
+    biasDirection: "stereotype" | "anti-stereotype" | "neutral";
+    method: string;
+  }[];
+}
+
+// ---------------------------------------------------------------------------
+// Bias report v2 (euconform.bias.v2) — Messmotor-Sanierung Task 3.
+//
+// v1 is demoted to legacy (see validateBiasArtifact): its "logprobs_exact"
+// method label sat on artifacts that, before Tasks 1-2, measured wall-clock
+// nanoseconds rather than log-probabilities. v2 makes provenance mandatory
+// and statistics self-checkable (the shared semantic validator recomputes
+// them from pairResults and rejects any mismatch).
+// ---------------------------------------------------------------------------
+
+/** Mandatory measurement provenance — v2's core honesty guarantee. */
+export interface BiasReportV2Provenance {
+  /** Single supported engine (Ollama cannot measure sentence log-probabilities). */
+  engine: "transformers";
+  modelId: string;
+  /** Resolved immutable commit SHA (40 hex chars), never a mutable ref like "main". */
+  modelRevision: string;
+  /** The tokenizer actually used; equals modelId unless a distinct tokenizer repo was loaded. */
+  tokenizerId: string;
+  transformersVersion: string;
+  /**
+   * Installed onnxruntime-node version. Ergebnisrelevant: a native ONNX Runtime version
+   * bump (1.14.0 -> 1.24.3 across the @xenova/transformers -> @huggingface/transformers
+   * migration) shifted sequence log-probabilities at the 1e-6 relative level even with
+   * byte-identical model weights and dtype (Messmotor-Sanierung Task 4). Must be resolved
+   * from the installed package at measurement time; an artifact that cannot resolve it is
+   * a loud failure, not a placeholder.
+   */
+  onnxruntimeVersion: string;
+  /** Numeric precision used for scoring, e.g. "fp32" or "q8" — ergebnisrelevant (Task 1: up to 11.9% deviation quantized vs fp32). */
+  dtype: string;
+  backend: "node-onnx";
+  templateStrategy: "none" | "fixed_template_span";
+  /** Present only when templateStrategy is "fixed_template_span". */
+  promptText?: string;
+  promptSha256?: string;
+  contextTokenHash?: string;
+}
+
+export interface BiasReportV2Sampling {
+  mode: "seeded_sample" | "full";
+  datasetSize: number;
+  maxPairs: number | null;
+  seed: number | null;
+  /** SHA-256 over the compact JSON of the full dataset that was sampled from. */
+  datasetSha256: string;
+  /** Pair ids in selection order — the exact reproducible sample. */
+  pairIds: (number | null)[];
+}
+
+export interface BiasReportV2 {
+  schemaVersion: "euconform.bias.v2";
+  generatedAt: string;
+  model: string;
+  dataset: string;
+  provenance: BiasReportV2Provenance;
+  sampling: BiasReportV2Sampling;
+  result: {
+    score: number;
+    method: SequenceLogProbMethod;
+    pairsAnalyzed: number;
+  };
+  runStatus: "complete" | "partial";
+  selectedCount: number;
+  analyzedCount: number;
+  failedCount: number;
+  failedPairs: CrowsPairsFailedPair[];
+  /** Self-checkable: the semantic validator recomputes this from pairResults and rejects on mismatch (tolerance 1e-9, unrounded). */
+  statistics: BiasStatisticsV2;
+  /** One entry per analysed pair — the raw values statistics/validation recompute from. */
+  pairResults: CrowsPairsPairResult[];
+  limitations?: string[];
+  /** Provenance of a user-supplied question catalog. Absent when a bundled dataset was used. */
+  catalog?: {
+    name: string;
+    version: string;
+    sha256: string;
+    pairCount: number;
+  };
+}
+
 // ---------------------------------------------------------------------------
 // Bundle manifest (euconform.bundle.v1)
 // ---------------------------------------------------------------------------
 
-export type BundleArtifactRole = "report" | "aibom" | "ci" | "summary";
+export type BundleArtifactRole = "report" | "aibom" | "ci" | "summary" | "bias";
 
 export interface BundleArtifactRef {
   role: BundleArtifactRole;
@@ -327,3 +469,71 @@ export interface ScanBundle {
   target: { name: string; rootPath: string };
   artifacts: BundleArtifactRef[];
 }
+
+// ---------------------------------------------------------------------------
+// External observations (euconform.external-observation.v1)
+// ---------------------------------------------------------------------------
+
+/**
+ * Provider-neutral envelope for an externally produced observation of a
+ * machine-readable declaration (rights, AI use, licence metadata, robots policy).
+ *
+ * It records what was visible at a point in time. It is NOT a licence, NOT an
+ * ownership determination, NOT proof of use and NOT permission — `result` only
+ * says whether a declaration was visible, and `unknown` must never be read as
+ * permission. The shape intentionally has no field that could express permission.
+ */
+export interface ExternalObservation {
+  schemaVersion: "euconform.external-observation.v1";
+  /** Provider-scoped unique id, so the exact record can be cited and re-fetched. */
+  observationId: string;
+  /** When the provider observed the declaration (not when the artifact was exported). */
+  observedAt: string;
+  /** Who produced the observation. Identification only; it does not change the meaning. */
+  provider: {
+    name: string;
+    version: string;
+    endpoint?: string;
+    contact?: string;
+  };
+  subject: {
+    kind: "url" | "repository" | "dataset" | "artifact";
+    identifier: string;
+    /** When the subject content was fetched; bounds the validity of the statement. */
+    retrievedAt: string;
+    /** Optional hash of the observed content. */
+    contentSha256?: string;
+  };
+  observation: {
+    kind:
+      | "rights-declaration"
+      | "ai-use-declaration"
+      | "license-metadata"
+      | "robots-policy"
+      | "terms-of-service";
+    /** Query vocabulary of the provider (e.g. ai-input, ai-train). Free-form on purpose. */
+    intendedUse: string[];
+    /**
+     * `declared` = a supported machine-readable declaration was visible — NOT that
+     * the use is permitted. `unknown` must never be converted into permission.
+     */
+    result: "declared" | "not-declared" | "unknown" | "not-applicable";
+    /** Present exactly when `result` is `declared`, so the claim stays checkable. */
+    declaration?: {
+      text: string;
+      sha256: string;
+      url?: string;
+      license?: string;
+    };
+  };
+  /** Mandatory coverage limits; an observation without stated limits is not reviewable. */
+  limitations: string[];
+  /** Optional integrity proof. Unsigned observations are valid but must be treated as unverified. */
+  signature?: {
+    algorithm: string;
+    keyId: string;
+    value: string;
+    signedPayloadSha256: string;
+    verifyUrl?: string;
+  };
+}

package/src/evidence/validate-bias-v2.ts

@@ -0,0 +1,47 @@
+/**
+ * The ONE validator entrypoint for `euconform.bias.v2` artifacts
+ * (Messmotor-Sanierung Task 3). Both `euconform validate` and `euconform verify`
+ * call this — before this task they used different validators (AJV-only vs. the
+ * lenient hand-rolled `validateBiasReport`) that could disagree on the same file.
+ *
+ * Validation has two layers:
+ *  1. Schema shape — delegated to the AJV schema (`bias-v2.schema.json` via
+ *     `packages/core/src/validation/schema-validator.ts`), so there is exactly
+ *     one source of truth for "what fields must exist". Node-only (reads the
+ *     schema from disk); the browser calls `assertBiasV2Semantics` from
+ *     `@euconform/core/evidence/browser` for layer 2.
+ *  2. Semantic recomputation — every statistic, and the partial-run invariants,
+ *     are recomputed from the raw `pairResults` and compared against the stored
+ *     values with an absolute tolerance of 1e-9 (ambiguity resolution #2:
+ *     statistics are stored UNROUNDED; rounding only happens in the Markdown
+ *     report, so a tolerance far below any human-rounding step is meaningful).
+ *     A mismatch means the artifact's numbers do not follow from its own raw
+ *     data — e.g. a manipulated Wilson bound — and the artifact is rejected
+ *     with a message naming exactly which field disagreed.
+ *     This layer lives in `bias-v2-semantics.ts` so it can also run in the browser.
+ */
+
+import { validate as validateAgainstSchema } from "../validation/schema-validator";
+import { assertBiasV2Semantics } from "./bias-v2-semantics";
+import type { BiasReportV2 } from "./types";
+
+/**
+ * `verify --strict` rejects a partial run outright (ambiguity resolution #3):
+ * a diagnostic run is, by definition, not the complete measurement a strict
+ * check demands. Non-strict callers get this back as a plain boolean instead.
+ */
+export function isPartialRun(report: BiasReportV2): boolean {
+  return report.runStatus === "partial";
+}
+
+export function validateBiasReportV2(data: unknown): BiasReportV2 {
+  const schemaResult = validateAgainstSchema(data);
+  if (!schemaResult.valid) {
+    const messages = schemaResult.errors.map((e) => `${e.path || "/"} ${e.message}`).join("; ");
+    throw new Error(
+      `Invalid euconform.bias.v2 artifact: ${messages || "schema validation failed"}`
+    );
+  }
+
+  return assertBiasV2Semantics(data);
+}

package/src/evidence/validate.ts

@@ -1,4 +1,27 @@
-import type { AiBillOfMaterials, CiReport, ScanBundle, ScanReport } from "./types";
+import type {
+  AiBillOfMaterials,
+  BiasReport,
+  BiasReportV2,
+  CiReport,
+  ExternalObservation,
+  ScanBundle,
+  ScanReport,
+} from "./types";
+import { validateBiasReportV2 } from "./validate-bias-v2";
+
+const V1_BIAS_LEGACY_WARNING =
+  "This artifact uses euconform.bias.v1: the method label 'logprobs_exact' on v1 bias " +
+  "artifacts is unreliable (Messmotor-Sanierung Task 1/2 found the measurement path it " +
+  "described could silently fall back to wall-clock nanoseconds instead of real " +
+  "log-probabilities). Re-measure with `euconform bias --engine transformers` to get a " +
+  "euconform.bias.v2 artifact.";
+
+/** Result of validating any bias artifact, v1 (legacy) or v2. */
+export interface BiasValidationResult {
+  legacy: boolean;
+  warning?: string;
+  document: BiasReport | BiasReportV2;
+}
 
 function assertObject(data: unknown, label: string): Record<string, unknown> {
   if (!data || typeof data !== "object" || Array.isArray(data)) {
@@ -7,10 +30,16 @@ function assertObject(data: unknown, label: string): Record<string, unknown> {
   return data as Record<string, unknown>;
 }
 
-function requireSchemaVersion(obj: Record<string, unknown>, expected: string, label: string): void {
-  if (obj.schemaVersion !== expected) {
+function requireSchemaVersion(
+  obj: Record<string, unknown>,
+  expected: string | string[],
+  label: string
+): void {
+  const allowed = Array.isArray(expected) ? expected : [expected];
+  if (!allowed.includes(obj.schemaVersion as string)) {
+    const quoted = allowed.map((v) => `"${v}"`).join(" or ");
     throw new Error(
-      `Invalid ${label} schema version: expected "${expected}", got "${String(obj.schemaVersion)}"`
+      `Invalid ${label} schema version: expected ${quoted}, got "${String(obj.schemaVersion)}"`
     );
   }
 }
@@ -102,7 +131,7 @@ export function validateScanReport(data: unknown): ScanReport {
 
 export function validateAiBillOfMaterials(data: unknown): AiBillOfMaterials {
   const obj = assertObject(data, "AIBOM");
-  requireSchemaVersion(obj, "euconform.aibom.v1", "AIBOM");
+  requireSchemaVersion(obj, ["euconform.aibom.v1", "euconform.aibom.v1.1"], "AIBOM");
   requireField(obj, "generatedAt", "string", "AIBOM");
 
   const project = requireField(obj, "project", "object", "AIBOM") as Record<string, unknown>;
@@ -149,7 +178,7 @@ export function validateCiReport(data: unknown): CiReport {
   return data as CiReport;
 }
 
-const VALID_BUNDLE_ROLES = new Set(["report", "aibom", "ci", "summary"]);
+const VALID_BUNDLE_ROLES = new Set(["report", "aibom", "ci", "summary", "bias"]);
 const SHA256_HEX = /^[a-f0-9]{64}$/;
 
 interface BundleArtifactFields {
@@ -207,13 +236,20 @@ function readBundleArtifactFields(
   };
 }
 
+/**
+ * Bundle decoupling (Messmotor-Sanierung Task 3): artifact schema versions are
+ * independent of the bundle's own schema version. A `euconform.bundle.v1`
+ * manifest may reference a `euconform.bias.v2` artifact — the bundle's job is
+ * to bind artifacts by hash and role, not to pin their versions. There is
+ * therefore no "same major as the bundle" check here; per-artifact schema
+ * shape is the responsibility of that artifact's own validator.
+ */
 function assertBundleArtifactIntegrity(
   fields: BundleArtifactFields,
   seenRoles: Set<string>,
-  seenFiles: Set<string>,
-  bundleMajor: string | null
+  seenFiles: Set<string>
 ): void {
-  const { role, fileName, sha256, declaredSchemaVersion } = fields;
+  const { role, fileName, sha256 } = fields;
 
   if (!VALID_BUNDLE_ROLES.has(role)) {
     throw new Error(`Invalid bundle artifact: unsupported role '${role}'`);
@@ -227,11 +263,6 @@ function assertBundleArtifactIntegrity(
   if (seenFiles.has(fileName)) {
     throw new Error(`Invalid bundle artifact: duplicate fileName '${fileName}'`);
   }
-  if (declaredSchemaVersion && bundleMajor && majorVersion(declaredSchemaVersion) !== bundleMajor) {
-    throw new Error(
-      `Invalid bundle artifact: '${fileName}' uses schemaVersion '${declaredSchemaVersion}' with a different major than the bundle`
-    );
-  }
 }
 
 function validateBundleHeader(obj: Record<string, unknown>): void {
@@ -247,6 +278,107 @@ function validateBundleHeader(obj: Record<string, unknown>): void {
   requireField(target, "rootPath", "string", "bundle", "target.rootPath");
 }
 
+export function validateBiasReport(data: unknown): BiasReport {
+  const obj = assertObject(data, "bias report");
+  requireSchemaVersion(obj, "euconform.bias.v1", "bias report");
+  requireField(obj, "generatedAt", "string", "bias report");
+  requireField(obj, "model", "string", "bias report");
+  requireField(obj, "dataset", "string", "bias report");
+
+  const result = requireField(obj, "result", "object", "bias report") as Record<string, unknown>;
+  requireField(result, "score", "number", "bias report", "result.score");
+  requireField(result, "method", "string", "bias report", "result.method");
+  requireField(result, "pairsAnalyzed", "number", "bias report", "result.pairsAnalyzed");
+  requireField(
+    result,
+    "stereotypicalPreference",
+    "number",
+    "bias report",
+    "result.stereotypicalPreference"
+  );
+
+  return data as BiasReport;
+}
+
+/**
+ * The ONE entry point for bias artifacts, dispatching on schemaVersion.
+ * `validate` (CLI) and `verify` both call this instead of picking their own
+ * validator, so they can no longer disagree on the same file.
+ *
+ * v1 keeps its existing (lenient) validation behaviour EXACTLY as before —
+ * `validateBiasReport` is untouched — but is now flagged `legacy: true` with a
+ * warning naming the incident. v2 is delegated to the shared semantic
+ * validator (`validate-bias-v2.ts`), which also recomputes statistics.
+ */
+export function validateBiasArtifact(data: unknown): BiasValidationResult {
+  const obj = assertObject(data, "bias report");
+  if (obj.schemaVersion === "euconform.bias.v2") {
+    return { legacy: false, document: validateBiasReportV2(data) };
+  }
+  return {
+    legacy: true,
+    warning: V1_BIAS_LEGACY_WARNING,
+    document: validateBiasReport(data),
+  };
+}
+
+/**
+ * Shape check for `euconform.external-observation.v1`. The schema is the source
+ * of truth for shape (`euconform validate` delegates to it); this function is the
+ * programmatic entrypoint, mirroring the other document validators.
+ *
+ * It deliberately performs no interpretation: whether a declaration exists is
+ * data, and nothing here may turn `unknown` into permission.
+ */
+export function validateExternalObservation(data: unknown): ExternalObservation {
+  const label = "external observation";
+  const obj = assertObject(data, label);
+  requireSchemaVersion(obj, "euconform.external-observation.v1", label);
+  requireField(obj, "observationId", "string", label);
+  requireField(obj, "observedAt", "string", label);
+
+  const provider = requireField(obj, "provider", "object", label) as Record<string, unknown>;
+  requireField(provider, "name", "string", label, "provider.name");
+  requireField(provider, "version", "string", label, "provider.version");
+
+  const subject = requireField(obj, "subject", "object", label) as Record<string, unknown>;
+  requireField(subject, "kind", "string", label, "subject.kind");
+  requireField(subject, "identifier", "string", label, "subject.identifier");
+  requireField(subject, "retrievedAt", "string", label, "subject.retrievedAt");
+
+  const observation = requireField(obj, "observation", "object", label) as Record<string, unknown>;
+  requireField(observation, "kind", "string", label, "observation.kind");
+  requireStringArray(observation, "intendedUse", label, "observation.intendedUse");
+  const result = requireField(
+    observation,
+    "result",
+    "string",
+    label,
+    "observation.result"
+  ) as string;
+
+  if (result === "declared") {
+    const declaration = requireField(
+      observation,
+      "declaration",
+      "object",
+      label,
+      "observation.declaration"
+    ) as Record<string, unknown>;
+    requireField(declaration, "text", "string", label, "observation.declaration.text");
+    requireField(declaration, "sha256", "string", label, "observation.declaration.sha256");
+  }
+
+  const limitations = requireStringArray(obj, "limitations", label);
+  if (limitations.length === 0) {
+    throw new Error(
+      `Invalid ${label}: 'limitations' must name at least one coverage limit — an observation without stated limits is not reviewable`
+    );
+  }
+
+  return data as ExternalObservation;
+}
+
 export function validateScanBundle(data: unknown): ScanBundle {
   const obj = assertObject(data, "bundle");
   validateBundleHeader(obj);
@@ -258,13 +390,12 @@ export function validateScanBundle(data: unknown): ScanBundle {
 
   const seenRoles = new Set<string>();
   const seenFiles = new Set<string>();
-  const bundleMajor = majorVersion("euconform.bundle.v1");
   let hasRequiredReport = false;
 
   for (let index = 0; index < artifacts.length; index++) {
     const raw = assertObject(artifacts[index], "bundle artifact");
     const fields = readBundleArtifactFields(raw, index);
-    assertBundleArtifactIntegrity(fields, seenRoles, seenFiles, bundleMajor);
+    assertBundleArtifactIntegrity(fields, seenRoles, seenFiles);
 
     seenRoles.add(fields.role);
     seenFiles.add(fields.fileName);
@@ -284,16 +415,19 @@ export function validateScanBundle(data: unknown): ScanBundle {
 export function validateEcefJsonDocument(
   schemaVersion: string,
   data: unknown
-): ScanReport | AiBillOfMaterials | CiReport | ScanBundle {
+): ScanReport | AiBillOfMaterials | CiReport | ScanBundle | ExternalObservation {
   switch (schemaVersion) {
     case "euconform.report.v1":
       return validateScanReport(data);
     case "euconform.aibom.v1":
+    case "euconform.aibom.v1.1":
       return validateAiBillOfMaterials(data);
     case "euconform.ci.v1":
       return validateCiReport(data);
     case "euconform.bundle.v1":
       return validateScanBundle(data);
+    case "euconform.external-observation.v1":
+      return validateExternalObservation(data);
     default:
       throw new Error(`Unsupported EuConform Evidence Format schemaVersion '${schemaVersion}'`);
   }