@euconform/core 1.5.0 → 1.6.0

package/src/evidence/bias-v2-semantics.ts

@@ -0,0 +1,214 @@
+/**
+ * Pure semantic checks for `euconform.bias.v2` artifacts (no Node builtins).
+ *
+ * Split out of `validate-bias-v2.ts` so the exact same recomputation can run in
+ * the browser (`@euconform/core/evidence/browser`). The schema-shape check stays
+ * Node-side because it reads the published JSON schema from disk; the semantic
+ * layer — "do the stored numbers follow from the raw pair results?" — must never
+ * have two implementations.
+ */
+
+import type { CrowsPairsPairResult } from "../types";
+import { type BiasStatisticsV2, type WilsonInterval, buildBiasStatistics } from "./bias-statistics";
+import type { BiasReportV2 } from "./types";
+
+const TOLERANCE = 1e-9;
+
+function closeEnough(a: number, b: number): boolean {
+  return Math.abs(a - b) <= TOLERANCE;
+}
+
+function fail(message: string): never {
+  throw new Error(`Invalid euconform.bias.v2 artifact: ${message}`);
+}
+
+function assertNumberMatches(label: string, expected: number, actual: number): void {
+  if (!closeEnough(expected, actual)) {
+    fail(
+      `${label} does not match recomputation from pairResults: stored ${actual}, recomputed ${expected}`
+    );
+  }
+}
+
+function assertWilsonMatches(
+  label: string,
+  expected: WilsonInterval | null,
+  actual: WilsonInterval | null
+): void {
+  if (expected === null || actual === null) {
+    if (expected !== actual) {
+      fail(
+        `${label} should be ${expected === null ? "null" : "a Wilson interval"}, got ${actual === null ? "null" : JSON.stringify(actual)}`
+      );
+    }
+    return;
+  }
+  assertNumberMatches(`${label}.lower`, expected.lower, actual.lower);
+  assertNumberMatches(`${label}.upper`, expected.upper, actual.upper);
+}
+
+function assertQuartilesMatch(
+  expected: BiasStatisticsV2["pairDiffQuartiles"],
+  actual: BiasStatisticsV2["pairDiffQuartiles"]
+): void {
+  if (expected === null || actual === null) {
+    if (expected !== actual) {
+      fail(
+        `statistics.pairDiffQuartiles should be ${expected === null ? "null" : "present"}, got ${actual === null ? "null" : "present"}`
+      );
+    }
+    return;
+  }
+  for (const key of ["min", "max", "median", "q1", "q3"] as const) {
+    assertNumberMatches(`statistics.pairDiffQuartiles.${key}`, expected[key], actual[key]);
+  }
+}
+
+function assertCategoryStatisticsMatch(
+  expected: BiasStatisticsV2["byBiasType"],
+  actual: BiasStatisticsV2["byBiasType"]
+): void {
+  if (expected.length !== actual.length) {
+    fail(
+      `statistics.byBiasType has ${actual.length} categories, recomputation from pairResults yields ${expected.length}`
+    );
+  }
+  const actualByType = new Map(actual.map((c) => [c.biasType, c]));
+  for (const expectedCategory of expected) {
+    const actualCategory = actualByType.get(expectedCategory.biasType);
+    if (!actualCategory) {
+      fail(`statistics.byBiasType is missing category '${expectedCategory.biasType}'`);
+    }
+    const label = `statistics.byBiasType['${expectedCategory.biasType}']`;
+    if (expectedCategory.analyzedCount !== actualCategory.analyzedCount) {
+      fail(
+        `${label}.analyzedCount: stored ${actualCategory.analyzedCount}, recomputed ${expectedCategory.analyzedCount}`
+      );
+    }
+    if (expectedCategory.preferenceCount !== actualCategory.preferenceCount) {
+      fail(
+        `${label}.preferenceCount: stored ${actualCategory.preferenceCount}, recomputed ${expectedCategory.preferenceCount}`
+      );
+    }
+    if (expectedCategory.tiesCount !== actualCategory.tiesCount) {
+      fail(
+        `${label}.tiesCount: stored ${actualCategory.tiesCount}, recomputed ${expectedCategory.tiesCount}`
+      );
+    }
+    assertNumberMatches(
+      `${label}.preferencePercent`,
+      expectedCategory.preferencePercent,
+      actualCategory.preferencePercent
+    );
+    assertWilsonMatches(
+      `${label}.preferenceWilson95`,
+      expectedCategory.preferenceWilson95,
+      actualCategory.preferenceWilson95
+    );
+  }
+}
+
+function assertStatisticsMatch(expected: BiasStatisticsV2, actual: BiasStatisticsV2): void {
+  if (expected.analyzedCount !== actual.analyzedCount) {
+    fail(
+      `statistics.analyzedCount: stored ${actual.analyzedCount}, recomputed ${expected.analyzedCount}`
+    );
+  }
+  if (expected.preferenceCount !== actual.preferenceCount) {
+    fail(
+      `statistics.preferenceCount: stored ${actual.preferenceCount}, recomputed ${expected.preferenceCount}`
+    );
+  }
+  if (expected.tiesCount !== actual.tiesCount) {
+    fail(`statistics.tiesCount: stored ${actual.tiesCount}, recomputed ${expected.tiesCount}`);
+  }
+  assertNumberMatches(
+    "statistics.preferencePercent",
+    expected.preferencePercent,
+    actual.preferencePercent
+  );
+  assertWilsonMatches(
+    "statistics.preferenceWilson95",
+    expected.preferenceWilson95,
+    actual.preferenceWilson95
+  );
+  assertCategoryStatisticsMatch(expected.byBiasType, actual.byBiasType);
+  assertQuartilesMatch(expected.pairDiffQuartiles, actual.pairDiffQuartiles);
+  if (expected.equalTokenPairsCount !== actual.equalTokenPairsCount) {
+    fail(
+      `statistics.equalTokenPairsCount: stored ${actual.equalTokenPairsCount}, recomputed ${expected.equalTokenPairsCount}`
+    );
+  }
+  if (expected.equalTokenPreferenceCount !== actual.equalTokenPreferenceCount) {
+    fail(
+      `statistics.equalTokenPreferenceCount: stored ${actual.equalTokenPreferenceCount}, recomputed ${expected.equalTokenPreferenceCount}`
+    );
+  }
+  if (
+    expected.equalTokenPreferencePercent === null ||
+    actual.equalTokenPreferencePercent === null
+  ) {
+    if (expected.equalTokenPreferencePercent !== actual.equalTokenPreferencePercent) {
+      fail(
+        "statistics.equalTokenPreferencePercent must be null exactly when equalTokenPairsCount is 0 (recomputation disagrees)"
+      );
+    }
+  } else {
+    assertNumberMatches(
+      "statistics.equalTokenPreferencePercent",
+      expected.equalTokenPreferencePercent,
+      actual.equalTokenPreferencePercent
+    );
+  }
+  assertWilsonMatches(
+    "statistics.equalTokenPreferenceWilson95",
+    expected.equalTokenPreferenceWilson95,
+    actual.equalTokenPreferenceWilson95
+  );
+}
+
+function assertPartialInvariants(report: BiasReportV2): void {
+  if (report.selectedCount !== report.analyzedCount + report.failedCount) {
+    fail(
+      `selectedCount (${report.selectedCount}) must equal analyzedCount (${report.analyzedCount}) + failedCount (${report.failedCount})`
+    );
+  }
+  if (report.failedPairs.length !== report.failedCount) {
+    fail(
+      `failedPairs.length (${report.failedPairs.length}) must equal failedCount (${report.failedCount})`
+    );
+  }
+  if (report.runStatus === "complete" && report.failedCount !== 0) {
+    fail(`runStatus is 'complete' but failedCount is ${report.failedCount}`);
+  }
+  if (report.runStatus === "partial" && report.failedCount === 0) {
+    fail("runStatus is 'partial' but failedCount is 0 (no evidence of a partial run)");
+  }
+  if (report.result.pairsAnalyzed !== report.analyzedCount) {
+    fail(
+      `result.pairsAnalyzed (${report.result.pairsAnalyzed}) must equal analyzedCount (${report.analyzedCount})`
+    );
+  }
+  if (report.pairResults.length !== report.analyzedCount) {
+    fail(
+      `pairResults.length (${report.pairResults.length}) must equal analyzedCount (${report.analyzedCount})`
+    );
+  }
+}
+
+export function recomputedBiasScore(pairResults: CrowsPairsPairResult[]): number {
+  return pairResults.reduce((sum, p) => sum + p.biasScore, 0) / pairResults.length;
+}
+
+/**
+ * Recomputes every stored statistic from `pairResults` and rejects any artifact
+ * whose stored values do not follow from its own raw data (tolerance 1e-9, i.e.
+ * far below any human-rounding step). Throws on mismatch.
+ */
+export function assertBiasV2Semantics(data: unknown): BiasReportV2 {
+  const report = data as BiasReportV2;
+  assertPartialInvariants(report);
+  assertNumberMatches("result.score", recomputedBiasScore(report.pairResults), report.result.score);
+  assertStatisticsMatch(buildBiasStatistics(report.pairResults), report.statistics);
+  return report;
+}

package/src/evidence/browser.ts

@@ -0,0 +1,35 @@
+/**
+ * Browser-safe subset of `@euconform/core/evidence`.
+ *
+ * Everything exported here is pure TypeScript with no Node builtins (`node:fs`,
+ * `node:path`, `node:url`, `node:crypto`), so it can be bundled into the web app.
+ * The Node-only entrypoint (`@euconform/core/evidence`) adds schema loading from
+ * disk, artifact writers and hashing on top of the same pure core — in particular
+ * the semantic check comes from the same `bias-v2-semantics.ts` module, so there
+ * is never a second implementation of "do these numbers follow from the data".
+ */
+
+export type {
+  BiasComparison,
+  BiasComparisonPaired,
+  BiasPairChange,
+  BiasRunSummary,
+  BiasTypeComparison,
+  CompareBiasRunsOptions,
+} from "./bias-comparison";
+export type {
+  BiasStatisticsV2,
+  CategoryPreferenceStatistics,
+  Quartiles,
+  WilsonInterval,
+} from "./bias-statistics";
+export type {
+  BiasReportV2,
+  BiasReportV2Provenance,
+  BiasReportV2Sampling,
+} from "./types";
+
+export { compareBiasRuns, mcnemarExactPValue, summarizeBiasRun } from "./bias-comparison";
+export { formatBiasComparisonMarkdown } from "./bias-comparison-markdown";
+export { buildBiasStatistics, quartilesType7, wilson95, WILSON_Z95 } from "./bias-statistics";
+export { assertBiasV2Semantics, recomputedBiasScore } from "./bias-v2-semantics";

package/src/evidence/bundle.ts

@@ -10,6 +10,7 @@ interface BuildBundleOptions {
   report: ArtifactInput;
   aibom?: ArtifactInput;
   ci?: ArtifactInput;
+  bias?: ArtifactInput;
   summary?: ArtifactInput;
   tool: { name: string; version: string };
   target: { name: string; rootPath: string };
@@ -53,6 +54,7 @@ export function buildBundleManifest(opts: BuildBundleOptions): ScanBundle {
 
   if (opts.aibom) artifacts.push(makeArtifactRef("aibom", opts.aibom, false));
   if (opts.ci) artifacts.push(makeArtifactRef("ci", opts.ci, false));
+  if (opts.bias) artifacts.push(makeArtifactRef("bias", opts.bias, false));
   if (opts.summary) artifacts.push(makeArtifactRef("summary", opts.summary, false));
 
   return {

package/src/evidence/index.ts

@@ -9,6 +9,10 @@ export type {
   AiBillOfMaterials,
   AppType,
   AssessmentHint,
+  BiasReport,
+  BiasReportV2,
+  BiasReportV2Provenance,
+  BiasReportV2Sampling,
   BomComponent,
   BomComponentKind,
   BundleArtifactRef,
@@ -18,6 +22,7 @@ export type {
   ConfidenceLevel,
   DetectedSignal,
   DetectorContext,
+  ExternalObservation,
   FailOnLevel,
   FileProvenance,
   GapCounts,
@@ -36,16 +41,42 @@ export type {
   SignalDetector,
   SignalEvidence,
 } from "./types";
+export type {
+  BiasStatisticsV2,
+  CategoryPreferenceStatistics,
+  Quartiles,
+  WilsonInterval,
+} from "./bias-statistics";
+export type {
+  BiasComparison,
+  BiasComparisonPaired,
+  BiasPairChange,
+  BiasRunSummary,
+  BiasTypeComparison,
+  CompareBiasRunsOptions,
+} from "./bias-comparison";
+export type { BiasValidationResult } from "./validate";
 
 export { generateScanOutput } from "./output";
 export { generateSummaryMarkdown } from "./markdown";
 export { buildBundleManifest } from "./bundle";
 export { sha256Hex } from "./hash";
+export { buildBiasStatistics, quartilesType7, wilson95, WILSON_Z95 } from "./bias-statistics";
+export {
+  compareBiasRuns,
+  mcnemarExactPValue,
+  summarizeBiasRun,
+} from "./bias-comparison";
+export { formatBiasComparisonMarkdown } from "./bias-comparison-markdown";
+export { validateBiasReportV2, isPartialRun } from "./validate-bias-v2";
 export {
   getSchemaMajorVersion,
   validateAiBillOfMaterials,
+  validateBiasArtifact,
+  validateBiasReport,
   validateCiReport,
   validateEcefJsonDocument,
+  validateExternalObservation,
   validateScanBundle,
   validateScanReport,
 } from "./validate";

package/src/evidence/types.ts

@@ -5,6 +5,9 @@
  * Classification is done by the wizard from human input, not by the scanner.
  */
 
+import type { CrowsPairsFailedPair, CrowsPairsPairResult, SequenceLogProbMethod } from "../types";
+import type { BiasStatisticsV2 } from "./bias-statistics";
+
 // ---------------------------------------------------------------------------
 // Signal types
 // ---------------------------------------------------------------------------
@@ -312,11 +315,143 @@ export interface ScanOutput {
   summaryMarkdown: string;
 }
 
+// ---------------------------------------------------------------------------
+// Bias report (euconform.bias.v1)
+// ---------------------------------------------------------------------------
+
+export interface BiasReport {
+  schemaVersion: "euconform.bias.v1";
+  generatedAt: string;
+  model: string;
+  dataset: string;
+  result: {
+    score: number;
+    method: string;
+    pairsAnalyzed: number;
+    stereotypicalPreference: number;
+  };
+  /** Absent in runs written before sampling was recorded; null fields mean the full dataset. */
+  sampling?: {
+    datasetSize: number;
+    maxPairs: number | null;
+    seed: number | null;
+  };
+  limitations?: string[];
+  /** Provenance of a user-supplied question catalog. Absent when a bundled dataset was used. */
+  catalog?: {
+    name: string;
+    version: string;
+    sha256: string;
+    pairCount: number;
+  };
+  /** Per-category aggregates over pairResults. Optional so pre-existing artifacts stay valid. */
+  byBiasType?: {
+    biasType: string;
+    pairsCount: number;
+    stereotypePreferenceCount: number;
+    stereotypePreferencePercent: number;
+    averageBiasScore: number;
+  }[];
+  /** One entry per analysed pair, enabling third-party recomputation of the score. */
+  pairResults?: {
+    pairId: number | null;
+    stereotype: string;
+    antiStereotype: string;
+    biasType: string | null;
+    stereotypeLogProb: number;
+    antiStereotypeLogProb: number;
+    biasScore: number;
+    biasDirection: "stereotype" | "anti-stereotype" | "neutral";
+    method: string;
+  }[];
+}
+
+// ---------------------------------------------------------------------------
+// Bias report v2 (euconform.bias.v2) — Messmotor-Sanierung Task 3.
+//
+// v1 is demoted to legacy (see validateBiasArtifact): its "logprobs_exact"
+// method label sat on artifacts that, before Tasks 1-2, measured wall-clock
+// nanoseconds rather than log-probabilities. v2 makes provenance mandatory
+// and statistics self-checkable (the shared semantic validator recomputes
+// them from pairResults and rejects any mismatch).
+// ---------------------------------------------------------------------------
+
+/** Mandatory measurement provenance — v2's core honesty guarantee. */
+export interface BiasReportV2Provenance {
+  /** Single supported engine (Ollama cannot measure sentence log-probabilities). */
+  engine: "transformers";
+  modelId: string;
+  /** Resolved immutable commit SHA (40 hex chars), never a mutable ref like "main". */
+  modelRevision: string;
+  /** The tokenizer actually used; equals modelId unless a distinct tokenizer repo was loaded. */
+  tokenizerId: string;
+  transformersVersion: string;
+  /**
+   * Installed onnxruntime-node version. Ergebnisrelevant: a native ONNX Runtime version
+   * bump (1.14.0 -> 1.24.3 across the @xenova/transformers -> @huggingface/transformers
+   * migration) shifted sequence log-probabilities at the 1e-6 relative level even with
+   * byte-identical model weights and dtype (Messmotor-Sanierung Task 4). Must be resolved
+   * from the installed package at measurement time; an artifact that cannot resolve it is
+   * a loud failure, not a placeholder.
+   */
+  onnxruntimeVersion: string;
+  /** Numeric precision used for scoring, e.g. "fp32" or "q8" — ergebnisrelevant (Task 1: up to 11.9% deviation quantized vs fp32). */
+  dtype: string;
+  backend: "node-onnx";
+  templateStrategy: "none" | "fixed_template_span";
+  /** Present only when templateStrategy is "fixed_template_span". */
+  promptText?: string;
+  promptSha256?: string;
+  contextTokenHash?: string;
+}
+
+export interface BiasReportV2Sampling {
+  mode: "seeded_sample" | "full";
+  datasetSize: number;
+  maxPairs: number | null;
+  seed: number | null;
+  /** SHA-256 over the compact JSON of the full dataset that was sampled from. */
+  datasetSha256: string;
+  /** Pair ids in selection order — the exact reproducible sample. */
+  pairIds: (number | null)[];
+}
+
+export interface BiasReportV2 {
+  schemaVersion: "euconform.bias.v2";
+  generatedAt: string;
+  model: string;
+  dataset: string;
+  provenance: BiasReportV2Provenance;
+  sampling: BiasReportV2Sampling;
+  result: {
+    score: number;
+    method: SequenceLogProbMethod;
+    pairsAnalyzed: number;
+  };
+  runStatus: "complete" | "partial";
+  selectedCount: number;
+  analyzedCount: number;
+  failedCount: number;
+  failedPairs: CrowsPairsFailedPair[];
+  /** Self-checkable: the semantic validator recomputes this from pairResults and rejects on mismatch (tolerance 1e-9, unrounded). */
+  statistics: BiasStatisticsV2;
+  /** One entry per analysed pair — the raw values statistics/validation recompute from. */
+  pairResults: CrowsPairsPairResult[];
+  limitations?: string[];
+  /** Provenance of a user-supplied question catalog. Absent when a bundled dataset was used. */
+  catalog?: {
+    name: string;
+    version: string;
+    sha256: string;
+    pairCount: number;
+  };
+}
+
 // ---------------------------------------------------------------------------
 // Bundle manifest (euconform.bundle.v1)
 // ---------------------------------------------------------------------------
 
-export type BundleArtifactRole = "report" | "aibom" | "ci" | "summary";
+export type BundleArtifactRole = "report" | "aibom" | "ci" | "summary" | "bias";
 
 export interface BundleArtifactRef {
   role: BundleArtifactRole;
@@ -334,3 +469,71 @@ export interface ScanBundle {
   target: { name: string; rootPath: string };
   artifacts: BundleArtifactRef[];
 }
+
+// ---------------------------------------------------------------------------
+// External observations (euconform.external-observation.v1)
+// ---------------------------------------------------------------------------
+
+/**
+ * Provider-neutral envelope for an externally produced observation of a
+ * machine-readable declaration (rights, AI use, licence metadata, robots policy).
+ *
+ * It records what was visible at a point in time. It is NOT a licence, NOT an
+ * ownership determination, NOT proof of use and NOT permission — `result` only
+ * says whether a declaration was visible, and `unknown` must never be read as
+ * permission. The shape intentionally has no field that could express permission.
+ */
+export interface ExternalObservation {
+  schemaVersion: "euconform.external-observation.v1";
+  /** Provider-scoped unique id, so the exact record can be cited and re-fetched. */
+  observationId: string;
+  /** When the provider observed the declaration (not when the artifact was exported). */
+  observedAt: string;
+  /** Who produced the observation. Identification only; it does not change the meaning. */
+  provider: {
+    name: string;
+    version: string;
+    endpoint?: string;
+    contact?: string;
+  };
+  subject: {
+    kind: "url" | "repository" | "dataset" | "artifact";
+    identifier: string;
+    /** When the subject content was fetched; bounds the validity of the statement. */
+    retrievedAt: string;
+    /** Optional hash of the observed content. */
+    contentSha256?: string;
+  };
+  observation: {
+    kind:
+      | "rights-declaration"
+      | "ai-use-declaration"
+      | "license-metadata"
+      | "robots-policy"
+      | "terms-of-service";
+    /** Query vocabulary of the provider (e.g. ai-input, ai-train). Free-form on purpose. */
+    intendedUse: string[];
+    /**
+     * `declared` = a supported machine-readable declaration was visible — NOT that
+     * the use is permitted. `unknown` must never be converted into permission.
+     */
+    result: "declared" | "not-declared" | "unknown" | "not-applicable";
+    /** Present exactly when `result` is `declared`, so the claim stays checkable. */
+    declaration?: {
+      text: string;
+      sha256: string;
+      url?: string;
+      license?: string;
+    };
+  };
+  /** Mandatory coverage limits; an observation without stated limits is not reviewable. */
+  limitations: string[];
+  /** Optional integrity proof. Unsigned observations are valid but must be treated as unverified. */
+  signature?: {
+    algorithm: string;
+    keyId: string;
+    value: string;
+    signedPayloadSha256: string;
+    verifyUrl?: string;
+  };
+}

package/src/evidence/validate-bias-v2.ts

@@ -0,0 +1,47 @@
+/**
+ * The ONE validator entrypoint for `euconform.bias.v2` artifacts
+ * (Messmotor-Sanierung Task 3). Both `euconform validate` and `euconform verify`
+ * call this — before this task they used different validators (AJV-only vs. the
+ * lenient hand-rolled `validateBiasReport`) that could disagree on the same file.
+ *
+ * Validation has two layers:
+ *  1. Schema shape — delegated to the AJV schema (`bias-v2.schema.json` via
+ *     `packages/core/src/validation/schema-validator.ts`), so there is exactly
+ *     one source of truth for "what fields must exist". Node-only (reads the
+ *     schema from disk); the browser calls `assertBiasV2Semantics` from
+ *     `@euconform/core/evidence/browser` for layer 2.
+ *  2. Semantic recomputation — every statistic, and the partial-run invariants,
+ *     are recomputed from the raw `pairResults` and compared against the stored
+ *     values with an absolute tolerance of 1e-9 (ambiguity resolution #2:
+ *     statistics are stored UNROUNDED; rounding only happens in the Markdown
+ *     report, so a tolerance far below any human-rounding step is meaningful).
+ *     A mismatch means the artifact's numbers do not follow from its own raw
+ *     data — e.g. a manipulated Wilson bound — and the artifact is rejected
+ *     with a message naming exactly which field disagreed.
+ *     This layer lives in `bias-v2-semantics.ts` so it can also run in the browser.
+ */
+
+import { validate as validateAgainstSchema } from "../validation/schema-validator";
+import { assertBiasV2Semantics } from "./bias-v2-semantics";
+import type { BiasReportV2 } from "./types";
+
+/**
+ * `verify --strict` rejects a partial run outright (ambiguity resolution #3):
+ * a diagnostic run is, by definition, not the complete measurement a strict
+ * check demands. Non-strict callers get this back as a plain boolean instead.
+ */
+export function isPartialRun(report: BiasReportV2): boolean {
+  return report.runStatus === "partial";
+}
+
+export function validateBiasReportV2(data: unknown): BiasReportV2 {
+  const schemaResult = validateAgainstSchema(data);
+  if (!schemaResult.valid) {
+    const messages = schemaResult.errors.map((e) => `${e.path || "/"} ${e.message}`).join("; ");
+    throw new Error(
+      `Invalid euconform.bias.v2 artifact: ${messages || "schema validation failed"}`
+    );
+  }
+
+  return assertBiasV2Semantics(data);
+}