@euconform/core 1.4.0 → 1.6.0

package/src/evidence/bias-statistics.ts

@@ -0,0 +1,180 @@
+/**
+ * Pure statistics for the `euconform.bias.v2` evidence format
+ * (Messmotor-Sanierung Task 3).
+ *
+ * These functions are used both by the producer (packages/cli bias command,
+ * via `buildBiasStatistics`) and by the shared semantic validator
+ * (`validate-bias-v2.ts`), which recomputes them from the raw `pairResults`
+ * and compares against the stored `statistics` block. Both sides MUST use the
+ * same `WILSON_Z95` constant — a mismatch would make every honestly-measured
+ * artifact fail its own validation.
+ *
+ * All values here are UNROUNDED. Rounding for human display happens only in
+ * the Markdown report, never in the artifact.
+ */
+
+import type { CrowsPairsPairResult } from "../types";
+
+/** z for a 95% Wilson score interval (two-sided normal quantile at 0.975). */
+export const WILSON_Z95 = 1.959963984540054;
+
+export interface WilsonInterval {
+  lower: number;
+  upper: number;
+}
+
+export interface Quartiles {
+  min: number;
+  max: number;
+  median: number;
+  q1: number;
+  q3: number;
+}
+
+/**
+ * Wilson score interval for a binomial proportion (Wilson, 1927).
+ *
+ *   p̂ = successes / n
+ *   center = (p̂ + z²/(2n)) / (1 + z²/n)
+ *   margin = z · sqrt(p̂(1-p̂)/n + z²/(4n²)) / (1 + z²/n)
+ *
+ * Returns null when n === 0 (the proportion, and therefore the interval, is
+ * undefined — e.g. the equal-token-pairs subset can legitimately be empty).
+ */
+export function wilson95(
+  successes: number,
+  n: number,
+  z: number = WILSON_Z95
+): WilsonInterval | null {
+  if (n === 0) return null;
+  if (successes < 0 || successes > n) {
+    throw new Error(`wilson95: successes (${successes}) must be within [0, n=${n}]`);
+  }
+
+  const pHat = successes / n;
+  const z2 = z * z;
+  const denominator = 1 + z2 / n;
+  const center = (pHat + z2 / (2 * n)) / denominator;
+  const margin = (z * Math.sqrt((pHat * (1 - pHat)) / n + z2 / (4 * n * n))) / denominator;
+
+  return { lower: center - margin, upper: center + margin };
+}
+
+/**
+ * Quantile type 7 (R default; numpy's 'linear' interpolation) — linear
+ * interpolation between order statistics, 0-indexed:
+ *
+ *   h = (n - 1) * p
+ *   value = v[floor(h)] + (h - floor(h)) * (v[floor(h) + 1] - v[floor(h)])
+ *
+ * Returns null for an empty input (undefined distribution).
+ */
+function quantileType7(sorted: number[], p: number): number {
+  const n = sorted.length;
+  if (n === 1) return sorted[0] as number;
+  const h = (n - 1) * p;
+  const lowerIndex = Math.floor(h);
+  const frac = h - lowerIndex;
+  const lower = sorted[lowerIndex] as number;
+  const upper = sorted[Math.min(lowerIndex + 1, n - 1)] as number;
+  return lower + frac * (upper - lower);
+}
+
+export function quartilesType7(values: number[]): Quartiles | null {
+  if (values.length === 0) return null;
+  const sorted = [...values].sort((a, b) => a - b);
+  return {
+    min: sorted[0] as number,
+    max: sorted[sorted.length - 1] as number,
+    q1: quantileType7(sorted, 0.25),
+    median: quantileType7(sorted, 0.5),
+    q3: quantileType7(sorted, 0.75),
+  };
+}
+
+export interface CategoryPreferenceStatistics {
+  biasType: string;
+  analyzedCount: number;
+  preferenceCount: number;
+  tiesCount: number;
+  preferencePercent: number;
+  preferenceWilson95: WilsonInterval | null;
+}
+
+export interface BiasStatisticsV2 {
+  analyzedCount: number;
+  /** Pairs where biasScore > 0 (stereotype preferred). Denominator: analyzedCount. */
+  preferenceCount: number;
+  /** Pairs where biasScore === 0, reported separately, not folded into either side. */
+  tiesCount: number;
+  preferencePercent: number;
+  preferenceWilson95: WilsonInterval | null;
+  byBiasType: CategoryPreferenceStatistics[];
+  /** Distribution of biasScore (stereotype - anti-stereotype) across analysed pairs. */
+  pairDiffQuartiles: Quartiles | null;
+  /** Confounder control: pairs whose stereotype/anti-stereotype token counts are equal. */
+  equalTokenPairsCount: number;
+  equalTokenPreferenceCount: number;
+  equalTokenPreferencePercent: number | null;
+  equalTokenPreferenceWilson95: WilsonInterval | null;
+}
+
+function preferenceStatsOf(pairs: CrowsPairsPairResult[]): {
+  analyzedCount: number;
+  preferenceCount: number;
+  tiesCount: number;
+  preferencePercent: number;
+  preferenceWilson95: WilsonInterval | null;
+} {
+  const analyzedCount = pairs.length;
+  const preferenceCount = pairs.filter((p) => p.biasScore > 0).length;
+  const tiesCount = pairs.filter((p) => p.biasScore === 0).length;
+  const preferencePercent = analyzedCount === 0 ? 0 : (preferenceCount / analyzedCount) * 100;
+  return {
+    analyzedCount,
+    preferenceCount,
+    tiesCount,
+    preferencePercent,
+    preferenceWilson95: wilson95(preferenceCount, analyzedCount),
+  };
+}
+
+function byBiasTypeStatistics(pairResults: CrowsPairsPairResult[]): CategoryPreferenceStatistics[] {
+  const groups = new Map<string, CrowsPairsPairResult[]>();
+  for (const pair of pairResults) {
+    const key = pair.biasType ?? "unspecified";
+    const bucket = groups.get(key);
+    if (bucket) bucket.push(pair);
+    else groups.set(key, [pair]);
+  }
+  return [...groups.entries()].map(([biasType, pairs]) => ({
+    biasType,
+    ...preferenceStatsOf(pairs),
+  }));
+}
+
+/**
+ * Builds the full v2 statistics block from raw per-pair measurements. Pure —
+ * no I/O, no rounding. This is the SAME function the semantic validator calls
+ * to recompute and cross-check a stored artifact's `statistics` block.
+ */
+export function buildBiasStatistics(pairResults: CrowsPairsPairResult[]): BiasStatisticsV2 {
+  const overall = preferenceStatsOf(pairResults);
+  const equalTokenPairs = pairResults.filter((p) => p.tokenCountDelta === 0);
+  const equalTokenStats = preferenceStatsOf(equalTokenPairs);
+
+  return {
+    analyzedCount: overall.analyzedCount,
+    preferenceCount: overall.preferenceCount,
+    tiesCount: overall.tiesCount,
+    preferencePercent: overall.preferencePercent,
+    preferenceWilson95: overall.preferenceWilson95,
+    byBiasType: byBiasTypeStatistics(pairResults),
+    pairDiffQuartiles: quartilesType7(pairResults.map((p) => p.biasScore)),
+    equalTokenPairsCount: equalTokenPairs.length,
+    equalTokenPreferenceCount: equalTokenStats.preferenceCount,
+    equalTokenPreferencePercent:
+      equalTokenPairs.length === 0 ? null : equalTokenStats.preferencePercent,
+    equalTokenPreferenceWilson95: equalTokenStats.preferenceWilson95,
+  };
+}

package/src/evidence/bias-v2-semantics.ts

@@ -0,0 +1,214 @@
+/**
+ * Pure semantic checks for `euconform.bias.v2` artifacts (no Node builtins).
+ *
+ * Split out of `validate-bias-v2.ts` so the exact same recomputation can run in
+ * the browser (`@euconform/core/evidence/browser`). The schema-shape check stays
+ * Node-side because it reads the published JSON schema from disk; the semantic
+ * layer — "do the stored numbers follow from the raw pair results?" — must never
+ * have two implementations.
+ */
+
+import type { CrowsPairsPairResult } from "../types";
+import { type BiasStatisticsV2, type WilsonInterval, buildBiasStatistics } from "./bias-statistics";
+import type { BiasReportV2 } from "./types";
+
+const TOLERANCE = 1e-9;
+
+function closeEnough(a: number, b: number): boolean {
+  return Math.abs(a - b) <= TOLERANCE;
+}
+
+function fail(message: string): never {
+  throw new Error(`Invalid euconform.bias.v2 artifact: ${message}`);
+}
+
+function assertNumberMatches(label: string, expected: number, actual: number): void {
+  if (!closeEnough(expected, actual)) {
+    fail(
+      `${label} does not match recomputation from pairResults: stored ${actual}, recomputed ${expected}`
+    );
+  }
+}
+
+function assertWilsonMatches(
+  label: string,
+  expected: WilsonInterval | null,
+  actual: WilsonInterval | null
+): void {
+  if (expected === null || actual === null) {
+    if (expected !== actual) {
+      fail(
+        `${label} should be ${expected === null ? "null" : "a Wilson interval"}, got ${actual === null ? "null" : JSON.stringify(actual)}`
+      );
+    }
+    return;
+  }
+  assertNumberMatches(`${label}.lower`, expected.lower, actual.lower);
+  assertNumberMatches(`${label}.upper`, expected.upper, actual.upper);
+}
+
+function assertQuartilesMatch(
+  expected: BiasStatisticsV2["pairDiffQuartiles"],
+  actual: BiasStatisticsV2["pairDiffQuartiles"]
+): void {
+  if (expected === null || actual === null) {
+    if (expected !== actual) {
+      fail(
+        `statistics.pairDiffQuartiles should be ${expected === null ? "null" : "present"}, got ${actual === null ? "null" : "present"}`
+      );
+    }
+    return;
+  }
+  for (const key of ["min", "max", "median", "q1", "q3"] as const) {
+    assertNumberMatches(`statistics.pairDiffQuartiles.${key}`, expected[key], actual[key]);
+  }
+}
+
+function assertCategoryStatisticsMatch(
+  expected: BiasStatisticsV2["byBiasType"],
+  actual: BiasStatisticsV2["byBiasType"]
+): void {
+  if (expected.length !== actual.length) {
+    fail(
+      `statistics.byBiasType has ${actual.length} categories, recomputation from pairResults yields ${expected.length}`
+    );
+  }
+  const actualByType = new Map(actual.map((c) => [c.biasType, c]));
+  for (const expectedCategory of expected) {
+    const actualCategory = actualByType.get(expectedCategory.biasType);
+    if (!actualCategory) {
+      fail(`statistics.byBiasType is missing category '${expectedCategory.biasType}'`);
+    }
+    const label = `statistics.byBiasType['${expectedCategory.biasType}']`;
+    if (expectedCategory.analyzedCount !== actualCategory.analyzedCount) {
+      fail(
+        `${label}.analyzedCount: stored ${actualCategory.analyzedCount}, recomputed ${expectedCategory.analyzedCount}`
+      );
+    }
+    if (expectedCategory.preferenceCount !== actualCategory.preferenceCount) {
+      fail(
+        `${label}.preferenceCount: stored ${actualCategory.preferenceCount}, recomputed ${expectedCategory.preferenceCount}`
+      );
+    }
+    if (expectedCategory.tiesCount !== actualCategory.tiesCount) {
+      fail(
+        `${label}.tiesCount: stored ${actualCategory.tiesCount}, recomputed ${expectedCategory.tiesCount}`
+      );
+    }
+    assertNumberMatches(
+      `${label}.preferencePercent`,
+      expectedCategory.preferencePercent,
+      actualCategory.preferencePercent
+    );
+    assertWilsonMatches(
+      `${label}.preferenceWilson95`,
+      expectedCategory.preferenceWilson95,
+      actualCategory.preferenceWilson95
+    );
+  }
+}
+
+function assertStatisticsMatch(expected: BiasStatisticsV2, actual: BiasStatisticsV2): void {
+  if (expected.analyzedCount !== actual.analyzedCount) {
+    fail(
+      `statistics.analyzedCount: stored ${actual.analyzedCount}, recomputed ${expected.analyzedCount}`
+    );
+  }
+  if (expected.preferenceCount !== actual.preferenceCount) {
+    fail(
+      `statistics.preferenceCount: stored ${actual.preferenceCount}, recomputed ${expected.preferenceCount}`
+    );
+  }
+  if (expected.tiesCount !== actual.tiesCount) {
+    fail(`statistics.tiesCount: stored ${actual.tiesCount}, recomputed ${expected.tiesCount}`);
+  }
+  assertNumberMatches(
+    "statistics.preferencePercent",
+    expected.preferencePercent,
+    actual.preferencePercent
+  );
+  assertWilsonMatches(
+    "statistics.preferenceWilson95",
+    expected.preferenceWilson95,
+    actual.preferenceWilson95
+  );
+  assertCategoryStatisticsMatch(expected.byBiasType, actual.byBiasType);
+  assertQuartilesMatch(expected.pairDiffQuartiles, actual.pairDiffQuartiles);
+  if (expected.equalTokenPairsCount !== actual.equalTokenPairsCount) {
+    fail(
+      `statistics.equalTokenPairsCount: stored ${actual.equalTokenPairsCount}, recomputed ${expected.equalTokenPairsCount}`
+    );
+  }
+  if (expected.equalTokenPreferenceCount !== actual.equalTokenPreferenceCount) {
+    fail(
+      `statistics.equalTokenPreferenceCount: stored ${actual.equalTokenPreferenceCount}, recomputed ${expected.equalTokenPreferenceCount}`
+    );
+  }
+  if (
+    expected.equalTokenPreferencePercent === null ||
+    actual.equalTokenPreferencePercent === null
+  ) {
+    if (expected.equalTokenPreferencePercent !== actual.equalTokenPreferencePercent) {
+      fail(
+        "statistics.equalTokenPreferencePercent must be null exactly when equalTokenPairsCount is 0 (recomputation disagrees)"
+      );
+    }
+  } else {
+    assertNumberMatches(
+      "statistics.equalTokenPreferencePercent",
+      expected.equalTokenPreferencePercent,
+      actual.equalTokenPreferencePercent
+    );
+  }
+  assertWilsonMatches(
+    "statistics.equalTokenPreferenceWilson95",
+    expected.equalTokenPreferenceWilson95,
+    actual.equalTokenPreferenceWilson95
+  );
+}
+
+function assertPartialInvariants(report: BiasReportV2): void {
+  if (report.selectedCount !== report.analyzedCount + report.failedCount) {
+    fail(
+      `selectedCount (${report.selectedCount}) must equal analyzedCount (${report.analyzedCount}) + failedCount (${report.failedCount})`
+    );
+  }
+  if (report.failedPairs.length !== report.failedCount) {
+    fail(
+      `failedPairs.length (${report.failedPairs.length}) must equal failedCount (${report.failedCount})`
+    );
+  }
+  if (report.runStatus === "complete" && report.failedCount !== 0) {
+    fail(`runStatus is 'complete' but failedCount is ${report.failedCount}`);
+  }
+  if (report.runStatus === "partial" && report.failedCount === 0) {
+    fail("runStatus is 'partial' but failedCount is 0 (no evidence of a partial run)");
+  }
+  if (report.result.pairsAnalyzed !== report.analyzedCount) {
+    fail(
+      `result.pairsAnalyzed (${report.result.pairsAnalyzed}) must equal analyzedCount (${report.analyzedCount})`
+    );
+  }
+  if (report.pairResults.length !== report.analyzedCount) {
+    fail(
+      `pairResults.length (${report.pairResults.length}) must equal analyzedCount (${report.analyzedCount})`
+    );
+  }
+}
+
+export function recomputedBiasScore(pairResults: CrowsPairsPairResult[]): number {
+  return pairResults.reduce((sum, p) => sum + p.biasScore, 0) / pairResults.length;
+}
+
+/**
+ * Recomputes every stored statistic from `pairResults` and rejects any artifact
+ * whose stored values do not follow from its own raw data (tolerance 1e-9, i.e.
+ * far below any human-rounding step). Throws on mismatch.
+ */
+export function assertBiasV2Semantics(data: unknown): BiasReportV2 {
+  const report = data as BiasReportV2;
+  assertPartialInvariants(report);
+  assertNumberMatches("result.score", recomputedBiasScore(report.pairResults), report.result.score);
+  assertStatisticsMatch(buildBiasStatistics(report.pairResults), report.statistics);
+  return report;
+}

package/src/evidence/browser.ts

@@ -0,0 +1,35 @@
+/**
+ * Browser-safe subset of `@euconform/core/evidence`.
+ *
+ * Everything exported here is pure TypeScript with no Node builtins (`node:fs`,
+ * `node:path`, `node:url`, `node:crypto`), so it can be bundled into the web app.
+ * The Node-only entrypoint (`@euconform/core/evidence`) adds schema loading from
+ * disk, artifact writers and hashing on top of the same pure core — in particular
+ * the semantic check comes from the same `bias-v2-semantics.ts` module, so there
+ * is never a second implementation of "do these numbers follow from the data".
+ */
+
+export type {
+  BiasComparison,
+  BiasComparisonPaired,
+  BiasPairChange,
+  BiasRunSummary,
+  BiasTypeComparison,
+  CompareBiasRunsOptions,
+} from "./bias-comparison";
+export type {
+  BiasStatisticsV2,
+  CategoryPreferenceStatistics,
+  Quartiles,
+  WilsonInterval,
+} from "./bias-statistics";
+export type {
+  BiasReportV2,
+  BiasReportV2Provenance,
+  BiasReportV2Sampling,
+} from "./types";
+
+export { compareBiasRuns, mcnemarExactPValue, summarizeBiasRun } from "./bias-comparison";
+export { formatBiasComparisonMarkdown } from "./bias-comparison-markdown";
+export { buildBiasStatistics, quartilesType7, wilson95, WILSON_Z95 } from "./bias-statistics";
+export { assertBiasV2Semantics, recomputedBiasScore } from "./bias-v2-semantics";

package/src/evidence/bundle.ts

@@ -10,6 +10,7 @@ interface BuildBundleOptions {
   report: ArtifactInput;
   aibom?: ArtifactInput;
   ci?: ArtifactInput;
+  bias?: ArtifactInput;
   summary?: ArtifactInput;
   tool: { name: string; version: string };
   target: { name: string; rootPath: string };
@@ -53,6 +54,7 @@ export function buildBundleManifest(opts: BuildBundleOptions): ScanBundle {
 
   if (opts.aibom) artifacts.push(makeArtifactRef("aibom", opts.aibom, false));
   if (opts.ci) artifacts.push(makeArtifactRef("ci", opts.ci, false));
+  if (opts.bias) artifacts.push(makeArtifactRef("bias", opts.bias, false));
   if (opts.summary) artifacts.push(makeArtifactRef("summary", opts.summary, false));
 
   return {

package/src/evidence/index.ts

@@ -9,6 +9,10 @@ export type {
   AiBillOfMaterials,
   AppType,
   AssessmentHint,
+  BiasReport,
+  BiasReportV2,
+  BiasReportV2Provenance,
+  BiasReportV2Sampling,
   BomComponent,
   BomComponentKind,
   BundleArtifactRef,
@@ -18,6 +22,7 @@ export type {
   ConfidenceLevel,
   DetectedSignal,
   DetectorContext,
+  ExternalObservation,
   FailOnLevel,
   FileProvenance,
   GapCounts,
@@ -36,16 +41,42 @@ export type {
   SignalDetector,
   SignalEvidence,
 } from "./types";
+export type {
+  BiasStatisticsV2,
+  CategoryPreferenceStatistics,
+  Quartiles,
+  WilsonInterval,
+} from "./bias-statistics";
+export type {
+  BiasComparison,
+  BiasComparisonPaired,
+  BiasPairChange,
+  BiasRunSummary,
+  BiasTypeComparison,
+  CompareBiasRunsOptions,
+} from "./bias-comparison";
+export type { BiasValidationResult } from "./validate";
 
 export { generateScanOutput } from "./output";
 export { generateSummaryMarkdown } from "./markdown";
 export { buildBundleManifest } from "./bundle";
 export { sha256Hex } from "./hash";
+export { buildBiasStatistics, quartilesType7, wilson95, WILSON_Z95 } from "./bias-statistics";
+export {
+  compareBiasRuns,
+  mcnemarExactPValue,
+  summarizeBiasRun,
+} from "./bias-comparison";
+export { formatBiasComparisonMarkdown } from "./bias-comparison-markdown";
+export { validateBiasReportV2, isPartialRun } from "./validate-bias-v2";
 export {
   getSchemaMajorVersion,
   validateAiBillOfMaterials,
+  validateBiasArtifact,
+  validateBiasReport,
   validateCiReport,
   validateEcefJsonDocument,
+  validateExternalObservation,
   validateScanBundle,
   validateScanReport,
 } from "./validate";

package/src/evidence/output.ts

@@ -81,6 +81,7 @@ const SOURCE_PRIORITY: Record<BomComponent["source"], number> = {
   config: 4,
   code: 3,
   "lock-file": 2,
+  "sbom-import": 2,
   docs: 1,
 };