@euconform/core 1.5.0 → 1.6.0

package/tests/evidence/bias-statistics.test.ts

@@ -0,0 +1,191 @@
+import { describe, expect, it } from "vitest";
+import {
+  WILSON_Z95,
+  buildBiasStatistics,
+  quartilesType7,
+  wilson95,
+} from "../../src/evidence/bias-statistics";
+import type { CrowsPairsPairResult } from "../../src/types";
+
+/**
+ * Hand-derived anchors (Messmotor-Sanierung Task 3 brief, ambiguity resolution #1:
+ * z = 1.959963984540054 for the 95% Wilson interval, the SAME constant the
+ * producer and the semantic validator must both use — a mismatch between the two
+ * would make every artifact invalid).
+ *
+ * Wilson score interval formula (Wilson 1927), for successes=x, n trials,
+ * p̂ = x/n:
+ *
+ *   center = (p̂ + z²/(2n)) / (1 + z²/n)
+ *   margin = z · sqrt(p̂(1-p̂)/n + z²/(4n²)) / (1 + z²/n)
+ *   [lower, upper] = [center - margin, center + margin]
+ *
+ * wilson95(8, 10): p̂ = 0.8, n = 10, z = 1.959963984540054
+ *   z²        = 3.8414588206941254
+ *   1+z²/n    = 1.3841458820694126
+ *   center    = (0.8 + 3.8414588206941254/20) / 1.3841458820694126
+ *             = (0.8 + 0.19207294103470627) / 1.3841458820694126
+ *             = 0.9920729410347063 / 1.3841458820694126
+ *             = 0.7167401600411333
+ *   margin    = 1.959963984540054 · sqrt(0.8*0.2/10 + 3.8414588206941254/400)
+ *               / 1.3841458820694126
+ *             = 1.959963984540054 · sqrt(0.016 + 0.009603647051735313)
+ *               / 1.3841458820694126
+ *             = 1.959963984540054 · sqrt(0.025603647051735313) / 1.3841458820694126
+ *             = 1.959963984540054 · 0.16001139694798413 / 1.3841458820694126
+ *             = 0.31358835220894676 / 1.3841458820694126
+ *             = 0.22657768850449148
+ *   lower     = 0.7167401600411333 - 0.22657768850449148 = 0.49016247153664183
+ *   upper     = 0.7167401600411333 + 0.22657768850449148 = 0.9433178485456247
+ *
+ * (Verified independently via a standalone Node evaluation of the same formula,
+ * not via this module's implementation.)
+ */
+describe("wilson95", () => {
+  it("matches the hand-derived interval for 8/10", () => {
+    expect(WILSON_Z95).toBeCloseTo(1.959963984540054, 15);
+    const interval = wilson95(8, 10);
+    expect(interval).not.toBeNull();
+    expect(interval?.lower).toBeCloseTo(0.49016247153664183, 12);
+    expect(interval?.upper).toBeCloseTo(0.9433178485456247, 12);
+  });
+
+  it("returns null for n === 0 (undefined/empty subset, e.g. no equal-token pairs)", () => {
+    expect(wilson95(0, 0)).toBeNull();
+  });
+
+  it("is symmetric around 0.5 for a 1/2 split", () => {
+    const interval = wilson95(1, 2);
+    expect(interval).not.toBeNull();
+    const center = ((interval?.lower ?? 0) + (interval?.upper ?? 0)) / 2;
+    expect(center).toBeCloseTo(0.5, 10);
+  });
+
+  it("rejects successes > n", () => {
+    expect(() => wilson95(11, 10)).toThrow();
+  });
+});
+
+/**
+ * quartilesType7([1, 2, 3, 4]): quantile type 7 (R default / numpy 'linear'),
+ * 0-indexed: h = (n-1)*p, value = v[floor(h)] + frac(h) * (v[floor(h)+1] - v[floor(h)]).
+ * n = 4, sorted values = [1, 2, 3, 4].
+ *
+ *   q1 (p=0.25): h = 3*0.25 = 0.75 -> v[0] + 0.75*(v[1]-v[0]) = 1 + 0.75*1 = 1.75
+ *   median (p=0.5): h = 3*0.5 = 1.5 -> v[1] + 0.5*(v[2]-v[1]) = 2 + 0.5*1 = 2.5
+ *   q3 (p=0.75): h = 3*0.75 = 2.25 -> v[2] + 0.25*(v[3]-v[2]) = 3 + 0.25*1 = 3.25
+ *   min = 1, max = 4
+ */
+describe("quartilesType7", () => {
+  it("matches the hand-derived quartiles for [1, 2, 3, 4]", () => {
+    const q = quartilesType7([1, 2, 3, 4]);
+    expect(q).not.toBeNull();
+    expect(q?.min).toBe(1);
+    expect(q?.max).toBe(4);
+    expect(q?.q1).toBeCloseTo(1.75, 12);
+    expect(q?.median).toBeCloseTo(2.5, 12);
+    expect(q?.q3).toBeCloseTo(3.25, 12);
+  });
+
+  it("does not mutate its input and is order-independent", () => {
+    const values = [4, 1, 3, 2];
+    const snapshot = [...values];
+    const q = quartilesType7(values);
+    expect(values).toEqual(snapshot);
+    expect(q?.median).toBeCloseTo(2.5, 12);
+  });
+
+  it("returns min=max=median=q1=q3 for a single value", () => {
+    const q = quartilesType7([5]);
+    expect(q).toEqual({ min: 5, max: 5, median: 5, q1: 5, q3: 5 });
+  });
+
+  it("returns null for an empty array", () => {
+    expect(quartilesType7([])).toBeNull();
+  });
+});
+
+function pair(overrides: Partial<CrowsPairsPairResult>): CrowsPairsPairResult {
+  return {
+    pairId: null,
+    stereotype: "s",
+    antiStereotype: "a",
+    biasType: null,
+    stereotypeLogProb: 0,
+    antiStereotypeLogProb: 0,
+    biasScore: 0,
+    biasDirection: "neutral",
+    method: "sequence_logprob_sum",
+    stereotypeTokenCount: 3,
+    antiStereotypeTokenCount: 3,
+    tokenCountDelta: 0,
+    stereotypeTokenHash: "x",
+    antiStereotypeTokenHash: "y",
+    ...overrides,
+  };
+}
+
+describe("buildBiasStatistics", () => {
+  it("counts preference/ties from bias>0 and bias===0, with the matching Wilson interval", () => {
+    const pairResults = [
+      pair({ biasScore: 0.3, biasDirection: "stereotype" }),
+      pair({ biasScore: 0.1, biasDirection: "stereotype" }),
+      pair({ biasScore: -0.2, biasDirection: "anti-stereotype" }),
+      pair({ biasScore: 0, biasDirection: "neutral" }),
+    ];
+    const stats = buildBiasStatistics(pairResults);
+
+    expect(stats.analyzedCount).toBe(4);
+    expect(stats.preferenceCount).toBe(2); // bias > 0
+    expect(stats.tiesCount).toBe(1); // bias === 0
+    expect(stats.preferencePercent).toBeCloseTo((2 / 4) * 100, 12);
+    // preferencePercent is the UNROUNDED raw value (rounding only in Markdown).
+    const expected = wilson95(2, 4);
+    expect(stats.preferenceWilson95?.lower).toBeCloseTo(expected?.lower ?? Number.NaN, 12);
+    expect(stats.preferenceWilson95?.upper).toBeCloseTo(expected?.upper ?? Number.NaN, 12);
+  });
+
+  it("computes pairDiffQuartiles from the raw biasScore distribution", () => {
+    const pairResults = [1, 2, 3, 4].map((biasScore) => pair({ biasScore }));
+    const stats = buildBiasStatistics(pairResults);
+    expect(stats.pairDiffQuartiles).toEqual({ min: 1, max: 4, median: 2.5, q1: 1.75, q3: 3.25 });
+  });
+
+  it("groups byBiasType with its own counts/percent/Wilson interval", () => {
+    const pairResults = [
+      pair({ biasType: "gender", biasScore: 0.2 }),
+      pair({ biasType: "gender", biasScore: -0.1 }),
+      pair({ biasType: "age", biasScore: 0.5 }),
+    ];
+    const stats = buildBiasStatistics(pairResults);
+    const gender = stats.byBiasType.find((b) => b.biasType === "gender");
+    expect(gender?.analyzedCount).toBe(2);
+    expect(gender?.preferenceCount).toBe(1);
+    expect(gender?.preferencePercent).toBeCloseTo(50, 12);
+  });
+
+  it("returns null equalToken stats when no pairs share token length (empty subset)", () => {
+    const pairResults = [
+      pair({ tokenCountDelta: 1 }),
+      pair({ tokenCountDelta: -2 }),
+      pair({ tokenCountDelta: 3 }),
+    ];
+    const stats = buildBiasStatistics(pairResults);
+    expect(stats.equalTokenPairsCount).toBe(0);
+    expect(stats.equalTokenPreferencePercent).toBeNull();
+    expect(stats.equalTokenPreferenceWilson95).toBeNull();
+  });
+
+  it("computes equalToken preference stats only over the tokenCountDelta===0 subset", () => {
+    const pairResults = [
+      pair({ tokenCountDelta: 0, biasScore: 0.4 }), // equal-length, preferred
+      pair({ tokenCountDelta: 0, biasScore: -0.1 }), // equal-length, not preferred
+      pair({ tokenCountDelta: 2, biasScore: 0.9 }), // unequal-length, excluded
+    ];
+    const stats = buildBiasStatistics(pairResults);
+    expect(stats.equalTokenPairsCount).toBe(2);
+    expect(stats.equalTokenPreferenceCount).toBe(1);
+    expect(stats.equalTokenPreferencePercent).toBeCloseTo(50, 12);
+    expect(stats.equalTokenPreferenceWilson95).not.toBeNull();
+  });
+});

package/tests/evidence/external-observation.test.ts

@@ -0,0 +1,167 @@
+import { describe, expect, it } from "vitest";
+import {
+  type ExternalObservation,
+  validateEcefJsonDocument,
+  validateExternalObservation,
+} from "../../src/evidence";
+import { validate } from "../../src/validation";
+
+const SHA = "a".repeat(64);
+
+function observation(overrides: Record<string, unknown> = {}): ExternalObservation {
+  return {
+    schemaVersion: "euconform.external-observation.v1",
+    observationId: "obs_01J9Z6K3Q8",
+    observedAt: "2026-10-29T12:00:00Z",
+    provider: {
+      name: "example-rights-observer",
+      version: "1.0.0",
+      endpoint: "https://api.example.com/v1/rights/preflight",
+    },
+    subject: {
+      kind: "url",
+      identifier: "https://example.com/dataset",
+      retrievedAt: "2026-10-29T11:59:41Z",
+      contentSha256: SHA,
+    },
+    observation: {
+      kind: "rights-declaration",
+      intendedUse: ["ai-input", "ai-train"],
+      result: "declared",
+      declaration: {
+        text: "AI training permitted for non-commercial research only.",
+        sha256: SHA,
+        url: "https://example.com/dataset/terms",
+        license: "CC-BY-NC-4.0",
+      },
+    },
+    limitations: ["Coverage limited to reviewed origins."],
+    ...overrides,
+  };
+}
+
+function schemaValid(document: unknown): boolean {
+  return validate(document).valid;
+}
+
+describe("euconform.external-observation.v1 schema", () => {
+  it("accepts a declared observation with a signature", () => {
+    const withSignature = observation({
+      signature: {
+        algorithm: "ed25519",
+        keyId: "provider-2026-01",
+        value: "base64-signature",
+        signedPayloadSha256: SHA,
+      },
+    });
+
+    expect(schemaValid(withSignature)).toBe(true);
+    expect(validate(withSignature).schemaType).toBe("external-observation.v1");
+  });
+
+  it("accepts 'unknown' without a declaration", () => {
+    const unknown = observation({
+      observation: { kind: "rights-declaration", intendedUse: ["ai-train"], result: "unknown" },
+    });
+
+    expect(schemaValid(unknown)).toBe(true);
+  });
+
+  it("requires the declaration when the result is 'declared'", () => {
+    const withoutDeclaration = observation({
+      observation: { kind: "rights-declaration", intendedUse: ["ai-train"], result: "declared" },
+    });
+
+    const result = validate(withoutDeclaration);
+    expect(result.valid).toBe(false);
+    expect(result.errors.some((e) => e.message.includes("declaration"))).toBe(true);
+  });
+
+  it("rejects permission-shaped results", () => {
+    const permitted = observation({
+      observation: { kind: "rights-declaration", intendedUse: ["ai-train"], result: "permitted" },
+    });
+
+    expect(schemaValid(permitted)).toBe(false);
+  });
+
+  it("rejects a permission field even though the rest is valid", () => {
+    const withPermission = { ...observation(), permission: true };
+
+    const result = validate(withPermission);
+    expect(result.valid).toBe(false);
+    expect(result.errors.some((e) => e.message.includes("additional properties"))).toBe(true);
+  });
+
+  it("requires at least one stated limitation", () => {
+    expect(schemaValid(observation({ limitations: [] }))).toBe(false);
+    expect(schemaValid(observation({ limitations: undefined }))).toBe(false);
+  });
+
+  it("rejects unknown schema versions", () => {
+    const result = validate({
+      ...observation(),
+      schemaVersion: "euconform.external-observation.v2",
+    });
+    expect(result.valid).toBe(false);
+    expect(result.errors[0]?.message).toMatch(/Unknown schemaVersion/);
+  });
+
+  it("requires the integrity fields of a present signature", () => {
+    const partialSignature = observation({
+      signature: { algorithm: "ed25519", keyId: "k", value: "v" },
+    });
+
+    expect(schemaValid(partialSignature)).toBe(false);
+  });
+});
+
+describe("validateExternalObservation", () => {
+  it("accepts a valid observation and returns it unchanged", () => {
+    const document = observation();
+    expect(validateExternalObservation(document)).toEqual(document);
+  });
+
+  it("rejects a missing provider name", () => {
+    const broken = observation({ provider: { version: "1.0.0" } });
+    expect(() => validateExternalObservation(broken)).toThrow(/provider\.name/);
+  });
+
+  it("rejects a missing subject retrieval timestamp", () => {
+    const broken = observation({
+      subject: { kind: "url", identifier: "https://example.com" },
+    });
+    expect(() => validateExternalObservation(broken)).toThrow(/subject\.retrievedAt/);
+  });
+
+  it("rejects a declared result without a checkable declaration", () => {
+    const broken = observation({
+      observation: { kind: "rights-declaration", intendedUse: ["ai-train"], result: "declared" },
+    });
+    expect(() => validateExternalObservation(broken)).toThrow(/observation\.declaration/);
+  });
+
+  it("rejects a declaration without its hash", () => {
+    const broken = observation({
+      observation: {
+        kind: "rights-declaration",
+        intendedUse: ["ai-train"],
+        result: "declared",
+        declaration: { text: "permitted" },
+      },
+    });
+    expect(() => validateExternalObservation(broken)).toThrow(/declaration\.sha256/);
+  });
+
+  it("explains why an observation without limitations is rejected", () => {
+    const broken = observation({ limitations: [] });
+    expect(() => validateExternalObservation(broken)).toThrow(/not reviewable/);
+  });
+
+  it("is reachable through the generic document dispatcher", () => {
+    const document = observation();
+    expect(validateEcefJsonDocument("euconform.external-observation.v1", document)).toEqual(
+      document
+    );
+  });
+});

package/tests/evidence/spec-schemas.test.ts

@@ -175,8 +175,9 @@ describe("EuConform Evidence Format schemas", () => {
   ]) as Array<[keyof typeof SCHEMA_FILES, JsonSchema]>;
 
   it("validates every documented example artifact set", () => {
+    // "external-observation" holds a standalone document, not a scan artifact set.
     const exampleDirs = readdirSync(EXAMPLES_DIR, { withFileTypes: true })
-      .filter((entry) => entry.isDirectory())
+      .filter((entry) => entry.isDirectory() && entry.name !== "external-observation")
       .map((entry) => entry.name)
       .sort();
 
@@ -232,6 +233,38 @@ describe("EuConform Evidence Format schemas", () => {
     expect(errors.some((e) => e.includes("unknownField"))).toBe(true);
   });
 
+  it("validates the published external observation example", () => {
+    const schema = readJson<JsonSchema>(
+      resolve(SCHEMAS_DIR, "external-observation-v1.schema.json")
+    );
+    const example = readJson<Record<string, unknown>>(
+      resolve(EXAMPLES_DIR, "external-observation/observation.json")
+    );
+
+    expect(validateAgainstSchema(schema, schema, example)).toEqual([]);
+    expect(example.schemaVersion).toBe("euconform.external-observation.v1");
+  });
+
+  it("rejects an external observation that could be read as permission", () => {
+    const schema = readJson<JsonSchema>(
+      resolve(SCHEMAS_DIR, "external-observation-v1.schema.json")
+    );
+    const example = readJson<Record<string, unknown>>(
+      resolve(EXAMPLES_DIR, "external-observation/observation.json")
+    );
+
+    const withPermission = { ...example, permission: true };
+    const extraFieldErrors = validateAgainstSchema(schema, schema, withPermission);
+    expect(extraFieldErrors.some((error) => error.includes("permission"))).toBe(true);
+
+    const permittedResult = {
+      ...example,
+      observation: { ...(example.observation as Record<string, unknown>), result: "permitted" },
+    };
+    const badResultErrors = validateAgainstSchema(schema, schema, permittedResult);
+    expect(badResultErrors.some((error) => error.includes("result"))).toBe(true);
+  });
+
   it("rejects invalid enum values", () => {
     const ciSchema = schemaEntries.find(([kind]) => kind === "ci")?.[1];
     if (!ciSchema) throw new Error("ci schema not found");
@@ -255,7 +288,7 @@ describe("EuConform Evidence Format schemas", () => {
     } as const;
 
     const exampleDirs = readdirSync(EXAMPLES_DIR, { withFileTypes: true })
-      .filter((entry) => entry.isDirectory())
+      .filter((entry) => entry.isDirectory() && entry.name !== "external-observation")
       .map((entry) => entry.name);
 
     for (const exampleDir of exampleDirs) {

package/tests/evidence/validate-bias-v2.test.ts

@@ -0,0 +1,180 @@
+import { describe, expect, it } from "vitest";
+import { buildBiasStatistics } from "../../src/evidence/bias-statistics";
+import { validateBiasReportV2 } from "../../src/evidence/validate-bias-v2";
+import type { CrowsPairsPairResult } from "../../src/types";
+
+function pair(overrides: Partial<CrowsPairsPairResult> = {}): CrowsPairsPairResult {
+  return {
+    pairId: 1,
+    stereotype: "Der Mann ist Ingenieur.",
+    antiStereotype: "Die Frau ist Ingenieurin.",
+    biasType: "gender",
+    stereotypeLogProb: -1.2,
+    antiStereotypeLogProb: -1.8,
+    biasScore: 0.6,
+    biasDirection: "stereotype",
+    method: "sequence_logprob_sum",
+    stereotypeTokenCount: 5,
+    antiStereotypeTokenCount: 5,
+    tokenCountDelta: 0,
+    stereotypeTokenHash: "a".repeat(64),
+    antiStereotypeTokenHash: "b".repeat(64),
+    ...overrides,
+  };
+}
+
+/** A minimal, internally-consistent euconform.bias.v2 artifact built from real pairResults. */
+function buildValidArtifact(pairResults: CrowsPairsPairResult[]) {
+  const statistics = buildBiasStatistics(pairResults);
+  const score = pairResults.reduce((sum, p) => sum + p.biasScore, 0) / pairResults.length;
+  return {
+    schemaVersion: "euconform.bias.v2" as const,
+    generatedAt: "2026-08-07T10:00:00.000Z",
+    model: "Xenova/gpt2",
+    dataset: "crows_pairs_de",
+    provenance: {
+      engine: "transformers",
+      modelId: "Xenova/gpt2",
+      modelRevision: "e".repeat(40),
+      tokenizerId: "Xenova/gpt2",
+      transformersVersion: "2.17.2",
+      onnxruntimeVersion: "1.14.0",
+      dtype: "fp32",
+      backend: "node-onnx",
+      templateStrategy: "none",
+    },
+    sampling: {
+      mode: "seeded_sample",
+      datasetSize: 100,
+      maxPairs: pairResults.length,
+      seed: 42,
+      datasetSha256: "c".repeat(64),
+      pairIds: pairResults.map((p) => p.pairId),
+    },
+    result: {
+      score,
+      method: "sequence_logprob_sum",
+      pairsAnalyzed: pairResults.length,
+    },
+    runStatus: "complete" as const,
+    selectedCount: pairResults.length,
+    analyzedCount: pairResults.length,
+    failedCount: 0,
+    failedPairs: [],
+    statistics,
+    pairResults,
+  };
+}
+
+describe("validateBiasReportV2", () => {
+  it("accepts a well-formed, internally-consistent artifact", () => {
+    const artifact = buildValidArtifact([
+      pair({ pairId: 1, biasScore: 0.6, biasDirection: "stereotype" }),
+      pair({ pairId: 2, biasScore: -0.2, biasDirection: "anti-stereotype" }),
+      pair({ pairId: 3, biasScore: 0.4, biasDirection: "stereotype", biasType: "age" }),
+    ]);
+    expect(() => validateBiasReportV2(artifact)).not.toThrow();
+  });
+
+  it("rejects an artifact missing required top-level fields (schema shape)", () => {
+    const artifact = buildValidArtifact([pair()]);
+    // biome-ignore lint/performance/noDelete: test needs a genuinely missing key, not undefined
+    delete (artifact as Record<string, unknown>).provenance;
+    expect(() => validateBiasReportV2(artifact)).toThrow(/provenance/);
+  });
+
+  it("rejects a manipulated Wilson bound (statistics don't follow from pairResults)", () => {
+    const artifact = buildValidArtifact([
+      pair({ pairId: 1, biasScore: 0.6 }),
+      pair({ pairId: 2, biasScore: -0.2 }),
+    ]);
+    const tampered = {
+      ...artifact,
+      statistics: {
+        ...artifact.statistics,
+        preferenceWilson95: artifact.statistics.preferenceWilson95
+          ? {
+              lower: artifact.statistics.preferenceWilson95.lower + 0.01,
+              upper: artifact.statistics.preferenceWilson95.upper,
+            }
+          : null,
+      },
+    };
+    expect(() => validateBiasReportV2(tampered)).toThrow(/preferenceWilson95/);
+  });
+
+  it("rejects a tampered pairDiffQuartiles median", () => {
+    const artifact = buildValidArtifact([
+      pair({ pairId: 1, biasScore: 1 }),
+      pair({ pairId: 2, biasScore: 2 }),
+      pair({ pairId: 3, biasScore: 3 }),
+      pair({ pairId: 4, biasScore: 4 }),
+    ]);
+    const tampered = {
+      ...artifact,
+      statistics: {
+        ...artifact.statistics,
+        pairDiffQuartiles: artifact.statistics.pairDiffQuartiles
+          ? { ...artifact.statistics.pairDiffQuartiles, median: 999 }
+          : null,
+      },
+    };
+    expect(() => validateBiasReportV2(tampered)).toThrow(/pairDiffQuartiles/);
+  });
+
+  it("rejects a wrong tiesCount", () => {
+    const artifact = buildValidArtifact([
+      pair({ pairId: 1, biasScore: 0 }),
+      pair({ pairId: 2, biasScore: 0.5 }),
+    ]);
+    const tampered = {
+      ...artifact,
+      statistics: { ...artifact.statistics, tiesCount: 0 },
+    };
+    expect(() => validateBiasReportV2(tampered)).toThrow(/tiesCount/);
+  });
+
+  it("rejects selectedCount/analyzedCount/failedCount invariant violations", () => {
+    const artifact = buildValidArtifact([pair({ pairId: 1 })]);
+    const tampered = { ...artifact, selectedCount: 5 };
+    expect(() => validateBiasReportV2(tampered)).toThrow(/selectedCount/);
+  });
+
+  it("rejects failedPairs.length not matching failedCount", () => {
+    const artifact = buildValidArtifact([pair({ pairId: 1 })]);
+    const tampered = { ...artifact, failedCount: 1, failedPairs: [] };
+    expect(() => validateBiasReportV2(tampered)).toThrow(/failedPairs|selectedCount/);
+  });
+
+  it("accepts a partial run with a matching failedPairs entry", () => {
+    const artifact = buildValidArtifact([pair({ pairId: 1 }), pair({ pairId: 2 })]);
+    const partial = {
+      ...artifact,
+      runStatus: "partial" as const,
+      selectedCount: 3,
+      failedCount: 1,
+      failedPairs: [{ pairId: 3, reason: "measurement failed" }],
+    };
+    expect(() => validateBiasReportV2(partial)).not.toThrow();
+  });
+
+  it("accepts the null equal-token-subset case (no pairs share token length)", () => {
+    const artifact = buildValidArtifact([
+      pair({ pairId: 1, tokenCountDelta: 1, stereotypeTokenCount: 6, antiStereotypeTokenCount: 5 }),
+      pair({
+        pairId: 2,
+        tokenCountDelta: -2,
+        stereotypeTokenCount: 3,
+        antiStereotypeTokenCount: 5,
+      }),
+    ]);
+    expect(artifact.statistics.equalTokenPreferencePercent).toBeNull();
+    expect(() => validateBiasReportV2(artifact)).not.toThrow();
+  });
+
+  it("rejects a mismatched top-level result.score", () => {
+    const artifact = buildValidArtifact([pair({ pairId: 1, biasScore: 0.6 })]);
+    const tampered = { ...artifact, result: { ...artifact.result, score: 99 } };
+    expect(() => validateBiasReportV2(tampered)).toThrow(/result\.score/);
+  });
+});

package/tests/fairness-metrics.test.ts

@@ -4,10 +4,44 @@ import {
   equalizedOdds,
   generateLoanDataset,
   predictiveParity,
+  runComprehensiveBiasAnalysis,
+  runCrowsPairsBiasAnalysis,
   runFairnessAnalysis,
   statisticalParityDifference,
 } from "../src/fairness-metrics";
-import type { PredictionData } from "../src/types";
+import type { SequenceLogProbProvider } from "../src/inference";
+import type { ScoredLogProb } from "../src/inference/transformers-inference";
+import type { BiasTestConfig, PredictionData } from "../src/types";
+
+/** A provider whose scoring always throws — used to prove failures propagate. */
+const alwaysFailingProvider: SequenceLogProbProvider = {
+  getScoredLogProb: async (): Promise<ScoredLogProb> => {
+    throw new Error("measurement unavailable");
+  },
+};
+
+const alwaysSucceedingProvider: SequenceLogProbProvider = {
+  getScoredLogProb: async (text: string): Promise<ScoredLogProb> => ({
+    score: text.includes("stereo") ? -1 : -2,
+    scoredTokenIds: [],
+    span: { start: 0, end: 0 },
+    contextTokenHash: "test-hash",
+    provenance: {
+      modelId: "test-model",
+      logitsDtype: "float32",
+      backend: "test",
+      libraryVersion: "test",
+    },
+  }),
+};
+
+const config: BiasTestConfig = {
+  dataset: [
+    { id: 1, sent_more: "stereo sentence", sent_less: "anti sentence", bias_type: "gender" },
+  ],
+  model: "test-model",
+  engine: "browser",
+};
 
 describe("Fairness Metrics", () => {
   describe("disparateImpact", () => {
@@ -184,4 +218,46 @@ describe("Fairness Metrics", () => {
       expect(groups.size).toBeGreaterThan(1);
     });
   });
+
+  /**
+   * These entry points used to catch ANY calculateCrowsPairsBias failure —
+   * including the core's intentional fail-closed abort — and return a
+   * fabricated BiasTestResult (overallPassed: false, samplesAnalyzed: 0,
+   * method: "logprobs_fallback_latency") instead of propagating it. That is
+   * exactly the silent-fallback pattern this remediation removes: a failed
+   * measurement must never look like a plausible result object.
+   */
+  describe("runCrowsPairsBiasAnalysis", () => {
+    it("returns a real result when measurement succeeds", async () => {
+      const result = await runCrowsPairsBiasAnalysis(config, alwaysSucceedingProvider);
+      expect(result.crowsPairsResult).toBeDefined();
+      expect(result.samplesAnalyzed).toBe(1);
+    });
+
+    it("propagates a measurement failure instead of fabricating a result", async () => {
+      await expect(runCrowsPairsBiasAnalysis(config, alwaysFailingProvider)).rejects.toThrow();
+    });
+  });
+
+  describe("runComprehensiveBiasAnalysis", () => {
+    it("returns a real result when measurement succeeds", async () => {
+      const result = await runComprehensiveBiasAnalysis(
+        [],
+        config,
+        alwaysSucceedingProvider,
+        "gender"
+      );
+      expect(result.crowsPairsResult).toBeDefined();
+    });
+
+    it("propagates a measurement failure instead of fabricating or falling back to traditional-only metrics", async () => {
+      const predictionData: PredictionData[] = [
+        { prediction: 1, protectedGroup: "A" },
+        { prediction: 0, protectedGroup: "B" },
+      ];
+      await expect(
+        runComprehensiveBiasAnalysis(predictionData, config, alwaysFailingProvider, "gender")
+      ).rejects.toThrow();
+    });
+  });
 });