@euconform/core 1.4.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/CHANGELOG.md +46 -0
- package/dist/datasets/crows-pairs.d.ts +18 -0
- package/dist/datasets/crows-pairs.d.ts.map +1 -1
- package/dist/datasets/crows-pairs.js +58 -0
- package/dist/evidence/bias-comparison-markdown.d.ts +11 -0
- package/dist/evidence/bias-comparison-markdown.d.ts.map +1 -0
- package/dist/evidence/bias-comparison-markdown.js +120 -0
- package/dist/evidence/bias-comparison.d.ts +133 -0
- package/dist/evidence/bias-comparison.d.ts.map +1 -0
- package/dist/evidence/bias-comparison.js +274 -0
- package/dist/evidence/bias-statistics.d.ts +72 -0
- package/dist/evidence/bias-statistics.d.ts.map +1 -0
- package/dist/evidence/bias-statistics.js +122 -0
- package/dist/evidence/bias-v2-semantics.d.ts +19 -0
- package/dist/evidence/bias-v2-semantics.d.ts.map +1 -0
- package/dist/evidence/bias-v2-semantics.js +133 -0
- package/dist/evidence/browser.d.ts +18 -0
- package/dist/evidence/browser.d.ts.map +1 -0
- package/dist/evidence/browser.js +14 -0
- package/dist/evidence/bundle.d.ts +1 -0
- package/dist/evidence/bundle.d.ts.map +1 -1
- package/dist/evidence/bundle.js +2 -0
- package/dist/evidence/index.d.ts +9 -2
- package/dist/evidence/index.d.ts.map +1 -1
- package/dist/evidence/index.js +5 -1
- package/dist/evidence/output.d.ts.map +1 -1
- package/dist/evidence/output.js +1 -0
- package/dist/evidence/types.d.ts +183 -3
- package/dist/evidence/types.d.ts.map +1 -1
- package/dist/evidence/validate-bias-v2.d.ts +31 -0
- package/dist/evidence/validate-bias-v2.d.ts.map +1 -0
- package/dist/evidence/validate-bias-v2.js +40 -0
- package/dist/evidence/validate.d.ts +29 -2
- package/dist/evidence/validate.d.ts.map +1 -1
- package/dist/evidence/validate.js +96 -11
- package/dist/fairness-metrics.d.ts +15 -11
- package/dist/fairness-metrics.d.ts.map +1 -1
- package/dist/fairness-metrics.js +59 -183
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/inference/browser-inference.d.ts +6 -57
- package/dist/inference/browser-inference.d.ts.map +1 -1
- package/dist/inference/browser-inference.js +6 -219
- package/dist/inference/capability-detection.d.ts +9 -2
- package/dist/inference/capability-detection.d.ts.map +1 -1
- package/dist/inference/capability-detection.js +18 -16
- package/dist/inference/index.d.ts +15 -2
- package/dist/inference/index.d.ts.map +1 -1
- package/dist/inference/index.js +12 -3
- package/dist/inference/inference-factory.d.ts +6 -3
- package/dist/inference/inference-factory.d.ts.map +1 -1
- package/dist/inference/inference-factory.js +7 -26
- package/dist/inference/ollama-client.d.ts +29 -0
- package/dist/inference/ollama-client.d.ts.map +1 -1
- package/dist/inference/ollama-client.js +29 -0
- package/dist/inference/scored-log-prob-types.d.ts +64 -0
- package/dist/inference/scored-log-prob-types.d.ts.map +1 -0
- package/dist/inference/scored-log-prob-types.js +12 -0
- package/dist/inference/sequence-logprob.d.ts +34 -0
- package/dist/inference/sequence-logprob.d.ts.map +1 -0
- package/dist/inference/sequence-logprob.js +87 -0
- package/dist/inference/template-scoring.d.ts +124 -0
- package/dist/inference/template-scoring.d.ts.map +1 -0
- package/dist/inference/template-scoring.js +164 -0
- package/dist/inference/transformers-inference.d.ts +124 -0
- package/dist/inference/transformers-inference.d.ts.map +1 -0
- package/dist/inference/transformers-inference.js +392 -0
- package/dist/legal-checks/bias-metrics.d.ts +5 -7
- package/dist/legal-checks/bias-metrics.d.ts.map +1 -1
- package/dist/legal-checks/bias-metrics.js +8 -101
- package/dist/metrics/crows-pairs-bias.d.ts +17 -9
- package/dist/metrics/crows-pairs-bias.d.ts.map +1 -1
- package/dist/metrics/crows-pairs-bias.js +147 -59
- package/dist/metrics/stereotype-bias.d.ts +13 -5
- package/dist/metrics/stereotype-bias.d.ts.map +1 -1
- package/dist/metrics/stereotype-bias.js +16 -99
- package/dist/sbom/component-mapper.d.ts +26 -0
- package/dist/sbom/component-mapper.d.ts.map +1 -0
- package/dist/sbom/component-mapper.js +83 -0
- package/dist/sbom/cyclonedx-import.d.ts +34 -0
- package/dist/sbom/cyclonedx-import.d.ts.map +1 -0
- package/dist/sbom/cyclonedx-import.js +131 -0
- package/dist/sbom/index.d.ts +6 -0
- package/dist/sbom/index.d.ts.map +1 -0
- package/dist/sbom/index.js +4 -0
- package/dist/sbom/known-packages.d.ts +13 -0
- package/dist/sbom/known-packages.d.ts.map +1 -0
- package/dist/sbom/known-packages.js +129 -0
- package/dist/sbom/purl.d.ts +14 -0
- package/dist/sbom/purl.d.ts.map +1 -0
- package/dist/sbom/purl.js +56 -0
- package/dist/sbom/types.d.ts +39 -0
- package/dist/sbom/types.d.ts.map +1 -0
- package/dist/sbom/types.js +5 -0
- package/dist/scanner/detectors/ai-framework-detector.js +1 -1
- package/dist/scanner/detectors/local-inference-detector.d.ts.map +1 -1
- package/dist/scanner/detectors/local-inference-detector.js +1 -1
- package/dist/scanner/detectors/package-json-detector.d.ts.map +1 -1
- package/dist/scanner/detectors/package-json-detector.js +1 -0
- package/dist/types.d.ts +103 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/validation/index.d.ts +2 -0
- package/dist/validation/index.d.ts.map +1 -0
- package/dist/validation/index.js +1 -0
- package/dist/validation/schema-validator.d.ts +18 -0
- package/dist/validation/schema-validator.d.ts.map +1 -0
- package/dist/validation/schema-validator.js +105 -0
- package/package.json +9 -3
- package/scripts/seed-logprob-fixture.mjs +179 -0
- package/scripts/ts-extensionless-resolve-hook.mjs +26 -0
- package/src/datasets/crows-pairs.ts +74 -0
- package/src/evidence/bias-comparison-markdown.ts +162 -0
- package/src/evidence/bias-comparison.ts +432 -0
- package/src/evidence/bias-statistics.ts +180 -0
- package/src/evidence/bias-v2-semantics.ts +214 -0
- package/src/evidence/browser.ts +35 -0
- package/src/evidence/bundle.ts +2 -0
- package/src/evidence/index.ts +31 -0
- package/src/evidence/output.ts +1 -0
- package/src/evidence/types.ts +213 -3
- package/src/evidence/validate-bias-v2.ts +47 -0
- package/src/evidence/validate.ts +151 -17
- package/src/fairness-metrics.ts +79 -222
- package/src/index.ts +7 -0
- package/src/inference/browser-inference.ts +14 -256
- package/src/inference/capability-detection.ts +32 -18
- package/src/inference/index.ts +33 -5
- package/src/inference/inference-factory.ts +9 -30
- package/src/inference/ollama-client.ts +29 -0
- package/src/inference/scored-log-prob-types.ts +66 -0
- package/src/inference/sequence-logprob.ts +136 -0
- package/src/inference/template-scoring.ts +248 -0
- package/src/inference/transformers-inference.ts +494 -0
- package/src/legal-checks/bias-metrics.ts +20 -118
- package/src/metrics/crows-pairs-bias.ts +180 -83
- package/src/metrics/stereotype-bias.ts +27 -119
- package/src/sbom/component-mapper.ts +107 -0
- package/src/sbom/cyclonedx-import.ts +198 -0
- package/src/sbom/index.ts +12 -0
- package/src/sbom/known-packages.ts +142 -0
- package/src/sbom/purl.ts +66 -0
- package/src/sbom/types.ts +34 -0
- package/src/scanner/detectors/ai-framework-detector.ts +1 -1
- package/src/scanner/detectors/local-inference-detector.ts +2 -1
- package/src/scanner/detectors/package-json-detector.ts +1 -0
- package/src/types.ts +108 -2
- package/src/validation/index.ts +7 -0
- package/src/validation/schema-validator.ts +147 -0
- package/tests/annex-iv-report.test.ts +1 -1
- package/tests/capability-detection.test.ts +57 -0
- package/tests/crows-pairs-bias.test.ts +286 -186
- package/tests/custom-dataset.test.ts +80 -0
- package/tests/evidence/bias-artifact-v2.test.ts +135 -0
- package/tests/evidence/bias-artifact.test.ts +78 -0
- package/tests/evidence/bias-comparison.test.ts +397 -0
- package/tests/evidence/bias-statistics.test.ts +191 -0
- package/tests/evidence/external-observation.test.ts +167 -0
- package/tests/evidence/spec-schemas.test.ts +35 -2
- package/tests/evidence/validate-bias-v2.test.ts +180 -0
- package/tests/fairness-metrics.test.ts +77 -1
- package/tests/fixtures/cyclonedx-minimal.json +13 -0
- package/tests/fixtures/cyclonedx-mixed.json +17 -0
- package/tests/fixtures/cyclonedx-ml-bom.json +25 -0
- package/tests/inference/browser-inference-scoring.test.ts +159 -0
- package/tests/inference/fixtures/logprob-golden.json +69 -0
- package/tests/inference/gemma-template.integration.test.ts +81 -0
- package/tests/inference/logprob-golden.integration.test.ts +128 -0
- package/tests/inference/synthetic-logits.test.ts +188 -0
- package/tests/inference/template-scoring.test.ts +249 -0
- package/tests/inference/transformers-inference-template-mode.test.ts +214 -0
- package/tests/integration/bias-calculation.integration.test.ts +119 -599
- package/tests/sbom/component-mapper.test.ts +258 -0
- package/tests/sbom/cyclonedx-import.test.ts +329 -0
- package/tests/sbom/purl.test.ts +102 -0
- package/tests/stilled-ollama-paths.test.ts +76 -0
- package/tests/validation/schema-validator.test.ts +219 -0
- package/src/inference/transformers.d.ts +0 -62
package/.turbo/turbo-build.log
CHANGED
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,51 @@
|
|
|
1
1
|
# @euconform/core
|
|
2
2
|
|
|
3
|
+
## 1.6.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 27048f0: Browser-safe evidence entrypoint and bias evidence viewer.
|
|
8
|
+
|
|
9
|
+
- New `@euconform/core/evidence/browser` export with the pure parts of the evidence layer (statistics, run comparison, Markdown rendering, semantic checks) — no Node builtins, so it can be bundled for the web app.
|
|
10
|
+
- The `euconform.bias.v2` semantic validation now lives in its own module and is shared by the Node validator and the browser, so "do the stored numbers follow from `pairResults`?" still has exactly one implementation.
|
|
11
|
+
- The web app gained a Bias Evidence page (`/bias`) that imports one or two `euconform.bias.json` artifacts, validates them locally, and shows the report or a full run comparison (deltas, per-bias-type shifts, McNemar exact test). Bias measurement itself stays in the CLI.
|
|
12
|
+
|
|
13
|
+
- 987c603: Compare two bias runs. The new `euconform compare <baseline> <candidate>` command diffs two `euconform.bias.v2` artifacts: mean-score and preference-quote deltas (with both Wilson intervals), per-bias-type shifts, the largest per-pair changes, and a McNemar exact test over preference flips so a shift is not mistaken for noise.
|
|
14
|
+
|
|
15
|
+
- Accepts artifact files or the directories a bias run wrote them to.
|
|
16
|
+
- Refuses comparisons that do not answer the same question with the same measurement setup (catalogue, scoring method, template strategy); different models/dtypes/library versions are listed as environment differences instead. `--allow-incomparable` writes the report explicitly marked `comparable: false`.
|
|
17
|
+
- Emits Markdown by default, JSON via `--output json`; both via `--output all --out-dir`.
|
|
18
|
+
|
|
19
|
+
- 2431a0d: New evidence document type `euconform.external-observation.v1`.
|
|
20
|
+
|
|
21
|
+
A provider-neutral envelope for third-party, time-stamped observations of machine-readable declarations (rights, AI use, licence metadata, robots policy). It fills the gap between structural evidence (`scan`) and behavioural evidence (`bias`): external observations come from another party, so they get their own document instead of being mixed into a EuConform artifact.
|
|
22
|
+
|
|
23
|
+
The envelope records **what was visible, when** — `provider`, `subject` (with retrieval time and optional content hash), `observation.kind`, the provider's query vocabulary, and one of four results: `declared`, `not-declared`, `unknown`, `not-applicable`. It deliberately has no field that could express permission, a `declared` result must carry the observed declaration text plus its SHA-256, and at least one stated limitation is mandatory. `unknown` can never be read as permission.
|
|
24
|
+
|
|
25
|
+
- `euconform validate` now accepts the new document type (schema-only: there are no numbers to recompute).
|
|
26
|
+
- New guide `docs/spec/external-observation-v1.md`, schema `external-observation-v1.schema.json`, and a published example.
|
|
27
|
+
- `validateExternalObservation()` and the `ExternalObservation` type are exported from `@euconform/core/evidence`; `validateEcefJsonDocument` dispatches the new version.
|
|
28
|
+
- External observations stay standalone: they are not bundle roles in `euconform.bundle.v1`.
|
|
29
|
+
|
|
30
|
+
- 6308581: Reproducible bias evidence. Bias measurements now produce the self-checkable `euconform.bias.v2` format: every run records catalog provenance, per-pair results, real scoring provenance (engine, weight dtype, output dtype, pinned model revision) and statistics that can be re-verified from the artifact itself.
|
|
31
|
+
|
|
32
|
+
- `euconform bias` accepts `--dataset` for user-supplied question catalogs and `--require-exact` to abort instead of falling back when the exact measurement method is unavailable.
|
|
33
|
+
- `--engine` is now an explicit contract; the Ollama path and `scan --bias` are disabled because they could not produce real log-probability scores.
|
|
34
|
+
- Migrated local inference to `@huggingface/transformers` 4.2.0 and added template-mode sentence log-probability scoring keyed on the tokenizer chat template, not the architecture name.
|
|
35
|
+
- Failed measurements are reported as unverified capability entries instead of fabricated results.
|
|
36
|
+
|
|
37
|
+
## 1.5.0
|
|
38
|
+
|
|
39
|
+
### Minor Changes
|
|
40
|
+
|
|
41
|
+
- 0f6c07d: Add schema validation, CycloneDX SBOM import, and AI BOM v1.1 schema revision
|
|
42
|
+
|
|
43
|
+
- Introduce `euconform.aibom.v1.1` schema with optional import provenance metadata
|
|
44
|
+
- Add `validate` command for checking EuConform JSON documents against published schemas
|
|
45
|
+
- Add `import` command for mapping CycloneDX JSON SBOMs into the EuConform AI BOM layer
|
|
46
|
+
- Add PURL parser with graceful handling of malformed percent-encoding
|
|
47
|
+
- Remove `validate` and `importCycloneDx` from core barrel export (available via `@euconform/core/validation` and `@euconform/core/sbom` subpath exports)
|
|
48
|
+
|
|
3
49
|
## 1.4.0
|
|
4
50
|
|
|
5
51
|
### Minor Changes
|
|
@@ -4,4 +4,22 @@ import type { CrowsPairsEntry } from "../types";
|
|
|
4
4
|
* Converts the structured JSON format to CrowsPairsEntry[] for the bias engine.
|
|
5
5
|
*/
|
|
6
6
|
export declare function loadCrowsPairsDataset(language: "en" | "de"): CrowsPairsEntry[];
|
|
7
|
+
export interface CustomDatasetMetadata {
|
|
8
|
+
name: string;
|
|
9
|
+
version: string;
|
|
10
|
+
sha256: string;
|
|
11
|
+
pairCount: number;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Load a user-supplied question catalog.
|
|
15
|
+
*
|
|
16
|
+
* Deliberately separate from loadCrowsPairsDataset: that one resolves a bundled
|
|
17
|
+
* asset by language code, this one takes a path the caller chose. The hash goes
|
|
18
|
+
* into the evidence artifact so "Fragekatalog + Version" is verifiable rather
|
|
19
|
+
* than merely stated.
|
|
20
|
+
*/
|
|
21
|
+
export declare function loadCustomDataset(filePath: string): {
|
|
22
|
+
pairs: CrowsPairsEntry[];
|
|
23
|
+
metadata: CustomDatasetMetadata;
|
|
24
|
+
};
|
|
7
25
|
//# sourceMappingURL=crows-pairs.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crows-pairs.d.ts","sourceRoot":"","sources":["../../src/datasets/crows-pairs.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"crows-pairs.d.ts","sourceRoot":"","sources":["../../src/datasets/crows-pairs.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AA6ChD;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,IAAI,GAAG,IAAI,GAAG,eAAe,EAAE,CAW9E;AAED,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG;IACnD,KAAK,EAAE,eAAe,EAAE,CAAC;IACzB,QAAQ,EAAE,qBAAqB,CAAC;CACjC,CAqDA"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
1
2
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
3
|
import { dirname, resolve } from "node:path";
|
|
3
4
|
import { fileURLToPath } from "node:url";
|
|
@@ -39,3 +40,60 @@ export function loadCrowsPairsDataset(language) {
|
|
|
39
40
|
bias_type: pair.biasType,
|
|
40
41
|
}));
|
|
41
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* Load a user-supplied question catalog.
|
|
45
|
+
*
|
|
46
|
+
* Deliberately separate from loadCrowsPairsDataset: that one resolves a bundled
|
|
47
|
+
* asset by language code, this one takes a path the caller chose. The hash goes
|
|
48
|
+
* into the evidence artifact so "Fragekatalog + Version" is verifiable rather
|
|
49
|
+
* than merely stated.
|
|
50
|
+
*/
|
|
51
|
+
export function loadCustomDataset(filePath) {
|
|
52
|
+
const resolved = resolve(filePath);
|
|
53
|
+
if (!existsSync(resolved)) {
|
|
54
|
+
throw new Error(`Fragekatalog nicht gefunden: ${resolved}`);
|
|
55
|
+
}
|
|
56
|
+
const raw = readFileSync(resolved, "utf-8");
|
|
57
|
+
let parsed;
|
|
58
|
+
try {
|
|
59
|
+
parsed = JSON.parse(raw);
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
throw new Error(`Fragekatalog ist kein gültiges JSON: ${resolved}`);
|
|
63
|
+
}
|
|
64
|
+
const doc = parsed;
|
|
65
|
+
if (!Array.isArray(doc.pairs) || doc.pairs.length === 0) {
|
|
66
|
+
throw new Error(`Fragekatalog enthält keine Paare: ${resolved}`);
|
|
67
|
+
}
|
|
68
|
+
const pairs = doc.pairs.map((entry, index) => {
|
|
69
|
+
const p = entry;
|
|
70
|
+
const stereotype = typeof p.stereotype === "string" ? p.stereotype : undefined;
|
|
71
|
+
const antiStereotype = typeof p.antiStereotype === "string" ? p.antiStereotype : undefined;
|
|
72
|
+
if (!stereotype || !antiStereotype) {
|
|
73
|
+
throw new Error(`Fragekatalog: Paar ${p.id ?? index} braucht 'stereotype' und 'antiStereotype' (${resolved})`);
|
|
74
|
+
}
|
|
75
|
+
return {
|
|
76
|
+
id: typeof p.id === "number" ? p.id : index + 1,
|
|
77
|
+
sent_more: stereotype,
|
|
78
|
+
sent_less: antiStereotype,
|
|
79
|
+
bias_type: typeof p.biasType === "string" ? p.biasType : "unspecified",
|
|
80
|
+
};
|
|
81
|
+
});
|
|
82
|
+
const meta = doc.metadata ?? {};
|
|
83
|
+
return {
|
|
84
|
+
pairs,
|
|
85
|
+
metadata: {
|
|
86
|
+
name: typeof meta.name === "string" ? meta.name : "unbenannter Katalog",
|
|
87
|
+
version: typeof meta.version === "string" ? meta.version : "unversioniert",
|
|
88
|
+
sha256: createHash("sha256").update(raw).digest("hex"),
|
|
89
|
+
// Deliberately NOT the resolved absolute filesystem path: euconform.bias.v2's
|
|
90
|
+
// `catalog` schema (additionalProperties: false) has no `path` field on purpose — an
|
|
91
|
+
// absolute, machine-specific path baked into an evidence artifact would defeat the
|
|
92
|
+
// talk's "you can recompute this" promise the moment the repo is cloned to a
|
|
93
|
+
// different machine (Messmotor-Sanierung Task 6 finding: `validate` rejected a real
|
|
94
|
+
// clinical-catalog run over exactly this field). The catalog's identity is already
|
|
95
|
+
// fully portable via name + version + sha256.
|
|
96
|
+
pairCount: pairs.length,
|
|
97
|
+
},
|
|
98
|
+
};
|
|
99
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Markdown rendering for `compareBiasRuns` output.
|
|
3
|
+
*
|
|
4
|
+
* Kept separate from the comparison logic so the numbers stay pure and testable;
|
|
5
|
+
* this module only formats. Mirrors the tone of the existing bias report: no
|
|
6
|
+
* severity labels, no pass/fail verdict — just the self-checkable numbers and
|
|
7
|
+
* the significance of the observed shift.
|
|
8
|
+
*/
|
|
9
|
+
import type { BiasComparison } from "./bias-comparison";
|
|
10
|
+
export declare function formatBiasComparisonMarkdown(comparison: BiasComparison): string;
|
|
11
|
+
//# sourceMappingURL=bias-comparison-markdown.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bias-comparison-markdown.d.ts","sourceRoot":"","sources":["../../src/evidence/bias-comparison-markdown.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAkB,MAAM,mBAAmB,CAAC;AAsIxE,wBAAgB,4BAA4B,CAAC,UAAU,EAAE,cAAc,GAAG,MAAM,CAkB/E"}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Markdown rendering for `compareBiasRuns` output.
|
|
3
|
+
*
|
|
4
|
+
* Kept separate from the comparison logic so the numbers stay pure and testable;
|
|
5
|
+
* this module only formats. Mirrors the tone of the existing bias report: no
|
|
6
|
+
* severity labels, no pass/fail verdict — just the self-checkable numbers and
|
|
7
|
+
* the significance of the observed shift.
|
|
8
|
+
*/
|
|
9
|
+
function percent(value) {
|
|
10
|
+
return `${value.toFixed(1)}%`;
|
|
11
|
+
}
|
|
12
|
+
function signed(value, digits) {
|
|
13
|
+
return `${value >= 0 ? "+" : ""}${value.toFixed(digits)}`;
|
|
14
|
+
}
|
|
15
|
+
function points(value) {
|
|
16
|
+
return `${value >= 0 ? "+" : ""}${value.toFixed(1)} pp`;
|
|
17
|
+
}
|
|
18
|
+
function wilson(interval) {
|
|
19
|
+
if (!interval)
|
|
20
|
+
return "n/a";
|
|
21
|
+
return `[${percent(interval.lower * 100)}, ${percent(interval.upper * 100)}]`;
|
|
22
|
+
}
|
|
23
|
+
function pValue(value) {
|
|
24
|
+
return value < 0.0001 ? value.toExponential(2) : value.toFixed(4);
|
|
25
|
+
}
|
|
26
|
+
function runHeading(label, run) {
|
|
27
|
+
return `**${label}:** ${run.model} — ${run.dataset} (${run.generatedAt})`;
|
|
28
|
+
}
|
|
29
|
+
function summarySection(comparison) {
|
|
30
|
+
const { baseline, candidate, deltas } = comparison;
|
|
31
|
+
const rows = [
|
|
32
|
+
`| Model | ${baseline.model} | ${candidate.model} | — |`,
|
|
33
|
+
`| Mean bias score | ${baseline.meanBiasScore.toFixed(4)} | ${candidate.meanBiasScore.toFixed(4)} | ${signed(deltas.meanBiasScore, 4)} |`,
|
|
34
|
+
`| Preference quote (bias > 0) | ${percent(baseline.preferencePercent)} (95% Wilson ${wilson(baseline.preferenceWilson95)}) | ${percent(candidate.preferencePercent)} (95% Wilson ${wilson(candidate.preferenceWilson95)}) | ${points(deltas.preferencePercentPoints)} |`,
|
|
35
|
+
`| Analyzed pairs | ${baseline.analyzedCount} | ${candidate.analyzedCount} | ${candidate.analyzedCount - baseline.analyzedCount} |`,
|
|
36
|
+
`| Ties (bias = 0) | ${baseline.tiesCount} | ${candidate.tiesCount} | ${candidate.tiesCount - baseline.tiesCount} |`,
|
|
37
|
+
`| Equal-token subset | ${baseline.equalTokenPairsCount} pairs | ${candidate.equalTokenPairsCount} pairs | ${candidate.equalTokenPairsCount - baseline.equalTokenPairsCount} |`,
|
|
38
|
+
`| Run status | ${baseline.runStatus} | ${candidate.runStatus} | — |`,
|
|
39
|
+
];
|
|
40
|
+
return [
|
|
41
|
+
"## Run Summary",
|
|
42
|
+
"",
|
|
43
|
+
"| Metric | Baseline | Candidate | Delta |",
|
|
44
|
+
"|--------|----------|-----------|-------|",
|
|
45
|
+
...rows,
|
|
46
|
+
"",
|
|
47
|
+
];
|
|
48
|
+
}
|
|
49
|
+
function pairedSection(comparison) {
|
|
50
|
+
const { paired } = comparison;
|
|
51
|
+
const lines = [
|
|
52
|
+
"## Paired Comparison",
|
|
53
|
+
"",
|
|
54
|
+
`- Pairs measured in both runs: ${paired.pairsCompared} (paired by ${paired.keyStrategy === "pairId" ? "pair id" : "sentence text"})`,
|
|
55
|
+
`- Only in baseline: ${paired.onlyInBaseline}, only in candidate: ${paired.onlyInCandidate}`,
|
|
56
|
+
`- Preference flips: baseline only ${paired.baselinePreferredOnly}, candidate only ${paired.candidatePreferredOnly}`,
|
|
57
|
+
];
|
|
58
|
+
if (!paired.mcnemarExact) {
|
|
59
|
+
lines.push("- McNemar exact test: n/a (no discordant pairs)");
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
const { discordantPairs, pValue: p, alpha, significant } = paired.mcnemarExact;
|
|
63
|
+
lines.push(`- McNemar exact test (two-sided, ${discordantPairs} discordant pairs): p = ${pValue(p)} (alpha ${alpha}) → ${significant ? "significant shift" : "no significant shift"}`);
|
|
64
|
+
}
|
|
65
|
+
return [...lines, ""];
|
|
66
|
+
}
|
|
67
|
+
function biasTypeSection(comparison) {
|
|
68
|
+
if (comparison.byBiasType.length === 0)
|
|
69
|
+
return [];
|
|
70
|
+
return [
|
|
71
|
+
"## Preference By Bias Type",
|
|
72
|
+
"",
|
|
73
|
+
"| Bias type | Baseline | Candidate | Delta | Pairs (baseline / candidate) |",
|
|
74
|
+
"|-----------|----------|-----------|-------|------------------------------|",
|
|
75
|
+
...comparison.byBiasType.map((row) => `| ${row.biasType} | ${percent(row.baselinePreferencePercent)} | ${percent(row.candidatePreferencePercent)} | ${points(row.deltaPoints)} | ${row.baselineAnalyzedCount} / ${row.candidateAnalyzedCount} |`),
|
|
76
|
+
"",
|
|
77
|
+
];
|
|
78
|
+
}
|
|
79
|
+
function changesSection(comparison) {
|
|
80
|
+
if (comparison.largestChanges.length === 0) {
|
|
81
|
+
return ["## Per-Pair Changes", "", "No pairs changed between the two runs.", ""];
|
|
82
|
+
}
|
|
83
|
+
return [
|
|
84
|
+
"## Largest Per-Pair Changes",
|
|
85
|
+
"",
|
|
86
|
+
"| Pair | Bias type | Baseline | Candidate | Delta |",
|
|
87
|
+
"|------|-----------|----------|-----------|-------|",
|
|
88
|
+
...comparison.largestChanges.map((change) => `| ${change.pairId ?? "—"} | ${change.biasType ?? "unspecified"} | ${change.baselineBiasScore.toFixed(4)} | ${change.candidateBiasScore.toFixed(4)} | ${signed(change.delta, 4)} |`),
|
|
89
|
+
"",
|
|
90
|
+
];
|
|
91
|
+
}
|
|
92
|
+
function comparabilitySection(comparison) {
|
|
93
|
+
const lines = [`**Comparable:** ${comparison.comparable ? "yes" : "no"}`, ""];
|
|
94
|
+
if (!comparison.comparable) {
|
|
95
|
+
lines.push("The two runs did not answer the same question with the same measurement setup. The deltas below are not a measurement of model behaviour and MUST NOT be read as one:", "", ...comparison.comparabilityIssues.map((issue) => `- ${issue}`), "");
|
|
96
|
+
}
|
|
97
|
+
if (comparison.environmentDifferences.length > 0) {
|
|
98
|
+
lines.push("## Environment Differences", "", "Expected for model-to-model comparisons; listed so the difference stays visible:", "", ...comparison.environmentDifferences.map((difference) => `- ${difference}`), "");
|
|
99
|
+
}
|
|
100
|
+
return lines;
|
|
101
|
+
}
|
|
102
|
+
export function formatBiasComparisonMarkdown(comparison) {
|
|
103
|
+
return [
|
|
104
|
+
"# EuConform Bias Comparison",
|
|
105
|
+
"",
|
|
106
|
+
runHeading("Baseline", comparison.baseline),
|
|
107
|
+
"",
|
|
108
|
+
runHeading("Candidate", comparison.candidate),
|
|
109
|
+
"",
|
|
110
|
+
...comparabilitySection(comparison),
|
|
111
|
+
...summarySection(comparison),
|
|
112
|
+
...pairedSection(comparison),
|
|
113
|
+
...biasTypeSection(comparison),
|
|
114
|
+
...changesSection(comparison),
|
|
115
|
+
"---",
|
|
116
|
+
"",
|
|
117
|
+
"*Generated by [EuConform CLI](https://github.com/Hiepler/EuConform). Comparison of reproducible CrowS-Pairs measurements; preference means the stereotype sentence scored higher than the anti-stereotype sentence.*",
|
|
118
|
+
"",
|
|
119
|
+
].join("\n");
|
|
120
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Comparison of two `euconform.bias.v2` artifacts (bias compare).
|
|
3
|
+
*
|
|
4
|
+
* Two bias runs are only comparable when they asked the same questions with the
|
|
5
|
+
* same measurement setup. This module therefore separates two things:
|
|
6
|
+
*
|
|
7
|
+
* - `comparabilityIssues`: setup differences that change what the numbers MEAN
|
|
8
|
+
* (different catalog, different scoring method, different template/prompt).
|
|
9
|
+
* Any entry here makes the comparison invalid.
|
|
10
|
+
* - `environmentDifferences`: differences that are expected and often the point
|
|
11
|
+
* of the comparison (different model, dtype, library versions) or that only
|
|
12
|
+
* affect precision/coverage (sampling, partial runs). They are reported but
|
|
13
|
+
* do not invalidate the comparison.
|
|
14
|
+
*
|
|
15
|
+
* Pure: no I/O, no rounding. All values stay unrounded so they can be re-checked.
|
|
16
|
+
*/
|
|
17
|
+
import type { SequenceLogProbMethod } from "../types";
|
|
18
|
+
import type { BiasStatisticsV2, WilsonInterval } from "./bias-statistics";
|
|
19
|
+
import type { BiasReportV2 } from "./types";
|
|
20
|
+
export interface BiasRunSummary {
|
|
21
|
+
/** Path the artifact was loaded from, when known. */
|
|
22
|
+
path?: string;
|
|
23
|
+
model: string;
|
|
24
|
+
dataset: string;
|
|
25
|
+
generatedAt: string;
|
|
26
|
+
method: SequenceLogProbMethod;
|
|
27
|
+
templateStrategy: "none" | "fixed_template_span";
|
|
28
|
+
runStatus: "complete" | "partial";
|
|
29
|
+
selectedCount: number;
|
|
30
|
+
analyzedCount: number;
|
|
31
|
+
failedCount: number;
|
|
32
|
+
/** Mean bias score as stored in `result.score` (validated against pairResults). */
|
|
33
|
+
meanBiasScore: number;
|
|
34
|
+
preferenceCount: number;
|
|
35
|
+
tiesCount: number;
|
|
36
|
+
preferencePercent: number;
|
|
37
|
+
preferenceWilson95: WilsonInterval | null;
|
|
38
|
+
byBiasType: BiasStatisticsV2["byBiasType"];
|
|
39
|
+
/** Confounder control: equal-token-length subset, carried through for honesty. */
|
|
40
|
+
equalTokenPairsCount: number;
|
|
41
|
+
equalTokenPreferencePercent: number | null;
|
|
42
|
+
datasetSha256: string;
|
|
43
|
+
catalog: BiasReportV2["catalog"] | null;
|
|
44
|
+
sampling: BiasReportV2["sampling"];
|
|
45
|
+
environment: {
|
|
46
|
+
modelRevision: string;
|
|
47
|
+
tokenizerId: string;
|
|
48
|
+
dtype: string;
|
|
49
|
+
transformersVersion: string;
|
|
50
|
+
onnxruntimeVersion: string;
|
|
51
|
+
backend: string;
|
|
52
|
+
promptSha256: string | null;
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
export interface BiasTypeComparison {
|
|
56
|
+
biasType: string;
|
|
57
|
+
baselineAnalyzedCount: number;
|
|
58
|
+
candidateAnalyzedCount: number;
|
|
59
|
+
baselinePreferencePercent: number;
|
|
60
|
+
candidatePreferencePercent: number;
|
|
61
|
+
/** candidate - baseline, in percentage points. */
|
|
62
|
+
deltaPoints: number;
|
|
63
|
+
}
|
|
64
|
+
export interface BiasPairChange {
|
|
65
|
+
pairId: number | null;
|
|
66
|
+
biasType: string | null;
|
|
67
|
+
stereotype: string;
|
|
68
|
+
antiStereotype: string;
|
|
69
|
+
baselineBiasScore: number;
|
|
70
|
+
candidateBiasScore: number;
|
|
71
|
+
/** candidate - baseline. Negative means the candidate became less biased. */
|
|
72
|
+
delta: number;
|
|
73
|
+
}
|
|
74
|
+
export interface BiasComparisonPaired {
|
|
75
|
+
/**
|
|
76
|
+
* How pairs of the two runs were matched. `pairId` is used only when BOTH runs
|
|
77
|
+
* carry non-null ids (a custom catalog without ids falls back to sentence text).
|
|
78
|
+
*/
|
|
79
|
+
keyStrategy: "pairId" | "sentenceText";
|
|
80
|
+
pairsCompared: number;
|
|
81
|
+
onlyInBaseline: number;
|
|
82
|
+
onlyInCandidate: number;
|
|
83
|
+
/** Pairs preferred in the baseline run but not in the candidate run. */
|
|
84
|
+
baselinePreferredOnly: number;
|
|
85
|
+
/** Pairs preferred in the candidate run but not in the baseline run. */
|
|
86
|
+
candidatePreferredOnly: number;
|
|
87
|
+
/**
|
|
88
|
+
* McNemar exact test (binomial, two-sided, p=0.5) over the discordant pairs:
|
|
89
|
+
* a test of whether the preference rate shifted between the two runs.
|
|
90
|
+
* "Preferred" means biasScore > 0; ties count as not preferred.
|
|
91
|
+
*/
|
|
92
|
+
mcnemarExact: {
|
|
93
|
+
discordantPairs: number;
|
|
94
|
+
pValue: number;
|
|
95
|
+
alpha: number;
|
|
96
|
+
significant: boolean;
|
|
97
|
+
} | null;
|
|
98
|
+
}
|
|
99
|
+
export interface BiasComparison {
|
|
100
|
+
reportType: "bias-comparison";
|
|
101
|
+
generatedAt: string;
|
|
102
|
+
alpha: number;
|
|
103
|
+
baseline: BiasRunSummary;
|
|
104
|
+
candidate: BiasRunSummary;
|
|
105
|
+
comparable: boolean;
|
|
106
|
+
comparabilityIssues: string[];
|
|
107
|
+
environmentDifferences: string[];
|
|
108
|
+
deltas: {
|
|
109
|
+
meanBiasScore: number;
|
|
110
|
+
/** candidate - baseline, in percentage points. */
|
|
111
|
+
preferencePercentPoints: number;
|
|
112
|
+
};
|
|
113
|
+
paired: BiasComparisonPaired;
|
|
114
|
+
byBiasType: BiasTypeComparison[];
|
|
115
|
+
largestChanges: BiasPairChange[];
|
|
116
|
+
}
|
|
117
|
+
export interface CompareBiasRunsOptions {
|
|
118
|
+
baselinePath?: string;
|
|
119
|
+
candidatePath?: string;
|
|
120
|
+
alpha?: number;
|
|
121
|
+
/** Maximum number of per-pair changes in `largestChanges` (default 10). */
|
|
122
|
+
topChanges?: number;
|
|
123
|
+
/** Override for `generatedAt` (tests). */
|
|
124
|
+
generatedAt?: string;
|
|
125
|
+
}
|
|
126
|
+
export declare function summarizeBiasRun(report: BiasReportV2, path?: string): BiasRunSummary;
|
|
127
|
+
/**
|
|
128
|
+
* Two-sided exact McNemar test: P = 2 * P(X <= min(b, c)) for
|
|
129
|
+
* X ~ Binomial(b + c, 0.5). Computed in log space so large runs do not underflow.
|
|
130
|
+
*/
|
|
131
|
+
export declare function mcnemarExactPValue(baselineOnly: number, candidateOnly: number): number;
|
|
132
|
+
export declare function compareBiasRuns(baselineReport: BiasReportV2, candidateReport: BiasReportV2, options?: CompareBiasRunsOptions): BiasComparison;
|
|
133
|
+
//# sourceMappingURL=bias-comparison.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bias-comparison.d.ts","sourceRoot":"","sources":["../../src/evidence/bias-comparison.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAwB,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAC5E,OAAO,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAC1E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAE5C,MAAM,WAAW,cAAc;IAC7B,qDAAqD;IACrD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,qBAAqB,CAAC;IAC9B,gBAAgB,EAAE,MAAM,GAAG,qBAAqB,CAAC;IACjD,SAAS,EAAE,UAAU,GAAG,SAAS,CAAC;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,mFAAmF;IACnF,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kBAAkB,EAAE,cAAc,GAAG,IAAI,CAAC;IAC1C,UAAU,EAAE,gBAAgB,CAAC,YAAY,CAAC,CAAC;IAC3C,kFAAkF;IAClF,oBAAoB,EAAE,MAAM,CAAC;IAC7B,2BAA2B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,YAAY,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;IACxC,QAAQ,EAAE,YAAY,CAAC,UAAU,CAAC,CAAC;IACnC,WAAW,EAAE;QACX,aAAa,EAAE,MAAM,CAAC;QACtB,WAAW,EAAE,MAAM,CAAC;QACpB,KAAK,EAAE,MAAM,CAAC;QACd,mBAAmB,EAAE,MAAM,CAAC;QAC5B,kBAAkB,EAAE,MAAM,CAAC;QAC3B,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;KAC7B,CAAC;CACH;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,yBAAyB,EAAE,MAAM,CAAC;IAClC,0BAA0B,EAAE,MAAM,CAAC;IACnC,kDAAkD;IAClD,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,6EAA6E;IAC7E,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,oBAAoB;IACnC;;;OAGG;IACH,WAAW,EAAE,QAAQ,GAAG,cAAc,CAAC;IACvC,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,wEAAwE;IACxE,qBAAqB,EAAE,MAAM,CAAC;IAC9B,wEAAwE;IACxE,sBAAsB,EAAE,MAAM,CAAC;IAC/B;;;;OAIG;IACH,YAAY,EAAE;QACZ,eAAe,EAAE,MAAM,CAAC;QACxB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,OAAO,CAAC;KACtB,GAAG,IAAI,CAAC;CACV;AAED,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,iBAAiB,CAAC;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,cAAc,CAAC;IACzB,SAAS,EAAE,cAAc,CAAC;IAC1B,UAAU,EAAE,OAAO,CAAC;IACpB,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,sBAAsB,EAAE,MAAM,EAAE,CAAC;IACjC,MAAM,EAAE;QACN,aAAa,EAAE,MAAM,CAAC;QACtB,kDAAkD;QAClD,uBAAuB,EAAE,MAAM,CAAC;KACjC,CAAC;IACF,MAAM,EAAE,oBAAoB,CAAC;IAC7B,UAAU,EAAE,kBAAkB,EAAE,CAAC;IACjC,cAAc,EAAE,cAAc,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,sBAAsB;IACrC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2EAA2E;IAC3E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,0CAA0C;IAC1C,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,cAAc,CAiCpF;AAyHD;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,MAAM,CAetF;AAuCD,wBAAgB,eAAe,CAC7B,cAAc,EAAE,YAAY,EAC5B,eAAe,EAAE,YAAY,EAC7B,OAAO,GAAE,sBAA2B,GACnC,cAAc,CAkFhB"}
|