@euconform/core 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/CHANGELOG.md +34 -0
- package/dist/datasets/crows-pairs.d.ts +18 -0
- package/dist/datasets/crows-pairs.d.ts.map +1 -1
- package/dist/datasets/crows-pairs.js +58 -0
- package/dist/evidence/bias-comparison-markdown.d.ts +11 -0
- package/dist/evidence/bias-comparison-markdown.d.ts.map +1 -0
- package/dist/evidence/bias-comparison-markdown.js +120 -0
- package/dist/evidence/bias-comparison.d.ts +133 -0
- package/dist/evidence/bias-comparison.d.ts.map +1 -0
- package/dist/evidence/bias-comparison.js +274 -0
- package/dist/evidence/bias-statistics.d.ts +72 -0
- package/dist/evidence/bias-statistics.d.ts.map +1 -0
- package/dist/evidence/bias-statistics.js +122 -0
- package/dist/evidence/bias-v2-semantics.d.ts +19 -0
- package/dist/evidence/bias-v2-semantics.d.ts.map +1 -0
- package/dist/evidence/bias-v2-semantics.js +133 -0
- package/dist/evidence/browser.d.ts +18 -0
- package/dist/evidence/browser.d.ts.map +1 -0
- package/dist/evidence/browser.js +14 -0
- package/dist/evidence/bundle.d.ts +1 -0
- package/dist/evidence/bundle.d.ts.map +1 -1
- package/dist/evidence/bundle.js +2 -0
- package/dist/evidence/index.d.ts +9 -2
- package/dist/evidence/index.d.ts.map +1 -1
- package/dist/evidence/index.js +5 -1
- package/dist/evidence/types.d.ts +175 -1
- package/dist/evidence/types.d.ts.map +1 -1
- package/dist/evidence/validate-bias-v2.d.ts +31 -0
- package/dist/evidence/validate-bias-v2.d.ts.map +1 -0
- package/dist/evidence/validate-bias-v2.js +40 -0
- package/dist/evidence/validate.d.ts +29 -2
- package/dist/evidence/validate.d.ts.map +1 -1
- package/dist/evidence/validate.js +90 -8
- package/dist/fairness-metrics.d.ts +15 -11
- package/dist/fairness-metrics.d.ts.map +1 -1
- package/dist/fairness-metrics.js +59 -183
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/inference/browser-inference.d.ts +6 -57
- package/dist/inference/browser-inference.d.ts.map +1 -1
- package/dist/inference/browser-inference.js +6 -219
- package/dist/inference/capability-detection.d.ts +9 -2
- package/dist/inference/capability-detection.d.ts.map +1 -1
- package/dist/inference/capability-detection.js +18 -16
- package/dist/inference/index.d.ts +15 -2
- package/dist/inference/index.d.ts.map +1 -1
- package/dist/inference/index.js +12 -3
- package/dist/inference/inference-factory.d.ts +6 -3
- package/dist/inference/inference-factory.d.ts.map +1 -1
- package/dist/inference/inference-factory.js +7 -26
- package/dist/inference/ollama-client.d.ts +29 -0
- package/dist/inference/ollama-client.d.ts.map +1 -1
- package/dist/inference/ollama-client.js +29 -0
- package/dist/inference/scored-log-prob-types.d.ts +64 -0
- package/dist/inference/scored-log-prob-types.d.ts.map +1 -0
- package/dist/inference/scored-log-prob-types.js +12 -0
- package/dist/inference/sequence-logprob.d.ts +34 -0
- package/dist/inference/sequence-logprob.d.ts.map +1 -0
- package/dist/inference/sequence-logprob.js +87 -0
- package/dist/inference/template-scoring.d.ts +124 -0
- package/dist/inference/template-scoring.d.ts.map +1 -0
- package/dist/inference/template-scoring.js +164 -0
- package/dist/inference/transformers-inference.d.ts +124 -0
- package/dist/inference/transformers-inference.d.ts.map +1 -0
- package/dist/inference/transformers-inference.js +392 -0
- package/dist/legal-checks/bias-metrics.d.ts +5 -7
- package/dist/legal-checks/bias-metrics.d.ts.map +1 -1
- package/dist/legal-checks/bias-metrics.js +8 -101
- package/dist/metrics/crows-pairs-bias.d.ts +17 -9
- package/dist/metrics/crows-pairs-bias.d.ts.map +1 -1
- package/dist/metrics/crows-pairs-bias.js +147 -59
- package/dist/metrics/stereotype-bias.d.ts +13 -5
- package/dist/metrics/stereotype-bias.d.ts.map +1 -1
- package/dist/metrics/stereotype-bias.js +16 -99
- package/dist/scanner/detectors/ai-framework-detector.js +1 -1
- package/dist/scanner/detectors/local-inference-detector.d.ts.map +1 -1
- package/dist/scanner/detectors/local-inference-detector.js +1 -1
- package/dist/scanner/detectors/package-json-detector.d.ts.map +1 -1
- package/dist/scanner/detectors/package-json-detector.js +1 -0
- package/dist/types.d.ts +103 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/validation/schema-validator.d.ts +1 -1
- package/dist/validation/schema-validator.d.ts.map +1 -1
- package/dist/validation/schema-validator.js +6 -0
- package/package.json +4 -2
- package/scripts/seed-logprob-fixture.mjs +179 -0
- package/scripts/ts-extensionless-resolve-hook.mjs +26 -0
- package/src/datasets/crows-pairs.ts +74 -0
- package/src/evidence/bias-comparison-markdown.ts +162 -0
- package/src/evidence/bias-comparison.ts +432 -0
- package/src/evidence/bias-statistics.ts +180 -0
- package/src/evidence/bias-v2-semantics.ts +214 -0
- package/src/evidence/browser.ts +35 -0
- package/src/evidence/bundle.ts +2 -0
- package/src/evidence/index.ts +31 -0
- package/src/evidence/types.ts +204 -1
- package/src/evidence/validate-bias-v2.ts +47 -0
- package/src/evidence/validate.ts +140 -13
- package/src/fairness-metrics.ts +79 -222
- package/src/index.ts +7 -0
- package/src/inference/browser-inference.ts +14 -256
- package/src/inference/capability-detection.ts +32 -18
- package/src/inference/index.ts +33 -5
- package/src/inference/inference-factory.ts +9 -30
- package/src/inference/ollama-client.ts +29 -0
- package/src/inference/scored-log-prob-types.ts +66 -0
- package/src/inference/sequence-logprob.ts +136 -0
- package/src/inference/template-scoring.ts +248 -0
- package/src/inference/transformers-inference.ts +494 -0
- package/src/legal-checks/bias-metrics.ts +20 -118
- package/src/metrics/crows-pairs-bias.ts +180 -83
- package/src/metrics/stereotype-bias.ts +27 -119
- package/src/scanner/detectors/ai-framework-detector.ts +1 -1
- package/src/scanner/detectors/local-inference-detector.ts +2 -1
- package/src/scanner/detectors/package-json-detector.ts +1 -0
- package/src/types.ts +108 -2
- package/src/validation/schema-validator.ts +15 -1
- package/tests/annex-iv-report.test.ts +1 -1
- package/tests/capability-detection.test.ts +57 -0
- package/tests/crows-pairs-bias.test.ts +286 -186
- package/tests/custom-dataset.test.ts +80 -0
- package/tests/evidence/bias-artifact-v2.test.ts +135 -0
- package/tests/evidence/bias-artifact.test.ts +78 -0
- package/tests/evidence/bias-comparison.test.ts +397 -0
- package/tests/evidence/bias-statistics.test.ts +191 -0
- package/tests/evidence/external-observation.test.ts +167 -0
- package/tests/evidence/spec-schemas.test.ts +35 -2
- package/tests/evidence/validate-bias-v2.test.ts +180 -0
- package/tests/fairness-metrics.test.ts +77 -1
- package/tests/inference/browser-inference-scoring.test.ts +159 -0
- package/tests/inference/fixtures/logprob-golden.json +69 -0
- package/tests/inference/gemma-template.integration.test.ts +81 -0
- package/tests/inference/logprob-golden.integration.test.ts +128 -0
- package/tests/inference/synthetic-logits.test.ts +188 -0
- package/tests/inference/template-scoring.test.ts +249 -0
- package/tests/inference/transformers-inference-template-mode.test.ts +214 -0
- package/tests/integration/bias-calculation.integration.test.ts +119 -599
- package/tests/stilled-ollama-paths.test.ts +76 -0
- package/src/inference/transformers.d.ts +0 -62
|
@@ -0,0 +1,392 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Browser-based inference using transformers.js
|
|
3
|
+
* For small models (≤3B params) that can run via WebGPU
|
|
4
|
+
*
|
|
5
|
+
* STRICT MODE: Real inference only. No demo data.
|
|
6
|
+
*/
|
|
7
|
+
import { sumSequenceLogProb } from "./sequence-logprob";
|
|
8
|
+
import { hasChatTemplate, isConditionalGenerationConfig, scoreTemplateContinuation, } from "./template-scoring";
|
|
9
|
+
async function sha256Hex(input) {
|
|
10
|
+
const bytes = new TextEncoder().encode(input);
|
|
11
|
+
const digest = await globalThis.crypto.subtle.digest("SHA-256", bytes);
|
|
12
|
+
return Array.from(new Uint8Array(digest))
|
|
13
|
+
.map((b) => b.toString(16).padStart(2, "0"))
|
|
14
|
+
.join("");
|
|
15
|
+
}
|
|
16
|
+
async function resolveTransformersVersion() {
|
|
17
|
+
try {
|
|
18
|
+
// v4's package.json declares a strict `exports` map with no "./package.json" subpath
|
|
19
|
+
// (unlike v2's), so `require("@huggingface/transformers/package.json")` throws
|
|
20
|
+
// ERR_PACKAGE_PATH_NOT_EXPORTED — surfaced while fixing Messmotor-Sanierung Task 4's
|
|
21
|
+
// onnxruntimeVersion provenance gap, where this exact function was silently returning
|
|
22
|
+
// "unknown" on every real run. `env.version` is v4's own documented public API for this
|
|
23
|
+
// and isn't subject to the exports map at all.
|
|
24
|
+
const { env } = await import("@huggingface/transformers");
|
|
25
|
+
return env.version ?? "unknown";
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
return "unknown";
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Resolve the tokenizer's BOS token id, aborting with a speaking error if undefined.
|
|
33
|
+
*
|
|
34
|
+
* @huggingface/transformers v4 exposes `bos_token` / `bos_token_id` directly as typed
|
|
35
|
+
* properties on `PreTrainedTokenizer` — the v2 lookup dance (`getToken("bos_token")` then
|
|
36
|
+
* `tokenizer.model.tokens_to_ids.get(...)`) is gone; deleting transformers.d.ts surfaced
|
|
37
|
+
* this (Messmotor-Sanierung Task 4, trap 3).
|
|
38
|
+
*/
|
|
39
|
+
function resolveBosTokenId(tokenizer, modelId) {
|
|
40
|
+
const bosToken = tokenizer.bos_token;
|
|
41
|
+
if (!bosToken) {
|
|
42
|
+
throw new Error(`getScoredLogProb: tokenizer for model "${modelId}" has no bos_token defined in its config; cannot construct a BOS-prefixed input for log-probability scoring.`);
|
|
43
|
+
}
|
|
44
|
+
const bosTokenId = tokenizer.bos_token_id;
|
|
45
|
+
if (bosTokenId === undefined || bosTokenId === null) {
|
|
46
|
+
throw new Error(`getScoredLogProb: tokenizer for model "${modelId}" could not resolve bos_token "${bosToken}" to a token id.`);
|
|
47
|
+
}
|
|
48
|
+
return bosTokenId;
|
|
49
|
+
}
|
|
50
|
+
// Supported models for browser inference
|
|
51
|
+
export const BROWSER_MODELS = [
|
|
52
|
+
{
|
|
53
|
+
id: "Xenova/phi-3-mini-4k-instruct",
|
|
54
|
+
name: "Phi-3 Mini",
|
|
55
|
+
size: "3.8B",
|
|
56
|
+
supported: true,
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
id: "Xenova/Phi-3.5-mini-instruct",
|
|
60
|
+
name: "Phi-3.5 Mini",
|
|
61
|
+
size: "3.8B",
|
|
62
|
+
supported: true,
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
id: "Xenova/gpt2",
|
|
66
|
+
name: "GPT-2",
|
|
67
|
+
size: "124M",
|
|
68
|
+
supported: true,
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
id: "Xenova/distilgpt2",
|
|
72
|
+
name: "DistilGPT-2",
|
|
73
|
+
size: "82M",
|
|
74
|
+
supported: true,
|
|
75
|
+
},
|
|
76
|
+
];
|
|
77
|
+
/**
|
|
78
|
+
* Check if WebGPU is available in the browser
|
|
79
|
+
*/
|
|
80
|
+
export async function checkWebGPU() {
|
|
81
|
+
if (typeof navigator === "undefined")
|
|
82
|
+
return false;
|
|
83
|
+
if (typeof window === "undefined")
|
|
84
|
+
return false;
|
|
85
|
+
try {
|
|
86
|
+
// Check for WebGPU support
|
|
87
|
+
const gpu = navigator.gpu;
|
|
88
|
+
if (!gpu)
|
|
89
|
+
return false;
|
|
90
|
+
// Try to get adapter
|
|
91
|
+
const adapter = await gpu.requestAdapter();
|
|
92
|
+
return adapter !== null;
|
|
93
|
+
}
|
|
94
|
+
catch {
|
|
95
|
+
return false;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Check if transformers.js is available
|
|
100
|
+
*/
|
|
101
|
+
export async function checkTransformersAvailable() {
|
|
102
|
+
try {
|
|
103
|
+
// Try dynamic import - this will fail if package not installed
|
|
104
|
+
await import("@huggingface/transformers");
|
|
105
|
+
return true;
|
|
106
|
+
}
|
|
107
|
+
catch {
|
|
108
|
+
return false;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Browser/Node inference engine using transformers.js — the single measurement
|
|
113
|
+
* engine for bias log-probabilities (see Messmotor-Sanierung: Ollama's logprobs
|
|
114
|
+
* only cover generated tokens, never prompt/sentence log-probability).
|
|
115
|
+
* STRICT MODE: Throws error if package not installed.
|
|
116
|
+
*
|
|
117
|
+
* Named `TransformersInference` (formerly `BrowserInference`); the old name is
|
|
118
|
+
* re-exported from `browser-inference.ts` for backward compatibility.
|
|
119
|
+
*/
|
|
120
|
+
export class TransformersInference {
|
|
121
|
+
modelId;
|
|
122
|
+
model = null;
|
|
123
|
+
tokenizer = null;
|
|
124
|
+
loadingPromise = null;
|
|
125
|
+
loadError = null;
|
|
126
|
+
statusCallback = null;
|
|
127
|
+
revision;
|
|
128
|
+
/**
|
|
129
|
+
* Public (not private) so callers building their own provenance/artifact records (e.g.
|
|
130
|
+
* the CLI's `run-bias-test.ts`) can read the CONFIGURED weight precision actually used to
|
|
131
|
+
* measure, instead of duplicating it as a separate hardcoded literal that could silently
|
|
132
|
+
* drift out of sync with the constructor argument (Messmotor-Sanierung Task 5 review
|
|
133
|
+
* finding).
|
|
134
|
+
*/
|
|
135
|
+
dtype;
|
|
136
|
+
/**
|
|
137
|
+
* Selected from the loaded TOKENIZER's chat template (`hasChatTemplate`), never from the
|
|
138
|
+
* model's architecture name and never from `modelId` — see `hasChatTemplate`
|
|
139
|
+
* (Messmotor-Sanierung Task 6c; supersedes the architecture-keyed detection from Task 5,
|
|
140
|
+
* which missed instruction-tuned `*ForCausalLM` models). `true` routes `getScoredLogProb`
|
|
141
|
+
* through the template-mode scorer instead of the Task 1 base (bare-sentence) mode.
|
|
142
|
+
*
|
|
143
|
+
* Independent of `loadTemplateMode`/`loadBaseMode` below, which pick the LOADER
|
|
144
|
+
* (AutoProcessor+Gemma4ForConditionalGeneration vs. AutoTokenizer+AutoModelForCausalLM)
|
|
145
|
+
* and stay keyed on architecture — a CausalLM-architecture instruction model (e.g.
|
|
146
|
+
* Llama-3.2-1B-Instruct) uses the base-mode loader but the template-mode scorer.
|
|
147
|
+
*/
|
|
148
|
+
isTemplateMode = false;
|
|
149
|
+
constructor(modelId = "Xenova/distilgpt2", statusCallback = null, options = {}) {
|
|
150
|
+
this.modelId = modelId;
|
|
151
|
+
this.statusCallback = statusCallback;
|
|
152
|
+
this.revision = options.revision ?? "main";
|
|
153
|
+
// Default "q8" preserves the prior effective behavior for browser generation
|
|
154
|
+
// (smaller/faster; the v2 default was `quantized: true`, which loaded 8-bit
|
|
155
|
+
// weights). Log-probability scoring for bias measurement must pass `dtype: "fp32"`
|
|
156
|
+
// explicitly (full precision) to avoid quantization noise in the measurement
|
|
157
|
+
// itself — see run-bias-test.ts and the golden integration test.
|
|
158
|
+
this.dtype = options.dtype ?? "q8";
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Check if browser inference is available
|
|
162
|
+
*/
|
|
163
|
+
async isAvailable() {
|
|
164
|
+
const hasTransformers = await checkTransformersAvailable();
|
|
165
|
+
return hasTransformers;
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Load the model (lazy loading)
|
|
169
|
+
* Handles concurrency with loadingPromise
|
|
170
|
+
*/
|
|
171
|
+
async load() {
|
|
172
|
+
if (this.model && this.tokenizer)
|
|
173
|
+
return;
|
|
174
|
+
if (this.loadingPromise)
|
|
175
|
+
return this.loadingPromise;
|
|
176
|
+
if (this.loadError)
|
|
177
|
+
throw this.loadError;
|
|
178
|
+
this.loadingPromise = this.performLoad().finally(() => {
|
|
179
|
+
this.loadingPromise = null;
|
|
180
|
+
});
|
|
181
|
+
return this.loadingPromise;
|
|
182
|
+
}
|
|
183
|
+
notifyStatus(status) {
|
|
184
|
+
if (this.statusCallback)
|
|
185
|
+
this.statusCallback(status);
|
|
186
|
+
}
|
|
187
|
+
async resolveDevice() {
|
|
188
|
+
const hasWebGPU = await checkWebGPU();
|
|
189
|
+
// v4 device names: Node's CPU backend is "cpu"; the browser's non-GPU backend is
|
|
190
|
+
// "wasm" (there is no "cpu" device in a browser). checkWebGPU() always resolves to
|
|
191
|
+
// false under Node, so this only matters for the (typeof window !== "undefined")
|
|
192
|
+
// branch — Messmotor-Sanierung Task 4, trap 2.
|
|
193
|
+
const isBrowserEnv = typeof window !== "undefined";
|
|
194
|
+
return hasWebGPU ? "webgpu" : isBrowserEnv ? "wasm" : "cpu";
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Conditional-generation/multimodal architecture (e.g. Gemma-4): AutoProcessor +
|
|
198
|
+
* Gemma4ForConditionalGeneration, per Task 5. `Gemma4ForConditionalGeneration` is
|
|
199
|
+
* exported directly by @huggingface/transformers@4.2.0 under that exact name (the ONNX
|
|
200
|
+
* repo's own example script uses the same pair) — no fallback to AutoModelForCausalLM is
|
|
201
|
+
* used or needed for this architecture.
|
|
202
|
+
*/
|
|
203
|
+
async loadTemplateMode(transformers, device) {
|
|
204
|
+
const { AutoTokenizer, AutoProcessor, Gemma4ForConditionalGeneration } = transformers;
|
|
205
|
+
this.notifyStatus("loading-processor");
|
|
206
|
+
const processor = await AutoProcessor.from_pretrained(this.modelId, {
|
|
207
|
+
revision: this.revision,
|
|
208
|
+
});
|
|
209
|
+
this.tokenizer =
|
|
210
|
+
processor.tokenizer ??
|
|
211
|
+
(await AutoTokenizer.from_pretrained(this.modelId, { revision: this.revision }));
|
|
212
|
+
this.notifyStatus("loading-model");
|
|
213
|
+
this.model = await Gemma4ForConditionalGeneration.from_pretrained(this.modelId, {
|
|
214
|
+
device,
|
|
215
|
+
dtype: this.dtype,
|
|
216
|
+
revision: this.revision,
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
/** Base mode (Task 1): classic causal LM, unchanged behavior. */
|
|
220
|
+
async loadBaseMode(transformers, device) {
|
|
221
|
+
const { AutoTokenizer, AutoModelForCausalLM } = transformers;
|
|
222
|
+
this.notifyStatus("loading-tokenizer");
|
|
223
|
+
this.tokenizer = await AutoTokenizer.from_pretrained(this.modelId, {
|
|
224
|
+
revision: this.revision,
|
|
225
|
+
});
|
|
226
|
+
this.notifyStatus("loading-model");
|
|
227
|
+
this.model = await AutoModelForCausalLM.from_pretrained(this.modelId, {
|
|
228
|
+
device,
|
|
229
|
+
dtype: this.dtype,
|
|
230
|
+
revision: this.revision,
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
async performLoad() {
|
|
234
|
+
this.notifyStatus("loading");
|
|
235
|
+
try {
|
|
236
|
+
const transformers = await import("@huggingface/transformers");
|
|
237
|
+
// LOADER selection is from the loaded CONFIG (`architectures`), never from `modelId`
|
|
238
|
+
// — see `isConditionalGenerationConfig` (Messmotor-Sanierung Task 5, ambiguity
|
|
239
|
+
// resolution 1). This is a loader decision only (which `from_pretrained` class pair
|
|
240
|
+
// to call); the SCORING MODE is decided separately below, from the tokenizer's chat
|
|
241
|
+
// template (Task 6c) — the two must not be conflated. This one extra
|
|
242
|
+
// `AutoConfig.from_pretrained` call runs for every model (base mode included); it
|
|
243
|
+
// hits the same on-disk cache as the tokenizer/model downloads, so the added cost is
|
|
244
|
+
// negligible next to those.
|
|
245
|
+
this.notifyStatus("loading-config");
|
|
246
|
+
const config = await transformers.AutoConfig.from_pretrained(this.modelId, {
|
|
247
|
+
revision: this.revision,
|
|
248
|
+
});
|
|
249
|
+
const useConditionalGenerationLoader = isConditionalGenerationConfig(config);
|
|
250
|
+
const device = await this.resolveDevice();
|
|
251
|
+
if (useConditionalGenerationLoader) {
|
|
252
|
+
await this.loadTemplateMode(transformers, device);
|
|
253
|
+
}
|
|
254
|
+
else {
|
|
255
|
+
await this.loadBaseMode(transformers, device);
|
|
256
|
+
}
|
|
257
|
+
if (!this.tokenizer) {
|
|
258
|
+
throw new Error(`performLoad: tokenizer missing after load for model "${this.modelId}" — cannot determine scoring mode.`);
|
|
259
|
+
}
|
|
260
|
+
// SCORING MODE selection (Messmotor-Sanierung Task 6c): keyed on the loaded
|
|
261
|
+
// tokenizer's chat template, independent of the loader chosen above — see
|
|
262
|
+
// `hasChatTemplate`.
|
|
263
|
+
this.isTemplateMode = hasChatTemplate(this.tokenizer);
|
|
264
|
+
this.notifyStatus("ready");
|
|
265
|
+
}
|
|
266
|
+
catch (err) {
|
|
267
|
+
this.loadError = err instanceof Error ? err : new Error(String(err));
|
|
268
|
+
this.notifyStatus("error");
|
|
269
|
+
throw this.loadError;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
/**
|
|
273
|
+
* Generate text.
|
|
274
|
+
*
|
|
275
|
+
* v4 note: previously this called a `pipeline("text-generation", ...)` built from the
|
|
276
|
+
* already-loaded model/tokenizer, which transformers.js re-resolves/re-loads internally
|
|
277
|
+
* — effectively a second model load that `getScoredLogProb`/`getLogProb` never used
|
|
278
|
+
* (Messmotor-Sanierung Task 4, trap 4). Calling `model.generate()` directly avoids that.
|
|
279
|
+
*/
|
|
280
|
+
async generate(prompt, maxTokens = 50) {
|
|
281
|
+
await this.load();
|
|
282
|
+
if (!this.tokenizer || !this.model) {
|
|
283
|
+
throw new Error("Model not loaded");
|
|
284
|
+
}
|
|
285
|
+
const tokenizer = this.tokenizer;
|
|
286
|
+
const model = this.model;
|
|
287
|
+
const { input_ids, attention_mask } = tokenizer(prompt, { return_tensor: true });
|
|
288
|
+
const outputs = await model.generate({
|
|
289
|
+
inputs: input_ids,
|
|
290
|
+
attention_mask,
|
|
291
|
+
max_new_tokens: maxTokens,
|
|
292
|
+
temperature: 0.7,
|
|
293
|
+
do_sample: true,
|
|
294
|
+
});
|
|
295
|
+
// model.generate()'s declared return type is `ModelOutput | Tensor`, but for causal LM
|
|
296
|
+
// decoding (no encoder-decoder, no streamer) it is always a plain Tensor of token ids —
|
|
297
|
+
// narrow via `unknown` rather than `any` since the union has no discriminant we can check.
|
|
298
|
+
const decoded = tokenizer.batch_decode(outputs, {
|
|
299
|
+
skip_special_tokens: true,
|
|
300
|
+
});
|
|
301
|
+
return decoded[0] ?? "";
|
|
302
|
+
}
|
|
303
|
+
/**
|
|
304
|
+
* Score a sentence's log-probability with full provenance.
|
|
305
|
+
* This is the "Gold Standard" for bias testing.
|
|
306
|
+
*
|
|
307
|
+
* BOS fix: GPT-2-family tokenizers do not insert a BOS token on their own, so the
|
|
308
|
+
* sentence is tokenized WITHOUT special tokens and a BOS token is explicitly
|
|
309
|
+
* prepended before feeding the model (input_ids and attention_mask are rebuilt
|
|
310
|
+
* together). This ensures the model sees P(first_token | BOS) instead of silently
|
|
311
|
+
* skipping the first sentence token. BOS itself is context and is never scored;
|
|
312
|
+
* every sentence token is scored via the half-open span [1, n+1).
|
|
313
|
+
*/
|
|
314
|
+
async getScoredLogProb(text) {
|
|
315
|
+
await this.load();
|
|
316
|
+
if (!this.tokenizer || !this.model) {
|
|
317
|
+
throw new Error("Model not loaded");
|
|
318
|
+
}
|
|
319
|
+
if (this.isTemplateMode) {
|
|
320
|
+
const libraryVersion = await resolveTransformersVersion();
|
|
321
|
+
return scoreTemplateContinuation({
|
|
322
|
+
tokenizer: this.tokenizer,
|
|
323
|
+
model: this.model,
|
|
324
|
+
modelId: this.modelId,
|
|
325
|
+
libraryVersion,
|
|
326
|
+
dtype: this.dtype,
|
|
327
|
+
}, text);
|
|
328
|
+
}
|
|
329
|
+
const tokenizer = this.tokenizer;
|
|
330
|
+
const model = this.model;
|
|
331
|
+
const bosTokenId = resolveBosTokenId(tokenizer, this.modelId);
|
|
332
|
+
// v4 renamed the tokenizer call's tensor-output option from `return_tensors: "pt"`
|
|
333
|
+
// (a v2 backend-name string) to `return_tensor: true` (a plain boolean) — surfaced by
|
|
334
|
+
// deleting transformers.d.ts; `return_tensors: "pt"` is not a recognized v4 option and
|
|
335
|
+
// would silently fail to type-check (Messmotor-Sanierung Task 4).
|
|
336
|
+
const encoded = tokenizer(text, { return_tensor: true, add_special_tokens: false });
|
|
337
|
+
const inputIdsTensor = encoded.input_ids;
|
|
338
|
+
const [batchSize] = inputIdsTensor.dims;
|
|
339
|
+
if (batchSize !== 1) {
|
|
340
|
+
throw new Error(`getScoredLogProb: batch_size must be 1, got ${batchSize}`);
|
|
341
|
+
}
|
|
342
|
+
const sentenceTokenIds = Array.from(inputIdsTensor.data, (v) => Number(v));
|
|
343
|
+
// Guard against double-BOS if a future tokenizer config already inserts it.
|
|
344
|
+
const finalTokenIds = sentenceTokenIds[0] === bosTokenId ? sentenceTokenIds : [bosTokenId, ...sentenceTokenIds];
|
|
345
|
+
const scoredTokenIds = finalTokenIds.slice(1);
|
|
346
|
+
const { Tensor } = await import("@huggingface/transformers");
|
|
347
|
+
const newLen = finalTokenIds.length;
|
|
348
|
+
const newInputIds = new Tensor("int64", BigInt64Array.from(finalTokenIds.map(BigInt)), [
|
|
349
|
+
1,
|
|
350
|
+
newLen,
|
|
351
|
+
]);
|
|
352
|
+
const newAttentionMask = new Tensor("int64", BigInt64Array.from(new Array(newLen).fill(1n)), [
|
|
353
|
+
1,
|
|
354
|
+
newLen,
|
|
355
|
+
]);
|
|
356
|
+
const outputs = await model({ input_ids: newInputIds, attention_mask: newAttentionMask });
|
|
357
|
+
const logits = outputs.logits;
|
|
358
|
+
const span = { start: 1, end: newLen };
|
|
359
|
+
const score = sumSequenceLogProb(logits, finalTokenIds, span);
|
|
360
|
+
const contextTokenHash = await sha256Hex(JSON.stringify([bosTokenId]));
|
|
361
|
+
const libraryVersion = await resolveTransformersVersion();
|
|
362
|
+
return {
|
|
363
|
+
score,
|
|
364
|
+
scoredTokenIds,
|
|
365
|
+
span,
|
|
366
|
+
contextTokenHash,
|
|
367
|
+
provenance: {
|
|
368
|
+
modelId: this.modelId,
|
|
369
|
+
dtype: this.dtype,
|
|
370
|
+
logitsDtype: String(logits.type ?? "unknown"),
|
|
371
|
+
backend: "node-onnx",
|
|
372
|
+
libraryVersion,
|
|
373
|
+
templateStrategy: "none",
|
|
374
|
+
},
|
|
375
|
+
method: "sequence_logprob_sum",
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
/**
|
|
379
|
+
* Calculate Log-Probability for a given text (score only).
|
|
380
|
+
* Thin wrapper around `getScoredLogProb` for callers that only need the score.
|
|
381
|
+
*/
|
|
382
|
+
async getLogProb(text) {
|
|
383
|
+
const { score } = await this.getScoredLogProb(text);
|
|
384
|
+
return score;
|
|
385
|
+
}
|
|
386
|
+
/**
|
|
387
|
+
* Get list of supported browser models
|
|
388
|
+
*/
|
|
389
|
+
static getSupportedModels() {
|
|
390
|
+
return BROWSER_MODELS.filter((m) => m.supported);
|
|
391
|
+
}
|
|
392
|
+
}
|
|
@@ -84,13 +84,11 @@ export interface CrowsPairsLogProbResult {
|
|
|
84
84
|
metricSources: Record<string, Citation[]>;
|
|
85
85
|
}
|
|
86
86
|
/**
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
*
|
|
90
|
-
* - deterministic sampling with seed (default 42)
|
|
91
|
-
* - modelId + engine recorded
|
|
87
|
+
* @deprecated Verwendet den entfernten Ollama-Logprob-Pfad (via `runStereotypeBiasTest`
|
|
88
|
+
* / `testPair`, and directly built its own `OllamaClient` for methodology detection);
|
|
89
|
+
* siehe bias --engine transformers.
|
|
92
90
|
*/
|
|
93
|
-
export declare function runCrowsPairsLogProbTest(
|
|
91
|
+
export declare function runCrowsPairsLogProbTest(_client: InferenceClient, _pairs: StereotypePair[], _options?: CrowsPairsLogProbOptions): Promise<CrowsPairsLogProbResult>;
|
|
94
92
|
export interface RegardScorer {
|
|
95
93
|
/** Returns a score in [-1, 1] (negative -> negative regard, positive -> positive regard) */
|
|
96
94
|
score(text: string): Promise<number>;
|
|
@@ -102,7 +100,7 @@ export interface ToxicityScorer {
|
|
|
102
100
|
modelId?: string;
|
|
103
101
|
}
|
|
104
102
|
/**
|
|
105
|
-
* Create a browser-side toxicity scorer using @
|
|
103
|
+
* Create a browser-side toxicity scorer using @huggingface/transformers (Detoxify-like classifier).
|
|
106
104
|
*
|
|
107
105
|
* Notes:
|
|
108
106
|
* - Model weights may be downloaded at runtime depending on caching environment.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bias-metrics.d.ts","sourceRoot":"","sources":["../../src/legal-checks/bias-metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,
|
|
1
|
+
{"version":3,"file":"bias-metrics.d.ts","sourceRoot":"","sources":["../../src/legal-checks/bias-metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC3E,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAE/E,MAAM,MAAM,kBAAkB,GAC1B,QAAQ,GACR,WAAW,GACX,KAAK,GACL,YAAY,GACZ,UAAU,GACV,oBAAoB,CAAC;AAEzB,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,eAAO,MAAM,cAAc,EAAE,QAAQ,EAWpC,CAAC;AAEF,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,CAqDrD,CAAC;AAMF,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,cAAc,EAAE,GAAG,cAAc,CAIlF;AAED,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,cAAc,EAAE,GAAG,cAAc,CAG9E;AAED,MAAM,WAAW,kBAAkB;IACjC,iEAAiE;IACjE,EAAE,EAAE,MAAM,CAAC;IACX,kBAAkB,EAAE,kBAAkB,CAAC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,qEAAqE;IACrE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC;IAC9B,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC;CAC/B;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,kBAAkB,EAAE,EAC3B,SAAS,SAAO,GACf,cAAc,CAuBhB;AAED,wBAAgB,4BAA4B,CAAC,MAAM,EAAE;IACnD,IAAI,EAAE,cAAc,EAAE,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,kBAAkB,EAAE,kBAAkB,GAAG,MAAM,CAAC;IAChD,mBAAmB,CAAC,EAAE,kBAAkB,EAAE,CAAC;IAC3C,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC,GAAG,cAAc,GAAG;IAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAA;CAAE,CA2BtF;AAMD,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACzD;AAED,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3E,WAAW,EAAE,QAAQ,GAAG,eAAe,GAAG,YAAY,CAAC;IACvD,OAAO,EAAE;QACP,gBAAgB,EAAE,MAAM,CAAC;QACzB,2BAA2B,EAAE,MAAM,CAAC;QACpC,gBAAgB,EAAE,MAAM,CAAC;QACzB,MAAM,EAAE,OAAO,CAAC;QAChB,QAAQ,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;KACvC,CAAC;IACF,UAAU,EAAE,KAAK,CAAC;QAChB,QAAQ,EAAE,QAAQ,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,2BAA2B,EAAE,MAAM,CAAC;QACpC,gBAAgB,EAAE,MAAM,CAAC;QACzB,MAAM,EAAE,OAAO,CAAC;QAChB,gBAAgB,EAAE,MAAM,CAAC;QACzB,QAAQ,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;KACvC,CAAC,CAAC;IACH,WAAW,CAAC,EAAE,cAAc,EAAE,CAAC;IAC/B,OAAO,EAAE,QAAQ,EAAE,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;CAC3C;AAED;;;;GAIG;AACH,wBAAsB,wBAAwB,CAC5C,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,cAAc,EAAE,EACxB,QAAQ,GAAE,wBAA6B,GACtC,OAAO,CAAC,uBAAuB,CAAC,CAIlC;AAKD,MAAM,WAAW,YAAY;IAC3B,4FAA4F;IAC5F,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,gEAAgE;IAChE,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;GAMG;AACH,wBAAsB,gCAAgC,CAAC,MAAM,CAAC,EAAE;IAC9D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,QAAQ,GAAG,KAAK,CAAC;CAC3B,GAAG,OAAO,CAAC,cAAc,CAAC,CA+B1B;AAED;;;GAGG;AACH,wBAAsB,8BAA8B,CAAC,MAAM,CAAC,EAAE;IAC5D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,QAAQ,GAAG,KAAK,CAAC;CAC3B,GAAG,OAAO,CAAC,YAAY,CAAC,CAuBxB;AAED;;;GAGG;AACH,wBAAgB,kCAAkC,CAAC,MAAM,EAAE;IACzD,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,cAAc,CAuBjB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,kBAAkB,CAAC;IAC1B,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC,kBAAkB,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/E,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,QAAQ,EAAE,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;CAC3C;AAED,wBAAsB,aAAa,CACjC,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,eAAe,EAAE,EAC1B,eAAe,SAAM,GACpB,OAAO,CAAC,YAAY,CAAC,CAwCvB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,QAAQ,EAAE,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;CAC3C;AAED,wBAAsB,eAAe,CACnC,MAAM,EAAE,cAAc,EACtB,KAAK,EAAE,MAAM,EAAE,EACf,YAAY,SAAM,GACjB,OAAO,CAAC,cAAc,CAAC,CAgCzB"}
|
|
@@ -9,7 +9,6 @@
|
|
|
9
9
|
* IMPORTANT: This module provides technical orientation, not legal advice.
|
|
10
10
|
*/
|
|
11
11
|
import { disparateImpact, equalizedOdds, statisticalParityDifference } from "../fairness-metrics";
|
|
12
|
-
import { runStereotypeBiasTest } from "../metrics";
|
|
13
12
|
export const AI_ACT_SOURCES = [
|
|
14
13
|
{ label: "EU AI Act", reference: "Verordnung (EU) 2024/1689" },
|
|
15
14
|
{
|
|
@@ -135,104 +134,16 @@ export function runArticle10NumericBiasSuite(params) {
|
|
|
135
134
|
metricSources: METRIC_SOURCES,
|
|
136
135
|
};
|
|
137
136
|
}
|
|
138
|
-
// ============================================================================
|
|
139
|
-
// Text / Generative model metrics (LLM)
|
|
140
|
-
// ============================================================================
|
|
141
|
-
function mulberry32(seed) {
|
|
142
|
-
let state = seed >>> 0;
|
|
143
|
-
return () => {
|
|
144
|
-
state = (state + 0x6d2b79f5) >>> 0;
|
|
145
|
-
let t = state;
|
|
146
|
-
t = Math.imul(t ^ (t >>> 15), t | 1);
|
|
147
|
-
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
|
148
|
-
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
149
|
-
};
|
|
150
|
-
}
|
|
151
|
-
function seededSample(arr, n, seed) {
|
|
152
|
-
const rng = mulberry32(seed);
|
|
153
|
-
const copy = arr.slice();
|
|
154
|
-
// Fisher-Yates
|
|
155
|
-
for (let i = copy.length - 1; i > 0; i--) {
|
|
156
|
-
const j = Math.floor(rng() * (i + 1));
|
|
157
|
-
const a = copy[i];
|
|
158
|
-
const b = copy[j];
|
|
159
|
-
if (a === undefined || b === undefined)
|
|
160
|
-
continue;
|
|
161
|
-
copy[i] = b;
|
|
162
|
-
copy[j] = a;
|
|
163
|
-
}
|
|
164
|
-
return copy.slice(0, Math.min(n, copy.length));
|
|
165
|
-
}
|
|
166
137
|
/**
|
|
167
|
-
*
|
|
168
|
-
*
|
|
169
|
-
*
|
|
170
|
-
* - deterministic sampling with seed (default 42)
|
|
171
|
-
* - modelId + engine recorded
|
|
138
|
+
* @deprecated Verwendet den entfernten Ollama-Logprob-Pfad (via `runStereotypeBiasTest`
|
|
139
|
+
* / `testPair`, and directly built its own `OllamaClient` for methodology detection);
|
|
140
|
+
* siehe bias --engine transformers.
|
|
172
141
|
*/
|
|
173
|
-
export async function runCrowsPairsLogProbTest(
|
|
174
|
-
|
|
175
|
-
const maxPairs = options.maxPairs ?? pairs.length;
|
|
176
|
-
const sampled = seededSample(pairs, maxPairs, seed);
|
|
177
|
-
const result = await runStereotypeBiasTest(client, sampled, {
|
|
178
|
-
maxPairs: sampled.length,
|
|
179
|
-
onProgress: options.onProgress,
|
|
180
|
-
});
|
|
181
|
-
// Determine methodology based on engine and actual capability
|
|
182
|
-
let methodology = "logits";
|
|
183
|
-
if (client.engine === "ollama") {
|
|
184
|
-
// For Ollama, we need to check if the model actually supports log-probabilities
|
|
185
|
-
// The OllamaClient.getLogProb now uses getLogProbWithFallback internally,
|
|
186
|
-
// which returns exact log-probs if supported. We can detect this by checking
|
|
187
|
-
// if the client has capability detection.
|
|
188
|
-
try {
|
|
189
|
-
// Import OllamaClient to check capability
|
|
190
|
-
const { OllamaClient } = await import("../inference/ollama-client");
|
|
191
|
-
const ollamaClient = new OllamaClient(client.modelId);
|
|
192
|
-
const supportsLogProbs = await ollamaClient.detectLogProbSupport();
|
|
193
|
-
methodology = supportsLogProbs ? "logits" : "latency-proxy";
|
|
194
|
-
}
|
|
195
|
-
catch {
|
|
196
|
-
// Fallback to latency-proxy if we can't detect
|
|
197
|
-
methodology = "latency-proxy";
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
|
-
if (client.engine === "demo")
|
|
201
|
-
methodology = "simulation";
|
|
202
|
-
return {
|
|
203
|
-
engine: client.engine,
|
|
204
|
-
modelId: client.modelId,
|
|
205
|
-
timestamp: result.timestamp,
|
|
206
|
-
dataset: {
|
|
207
|
-
name: "CrowS-Pairs-DE (compatible)",
|
|
208
|
-
version: "local",
|
|
209
|
-
maxPairs: sampled.length,
|
|
210
|
-
seed,
|
|
211
|
-
},
|
|
212
|
-
methodology,
|
|
213
|
-
overall: {
|
|
214
|
-
averageBiasScore: Number(result.overallBiasScore.toFixed(6)),
|
|
215
|
-
stereotypePreferencePercent: Number(result.overallStereotypePreference.toFixed(2)),
|
|
216
|
-
thresholdPercent: result.threshold,
|
|
217
|
-
passed: result.passed,
|
|
218
|
-
severity: result.overallSeverity,
|
|
219
|
-
},
|
|
220
|
-
byBiasType: result.categoryResults.map((c) => ({
|
|
221
|
-
biasType: c.biasType,
|
|
222
|
-
pairsCount: c.pairsCount,
|
|
223
|
-
stereotypePreferencePercent: Number(c.stereotypePreference.toFixed(2)),
|
|
224
|
-
averageBiasScore: Number(c.averageBiasScore.toFixed(6)),
|
|
225
|
-
passed: c.passed,
|
|
226
|
-
thresholdPercent: c.threshold,
|
|
227
|
-
severity: c.severity,
|
|
228
|
-
})),
|
|
229
|
-
pairResults: result.pairResults,
|
|
230
|
-
sources: AI_ACT_SOURCES,
|
|
231
|
-
metricSources: METRIC_SOURCES,
|
|
232
|
-
};
|
|
142
|
+
export async function runCrowsPairsLogProbTest(_client, _pairs, _options = {}) {
|
|
143
|
+
throw new Error("runCrowsPairsLogProbTest verwendet den entfernten Ollama-Logprob-Pfad; siehe bias --engine transformers.");
|
|
233
144
|
}
|
|
234
145
|
/**
|
|
235
|
-
* Create a browser-side toxicity scorer using @
|
|
146
|
+
* Create a browser-side toxicity scorer using @huggingface/transformers (Detoxify-like classifier).
|
|
236
147
|
*
|
|
237
148
|
* Notes:
|
|
238
149
|
* - Model weights may be downloaded at runtime depending on caching environment.
|
|
@@ -242,9 +153,7 @@ export async function createTransformersToxicityScorer(params) {
|
|
|
242
153
|
const modelId = params?.modelId ?? "Xenova/toxic-bert";
|
|
243
154
|
const device = params?.device;
|
|
244
155
|
// Dynamic import to avoid forcing downloads in Node test environments.
|
|
245
|
-
|
|
246
|
-
const { pipeline } = (await import("@xenova/transformers"));
|
|
247
|
-
// biome-ignore lint/suspicious/noExplicitAny: library internal
|
|
156
|
+
const { pipeline } = await import("@huggingface/transformers");
|
|
248
157
|
const clf = await pipeline("text-classification", modelId, device ? { device } : undefined);
|
|
249
158
|
return {
|
|
250
159
|
modelId,
|
|
@@ -274,9 +183,7 @@ export async function createTransformersToxicityScorer(params) {
|
|
|
274
183
|
export async function createTransformersRegardScorer(params) {
|
|
275
184
|
const modelId = params?.modelId ?? "Xenova/cardiffnlp/twitter-xlm-roberta-base-sentiment";
|
|
276
185
|
const device = params?.device;
|
|
277
|
-
|
|
278
|
-
const { pipeline } = (await import("@xenova/transformers"));
|
|
279
|
-
// biome-ignore lint/suspicious/noExplicitAny: library internal
|
|
186
|
+
const { pipeline } = await import("@huggingface/transformers");
|
|
280
187
|
const clf = await pipeline("text-classification", modelId, device ? { device } : undefined);
|
|
281
188
|
return {
|
|
282
189
|
modelId,
|
|
@@ -1,20 +1,28 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* CrowS-Pairs Bias Calculation Engine
|
|
3
|
-
* Implements the CrowS-Pairs methodology (Nangia et al., 2020) for measuring stereotypical bias
|
|
4
|
-
*
|
|
3
|
+
* Implements the CrowS-Pairs methodology (Nangia et al., 2020) for measuring stereotypical bias.
|
|
4
|
+
*
|
|
5
|
+
* Single measurement engine: any provider passed in must implement
|
|
6
|
+
* `SequenceLogProbProvider.getScoredLogProb`, a real, cross-checked sentence
|
|
7
|
+
* log-probability (see Task 1 / Messmotor-Sanierung). Ollama never measured this
|
|
8
|
+
* (its logprobs cover only generated tokens, not the prompt/sentence), so
|
|
9
|
+
* `OllamaClient` is structurally excluded — it has no `getScoredLogProb` method.
|
|
10
|
+
*
|
|
11
|
+
* Fail-closed by default: a pair that fails measurement aborts the whole run.
|
|
12
|
+
* `config.allowPartial` switches to a diagnostic mode that collects failures
|
|
13
|
+
* into `failedPairs` and reports `runStatus: "partial"` instead.
|
|
5
14
|
*/
|
|
6
|
-
import {
|
|
7
|
-
import { OllamaClient } from "../inference/ollama-client";
|
|
15
|
+
import type { SequenceLogProbProvider } from "../inference";
|
|
8
16
|
import type { BiasTestConfig, CrowsPairsBiasResult, CrowsPairsEntry } from "../types";
|
|
9
17
|
/**
|
|
10
|
-
* Calculate CrowS-Pairs bias score using the
|
|
18
|
+
* Calculate CrowS-Pairs bias score using the given sequence-log-probability provider.
|
|
11
19
|
* Implements the methodology: mean(logprob_stereo - logprob_anti)
|
|
12
20
|
*
|
|
13
|
-
* @param config - Configuration including dataset, model, and
|
|
14
|
-
* @param
|
|
15
|
-
* @returns Promise<CrowsPairsBiasResult> with bias score and
|
|
21
|
+
* @param config - Configuration including dataset, model, engine, and allowPartial
|
|
22
|
+
* @param provider - A `SequenceLogProbProvider` (currently: `TransformersInference`)
|
|
23
|
+
* @returns Promise<CrowsPairsBiasResult> with bias score, metadata, and run status
|
|
16
24
|
*/
|
|
17
|
-
export declare function calculateCrowsPairsBias(config: BiasTestConfig,
|
|
25
|
+
export declare function calculateCrowsPairsBias(config: BiasTestConfig, provider: SequenceLogProbProvider): Promise<CrowsPairsBiasResult>;
|
|
18
26
|
/**
|
|
19
27
|
* Calculate bias using log-probability method (exact implementation)
|
|
20
28
|
* This is the gold standard method when log-probabilities are available
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crows-pairs-bias.d.ts","sourceRoot":"","sources":["../../src/metrics/crows-pairs-bias.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"crows-pairs-bias.d.ts","sourceRoot":"","sources":["../../src/metrics/crows-pairs-bias.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,KAAK,EACV,cAAc,EACd,oBAAoB,EAEpB,eAAe,EAIhB,MAAM,UAAU,CAAC;AAoBlB;;;;;;;GAOG;AACH,wBAAsB,uBAAuB,CAC3C,MAAM,EAAE,cAAc,EACtB,QAAQ,EAAE,uBAAuB,GAChC,OAAO,CAAC,oBAAoB,CAAC,CAsD/B;AA0ID;;;;;;;GAOG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,eAAe,EAAE,EAC1B,UAAU,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,GAChD,OAAO,CAAC,MAAM,EAAE,CAAC,CAiBnB;AAED;;;;;;;GAOG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,eAAe,EAAE,EAC1B,UAAU,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,GAChD,OAAO,CAAC,MAAM,EAAE,CAAC,CAqBnB;AAED;;;GAGG;AACH,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,IAAI,eAAe,EAAE,CAyB1F"}
|