@euconform/core 1.5.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/.turbo/turbo-build.log +1 -1
  2. package/CHANGELOG.md +34 -0
  3. package/dist/datasets/crows-pairs.d.ts +18 -0
  4. package/dist/datasets/crows-pairs.d.ts.map +1 -1
  5. package/dist/datasets/crows-pairs.js +58 -0
  6. package/dist/evidence/bias-comparison-markdown.d.ts +11 -0
  7. package/dist/evidence/bias-comparison-markdown.d.ts.map +1 -0
  8. package/dist/evidence/bias-comparison-markdown.js +120 -0
  9. package/dist/evidence/bias-comparison.d.ts +133 -0
  10. package/dist/evidence/bias-comparison.d.ts.map +1 -0
  11. package/dist/evidence/bias-comparison.js +274 -0
  12. package/dist/evidence/bias-statistics.d.ts +72 -0
  13. package/dist/evidence/bias-statistics.d.ts.map +1 -0
  14. package/dist/evidence/bias-statistics.js +122 -0
  15. package/dist/evidence/bias-v2-semantics.d.ts +19 -0
  16. package/dist/evidence/bias-v2-semantics.d.ts.map +1 -0
  17. package/dist/evidence/bias-v2-semantics.js +133 -0
  18. package/dist/evidence/browser.d.ts +18 -0
  19. package/dist/evidence/browser.d.ts.map +1 -0
  20. package/dist/evidence/browser.js +14 -0
  21. package/dist/evidence/bundle.d.ts +1 -0
  22. package/dist/evidence/bundle.d.ts.map +1 -1
  23. package/dist/evidence/bundle.js +2 -0
  24. package/dist/evidence/index.d.ts +9 -2
  25. package/dist/evidence/index.d.ts.map +1 -1
  26. package/dist/evidence/index.js +5 -1
  27. package/dist/evidence/types.d.ts +175 -1
  28. package/dist/evidence/types.d.ts.map +1 -1
  29. package/dist/evidence/validate-bias-v2.d.ts +31 -0
  30. package/dist/evidence/validate-bias-v2.d.ts.map +1 -0
  31. package/dist/evidence/validate-bias-v2.js +40 -0
  32. package/dist/evidence/validate.d.ts +29 -2
  33. package/dist/evidence/validate.d.ts.map +1 -1
  34. package/dist/evidence/validate.js +90 -8
  35. package/dist/fairness-metrics.d.ts +15 -11
  36. package/dist/fairness-metrics.d.ts.map +1 -1
  37. package/dist/fairness-metrics.js +59 -183
  38. package/dist/index.d.ts +2 -2
  39. package/dist/index.d.ts.map +1 -1
  40. package/dist/index.js +1 -1
  41. package/dist/inference/browser-inference.d.ts +6 -57
  42. package/dist/inference/browser-inference.d.ts.map +1 -1
  43. package/dist/inference/browser-inference.js +6 -219
  44. package/dist/inference/capability-detection.d.ts +9 -2
  45. package/dist/inference/capability-detection.d.ts.map +1 -1
  46. package/dist/inference/capability-detection.js +18 -16
  47. package/dist/inference/index.d.ts +15 -2
  48. package/dist/inference/index.d.ts.map +1 -1
  49. package/dist/inference/index.js +12 -3
  50. package/dist/inference/inference-factory.d.ts +6 -3
  51. package/dist/inference/inference-factory.d.ts.map +1 -1
  52. package/dist/inference/inference-factory.js +7 -26
  53. package/dist/inference/ollama-client.d.ts +29 -0
  54. package/dist/inference/ollama-client.d.ts.map +1 -1
  55. package/dist/inference/ollama-client.js +29 -0
  56. package/dist/inference/scored-log-prob-types.d.ts +64 -0
  57. package/dist/inference/scored-log-prob-types.d.ts.map +1 -0
  58. package/dist/inference/scored-log-prob-types.js +12 -0
  59. package/dist/inference/sequence-logprob.d.ts +34 -0
  60. package/dist/inference/sequence-logprob.d.ts.map +1 -0
  61. package/dist/inference/sequence-logprob.js +87 -0
  62. package/dist/inference/template-scoring.d.ts +124 -0
  63. package/dist/inference/template-scoring.d.ts.map +1 -0
  64. package/dist/inference/template-scoring.js +164 -0
  65. package/dist/inference/transformers-inference.d.ts +124 -0
  66. package/dist/inference/transformers-inference.d.ts.map +1 -0
  67. package/dist/inference/transformers-inference.js +392 -0
  68. package/dist/legal-checks/bias-metrics.d.ts +5 -7
  69. package/dist/legal-checks/bias-metrics.d.ts.map +1 -1
  70. package/dist/legal-checks/bias-metrics.js +8 -101
  71. package/dist/metrics/crows-pairs-bias.d.ts +17 -9
  72. package/dist/metrics/crows-pairs-bias.d.ts.map +1 -1
  73. package/dist/metrics/crows-pairs-bias.js +147 -59
  74. package/dist/metrics/stereotype-bias.d.ts +13 -5
  75. package/dist/metrics/stereotype-bias.d.ts.map +1 -1
  76. package/dist/metrics/stereotype-bias.js +16 -99
  77. package/dist/scanner/detectors/ai-framework-detector.js +1 -1
  78. package/dist/scanner/detectors/local-inference-detector.d.ts.map +1 -1
  79. package/dist/scanner/detectors/local-inference-detector.js +1 -1
  80. package/dist/scanner/detectors/package-json-detector.d.ts.map +1 -1
  81. package/dist/scanner/detectors/package-json-detector.js +1 -0
  82. package/dist/types.d.ts +103 -2
  83. package/dist/types.d.ts.map +1 -1
  84. package/dist/validation/schema-validator.d.ts +1 -1
  85. package/dist/validation/schema-validator.d.ts.map +1 -1
  86. package/dist/validation/schema-validator.js +6 -0
  87. package/package.json +4 -2
  88. package/scripts/seed-logprob-fixture.mjs +179 -0
  89. package/scripts/ts-extensionless-resolve-hook.mjs +26 -0
  90. package/src/datasets/crows-pairs.ts +74 -0
  91. package/src/evidence/bias-comparison-markdown.ts +162 -0
  92. package/src/evidence/bias-comparison.ts +432 -0
  93. package/src/evidence/bias-statistics.ts +180 -0
  94. package/src/evidence/bias-v2-semantics.ts +214 -0
  95. package/src/evidence/browser.ts +35 -0
  96. package/src/evidence/bundle.ts +2 -0
  97. package/src/evidence/index.ts +31 -0
  98. package/src/evidence/types.ts +204 -1
  99. package/src/evidence/validate-bias-v2.ts +47 -0
  100. package/src/evidence/validate.ts +140 -13
  101. package/src/fairness-metrics.ts +79 -222
  102. package/src/index.ts +7 -0
  103. package/src/inference/browser-inference.ts +14 -256
  104. package/src/inference/capability-detection.ts +32 -18
  105. package/src/inference/index.ts +33 -5
  106. package/src/inference/inference-factory.ts +9 -30
  107. package/src/inference/ollama-client.ts +29 -0
  108. package/src/inference/scored-log-prob-types.ts +66 -0
  109. package/src/inference/sequence-logprob.ts +136 -0
  110. package/src/inference/template-scoring.ts +248 -0
  111. package/src/inference/transformers-inference.ts +494 -0
  112. package/src/legal-checks/bias-metrics.ts +20 -118
  113. package/src/metrics/crows-pairs-bias.ts +180 -83
  114. package/src/metrics/stereotype-bias.ts +27 -119
  115. package/src/scanner/detectors/ai-framework-detector.ts +1 -1
  116. package/src/scanner/detectors/local-inference-detector.ts +2 -1
  117. package/src/scanner/detectors/package-json-detector.ts +1 -0
  118. package/src/types.ts +108 -2
  119. package/src/validation/schema-validator.ts +15 -1
  120. package/tests/annex-iv-report.test.ts +1 -1
  121. package/tests/capability-detection.test.ts +57 -0
  122. package/tests/crows-pairs-bias.test.ts +286 -186
  123. package/tests/custom-dataset.test.ts +80 -0
  124. package/tests/evidence/bias-artifact-v2.test.ts +135 -0
  125. package/tests/evidence/bias-artifact.test.ts +78 -0
  126. package/tests/evidence/bias-comparison.test.ts +397 -0
  127. package/tests/evidence/bias-statistics.test.ts +191 -0
  128. package/tests/evidence/external-observation.test.ts +167 -0
  129. package/tests/evidence/spec-schemas.test.ts +35 -2
  130. package/tests/evidence/validate-bias-v2.test.ts +180 -0
  131. package/tests/fairness-metrics.test.ts +77 -1
  132. package/tests/inference/browser-inference-scoring.test.ts +159 -0
  133. package/tests/inference/fixtures/logprob-golden.json +69 -0
  134. package/tests/inference/gemma-template.integration.test.ts +81 -0
  135. package/tests/inference/logprob-golden.integration.test.ts +128 -0
  136. package/tests/inference/synthetic-logits.test.ts +188 -0
  137. package/tests/inference/template-scoring.test.ts +249 -0
  138. package/tests/inference/transformers-inference-template-mode.test.ts +214 -0
  139. package/tests/integration/bias-calculation.integration.test.ts +119 -599
  140. package/tests/stilled-ollama-paths.test.ts +76 -0
  141. package/src/inference/transformers.d.ts +0 -62
@@ -0,0 +1,392 @@
1
+ /**
2
+ * Browser-based inference using transformers.js
3
+ * For small models (≤3B params) that can run via WebGPU
4
+ *
5
+ * STRICT MODE: Real inference only. No demo data.
6
+ */
7
+ import { sumSequenceLogProb } from "./sequence-logprob";
8
+ import { hasChatTemplate, isConditionalGenerationConfig, scoreTemplateContinuation, } from "./template-scoring";
9
+ async function sha256Hex(input) {
10
+ const bytes = new TextEncoder().encode(input);
11
+ const digest = await globalThis.crypto.subtle.digest("SHA-256", bytes);
12
+ return Array.from(new Uint8Array(digest))
13
+ .map((b) => b.toString(16).padStart(2, "0"))
14
+ .join("");
15
+ }
16
+ async function resolveTransformersVersion() {
17
+ try {
18
+ // v4's package.json declares a strict `exports` map with no "./package.json" subpath
19
+ // (unlike v2's), so `require("@huggingface/transformers/package.json")` throws
20
+ // ERR_PACKAGE_PATH_NOT_EXPORTED — surfaced while fixing Messmotor-Sanierung Task 4's
21
+ // onnxruntimeVersion provenance gap, where this exact function was silently returning
22
+ // "unknown" on every real run. `env.version` is v4's own documented public API for this
23
+ // and isn't subject to the exports map at all.
24
+ const { env } = await import("@huggingface/transformers");
25
+ return env.version ?? "unknown";
26
+ }
27
+ catch {
28
+ return "unknown";
29
+ }
30
+ }
31
+ /**
32
+ * Resolve the tokenizer's BOS token id, aborting with a speaking error if undefined.
33
+ *
34
+ * @huggingface/transformers v4 exposes `bos_token` / `bos_token_id` directly as typed
35
+ * properties on `PreTrainedTokenizer` — the v2 lookup dance (`getToken("bos_token")` then
36
+ * `tokenizer.model.tokens_to_ids.get(...)`) is gone; deleting transformers.d.ts surfaced
37
+ * this (Messmotor-Sanierung Task 4, trap 3).
38
+ */
39
+ function resolveBosTokenId(tokenizer, modelId) {
40
+ const bosToken = tokenizer.bos_token;
41
+ if (!bosToken) {
42
+ throw new Error(`getScoredLogProb: tokenizer for model "${modelId}" has no bos_token defined in its config; cannot construct a BOS-prefixed input for log-probability scoring.`);
43
+ }
44
+ const bosTokenId = tokenizer.bos_token_id;
45
+ if (bosTokenId === undefined || bosTokenId === null) {
46
+ throw new Error(`getScoredLogProb: tokenizer for model "${modelId}" could not resolve bos_token "${bosToken}" to a token id.`);
47
+ }
48
+ return bosTokenId;
49
+ }
50
+ // Supported models for browser inference
51
+ export const BROWSER_MODELS = [
52
+ {
53
+ id: "Xenova/phi-3-mini-4k-instruct",
54
+ name: "Phi-3 Mini",
55
+ size: "3.8B",
56
+ supported: true,
57
+ },
58
+ {
59
+ id: "Xenova/Phi-3.5-mini-instruct",
60
+ name: "Phi-3.5 Mini",
61
+ size: "3.8B",
62
+ supported: true,
63
+ },
64
+ {
65
+ id: "Xenova/gpt2",
66
+ name: "GPT-2",
67
+ size: "124M",
68
+ supported: true,
69
+ },
70
+ {
71
+ id: "Xenova/distilgpt2",
72
+ name: "DistilGPT-2",
73
+ size: "82M",
74
+ supported: true,
75
+ },
76
+ ];
77
+ /**
78
+ * Check if WebGPU is available in the browser
79
+ */
80
+ export async function checkWebGPU() {
81
+ if (typeof navigator === "undefined")
82
+ return false;
83
+ if (typeof window === "undefined")
84
+ return false;
85
+ try {
86
+ // Check for WebGPU support
87
+ const gpu = navigator.gpu;
88
+ if (!gpu)
89
+ return false;
90
+ // Try to get adapter
91
+ const adapter = await gpu.requestAdapter();
92
+ return adapter !== null;
93
+ }
94
+ catch {
95
+ return false;
96
+ }
97
+ }
98
+ /**
99
+ * Check if transformers.js is available
100
+ */
101
+ export async function checkTransformersAvailable() {
102
+ try {
103
+ // Try dynamic import - this will fail if package not installed
104
+ await import("@huggingface/transformers");
105
+ return true;
106
+ }
107
+ catch {
108
+ return false;
109
+ }
110
+ }
111
+ /**
112
+ * Browser/Node inference engine using transformers.js — the single measurement
113
+ * engine for bias log-probabilities (see Messmotor-Sanierung: Ollama's logprobs
114
+ * only cover generated tokens, never prompt/sentence log-probability).
115
+ * STRICT MODE: Throws error if package not installed.
116
+ *
117
+ * Named `TransformersInference` (formerly `BrowserInference`); the old name is
118
+ * re-exported from `browser-inference.ts` for backward compatibility.
119
+ */
120
+ export class TransformersInference {
121
+ modelId;
122
+ model = null;
123
+ tokenizer = null;
124
+ loadingPromise = null;
125
+ loadError = null;
126
+ statusCallback = null;
127
+ revision;
128
+ /**
129
+ * Public (not private) so callers building their own provenance/artifact records (e.g.
130
+ * the CLI's `run-bias-test.ts`) can read the CONFIGURED weight precision actually used to
131
+ * measure, instead of duplicating it as a separate hardcoded literal that could silently
132
+ * drift out of sync with the constructor argument (Messmotor-Sanierung Task 5 review
133
+ * finding).
134
+ */
135
+ dtype;
136
+ /**
137
+ * Selected from the loaded TOKENIZER's chat template (`hasChatTemplate`), never from the
138
+ * model's architecture name and never from `modelId` — see `hasChatTemplate`
139
+ * (Messmotor-Sanierung Task 6c; supersedes the architecture-keyed detection from Task 5,
140
+ * which missed instruction-tuned `*ForCausalLM` models). `true` routes `getScoredLogProb`
141
+ * through the template-mode scorer instead of the Task 1 base (bare-sentence) mode.
142
+ *
143
+ * Independent of `loadTemplateMode`/`loadBaseMode` below, which pick the LOADER
144
+ * (AutoProcessor+Gemma4ForConditionalGeneration vs. AutoTokenizer+AutoModelForCausalLM)
145
+ * and stay keyed on architecture — a CausalLM-architecture instruction model (e.g.
146
+ * Llama-3.2-1B-Instruct) uses the base-mode loader but the template-mode scorer.
147
+ */
148
+ isTemplateMode = false;
149
+ constructor(modelId = "Xenova/distilgpt2", statusCallback = null, options = {}) {
150
+ this.modelId = modelId;
151
+ this.statusCallback = statusCallback;
152
+ this.revision = options.revision ?? "main";
153
+ // Default "q8" preserves the prior effective behavior for browser generation
154
+ // (smaller/faster; the v2 default was `quantized: true`, which loaded 8-bit
155
+ // weights). Log-probability scoring for bias measurement must pass `dtype: "fp32"`
156
+ // explicitly (full precision) to avoid quantization noise in the measurement
157
+ // itself — see run-bias-test.ts and the golden integration test.
158
+ this.dtype = options.dtype ?? "q8";
159
+ }
160
+ /**
161
+ * Check if browser inference is available
162
+ */
163
+ async isAvailable() {
164
+ const hasTransformers = await checkTransformersAvailable();
165
+ return hasTransformers;
166
+ }
167
+ /**
168
+ * Load the model (lazy loading)
169
+ * Handles concurrency with loadingPromise
170
+ */
171
+ async load() {
172
+ if (this.model && this.tokenizer)
173
+ return;
174
+ if (this.loadingPromise)
175
+ return this.loadingPromise;
176
+ if (this.loadError)
177
+ throw this.loadError;
178
+ this.loadingPromise = this.performLoad().finally(() => {
179
+ this.loadingPromise = null;
180
+ });
181
+ return this.loadingPromise;
182
+ }
183
+ notifyStatus(status) {
184
+ if (this.statusCallback)
185
+ this.statusCallback(status);
186
+ }
187
+ async resolveDevice() {
188
+ const hasWebGPU = await checkWebGPU();
189
+ // v4 device names: Node's CPU backend is "cpu"; the browser's non-GPU backend is
190
+ // "wasm" (there is no "cpu" device in a browser). checkWebGPU() always resolves to
191
+ // false under Node, so this only matters for the (typeof window !== "undefined")
192
+ // branch — Messmotor-Sanierung Task 4, trap 2.
193
+ const isBrowserEnv = typeof window !== "undefined";
194
+ return hasWebGPU ? "webgpu" : isBrowserEnv ? "wasm" : "cpu";
195
+ }
196
+ /**
197
+ * Conditional-generation/multimodal architecture (e.g. Gemma-4): AutoProcessor +
198
+ * Gemma4ForConditionalGeneration, per Task 5. `Gemma4ForConditionalGeneration` is
199
+ * exported directly by @huggingface/transformers@4.2.0 under that exact name (the ONNX
200
+ * repo's own example script uses the same pair) — no fallback to AutoModelForCausalLM is
201
+ * used or needed for this architecture.
202
+ */
203
+ async loadTemplateMode(transformers, device) {
204
+ const { AutoTokenizer, AutoProcessor, Gemma4ForConditionalGeneration } = transformers;
205
+ this.notifyStatus("loading-processor");
206
+ const processor = await AutoProcessor.from_pretrained(this.modelId, {
207
+ revision: this.revision,
208
+ });
209
+ this.tokenizer =
210
+ processor.tokenizer ??
211
+ (await AutoTokenizer.from_pretrained(this.modelId, { revision: this.revision }));
212
+ this.notifyStatus("loading-model");
213
+ this.model = await Gemma4ForConditionalGeneration.from_pretrained(this.modelId, {
214
+ device,
215
+ dtype: this.dtype,
216
+ revision: this.revision,
217
+ });
218
+ }
219
+ /** Base mode (Task 1): classic causal LM, unchanged behavior. */
220
+ async loadBaseMode(transformers, device) {
221
+ const { AutoTokenizer, AutoModelForCausalLM } = transformers;
222
+ this.notifyStatus("loading-tokenizer");
223
+ this.tokenizer = await AutoTokenizer.from_pretrained(this.modelId, {
224
+ revision: this.revision,
225
+ });
226
+ this.notifyStatus("loading-model");
227
+ this.model = await AutoModelForCausalLM.from_pretrained(this.modelId, {
228
+ device,
229
+ dtype: this.dtype,
230
+ revision: this.revision,
231
+ });
232
+ }
233
+ async performLoad() {
234
+ this.notifyStatus("loading");
235
+ try {
236
+ const transformers = await import("@huggingface/transformers");
237
+ // LOADER selection is from the loaded CONFIG (`architectures`), never from `modelId`
238
+ // — see `isConditionalGenerationConfig` (Messmotor-Sanierung Task 5, ambiguity
239
+ // resolution 1). This is a loader decision only (which `from_pretrained` class pair
240
+ // to call); the SCORING MODE is decided separately below, from the tokenizer's chat
241
+ // template (Task 6c) — the two must not be conflated. This one extra
242
+ // `AutoConfig.from_pretrained` call runs for every model (base mode included); it
243
+ // hits the same on-disk cache as the tokenizer/model downloads, so the added cost is
244
+ // negligible next to those.
245
+ this.notifyStatus("loading-config");
246
+ const config = await transformers.AutoConfig.from_pretrained(this.modelId, {
247
+ revision: this.revision,
248
+ });
249
+ const useConditionalGenerationLoader = isConditionalGenerationConfig(config);
250
+ const device = await this.resolveDevice();
251
+ if (useConditionalGenerationLoader) {
252
+ await this.loadTemplateMode(transformers, device);
253
+ }
254
+ else {
255
+ await this.loadBaseMode(transformers, device);
256
+ }
257
+ if (!this.tokenizer) {
258
+ throw new Error(`performLoad: tokenizer missing after load for model "${this.modelId}" — cannot determine scoring mode.`);
259
+ }
260
+ // SCORING MODE selection (Messmotor-Sanierung Task 6c): keyed on the loaded
261
+ // tokenizer's chat template, independent of the loader chosen above — see
262
+ // `hasChatTemplate`.
263
+ this.isTemplateMode = hasChatTemplate(this.tokenizer);
264
+ this.notifyStatus("ready");
265
+ }
266
+ catch (err) {
267
+ this.loadError = err instanceof Error ? err : new Error(String(err));
268
+ this.notifyStatus("error");
269
+ throw this.loadError;
270
+ }
271
+ }
272
+ /**
273
+ * Generate text.
274
+ *
275
+ * v4 note: previously this called a `pipeline("text-generation", ...)` built from the
276
+ * already-loaded model/tokenizer, which transformers.js re-resolves/re-loads internally
277
+ * — effectively a second model load that `getScoredLogProb`/`getLogProb` never used
278
+ * (Messmotor-Sanierung Task 4, trap 4). Calling `model.generate()` directly avoids that.
279
+ */
280
+ async generate(prompt, maxTokens = 50) {
281
+ await this.load();
282
+ if (!this.tokenizer || !this.model) {
283
+ throw new Error("Model not loaded");
284
+ }
285
+ const tokenizer = this.tokenizer;
286
+ const model = this.model;
287
+ const { input_ids, attention_mask } = tokenizer(prompt, { return_tensor: true });
288
+ const outputs = await model.generate({
289
+ inputs: input_ids,
290
+ attention_mask,
291
+ max_new_tokens: maxTokens,
292
+ temperature: 0.7,
293
+ do_sample: true,
294
+ });
295
+ // model.generate()'s declared return type is `ModelOutput | Tensor`, but for causal LM
296
+ // decoding (no encoder-decoder, no streamer) it is always a plain Tensor of token ids —
297
+ // narrow via `unknown` rather than `any` since the union has no discriminant we can check.
298
+ const decoded = tokenizer.batch_decode(outputs, {
299
+ skip_special_tokens: true,
300
+ });
301
+ return decoded[0] ?? "";
302
+ }
303
+ /**
304
+ * Score a sentence's log-probability with full provenance.
305
+ * This is the "Gold Standard" for bias testing.
306
+ *
307
+ * BOS fix: GPT-2-family tokenizers do not insert a BOS token on their own, so the
308
+ * sentence is tokenized WITHOUT special tokens and a BOS token is explicitly
309
+ * prepended before feeding the model (input_ids and attention_mask are rebuilt
310
+ * together). This ensures the model sees P(first_token | BOS) instead of silently
311
+ * skipping the first sentence token. BOS itself is context and is never scored;
312
+ * every sentence token is scored via the half-open span [1, n+1).
313
+ */
314
+ async getScoredLogProb(text) {
315
+ await this.load();
316
+ if (!this.tokenizer || !this.model) {
317
+ throw new Error("Model not loaded");
318
+ }
319
+ if (this.isTemplateMode) {
320
+ const libraryVersion = await resolveTransformersVersion();
321
+ return scoreTemplateContinuation({
322
+ tokenizer: this.tokenizer,
323
+ model: this.model,
324
+ modelId: this.modelId,
325
+ libraryVersion,
326
+ dtype: this.dtype,
327
+ }, text);
328
+ }
329
+ const tokenizer = this.tokenizer;
330
+ const model = this.model;
331
+ const bosTokenId = resolveBosTokenId(tokenizer, this.modelId);
332
+ // v4 renamed the tokenizer call's tensor-output option from `return_tensors: "pt"`
333
+ // (a v2 backend-name string) to `return_tensor: true` (a plain boolean) — surfaced by
334
+ // deleting transformers.d.ts; `return_tensors: "pt"` is not a recognized v4 option and
335
+ // would silently fail to type-check (Messmotor-Sanierung Task 4).
336
+ const encoded = tokenizer(text, { return_tensor: true, add_special_tokens: false });
337
+ const inputIdsTensor = encoded.input_ids;
338
+ const [batchSize] = inputIdsTensor.dims;
339
+ if (batchSize !== 1) {
340
+ throw new Error(`getScoredLogProb: batch_size must be 1, got ${batchSize}`);
341
+ }
342
+ const sentenceTokenIds = Array.from(inputIdsTensor.data, (v) => Number(v));
343
+ // Guard against double-BOS if a future tokenizer config already inserts it.
344
+ const finalTokenIds = sentenceTokenIds[0] === bosTokenId ? sentenceTokenIds : [bosTokenId, ...sentenceTokenIds];
345
+ const scoredTokenIds = finalTokenIds.slice(1);
346
+ const { Tensor } = await import("@huggingface/transformers");
347
+ const newLen = finalTokenIds.length;
348
+ const newInputIds = new Tensor("int64", BigInt64Array.from(finalTokenIds.map(BigInt)), [
349
+ 1,
350
+ newLen,
351
+ ]);
352
+ const newAttentionMask = new Tensor("int64", BigInt64Array.from(new Array(newLen).fill(1n)), [
353
+ 1,
354
+ newLen,
355
+ ]);
356
+ const outputs = await model({ input_ids: newInputIds, attention_mask: newAttentionMask });
357
+ const logits = outputs.logits;
358
+ const span = { start: 1, end: newLen };
359
+ const score = sumSequenceLogProb(logits, finalTokenIds, span);
360
+ const contextTokenHash = await sha256Hex(JSON.stringify([bosTokenId]));
361
+ const libraryVersion = await resolveTransformersVersion();
362
+ return {
363
+ score,
364
+ scoredTokenIds,
365
+ span,
366
+ contextTokenHash,
367
+ provenance: {
368
+ modelId: this.modelId,
369
+ dtype: this.dtype,
370
+ logitsDtype: String(logits.type ?? "unknown"),
371
+ backend: "node-onnx",
372
+ libraryVersion,
373
+ templateStrategy: "none",
374
+ },
375
+ method: "sequence_logprob_sum",
376
+ };
377
+ }
378
+ /**
379
+ * Calculate Log-Probability for a given text (score only).
380
+ * Thin wrapper around `getScoredLogProb` for callers that only need the score.
381
+ */
382
+ async getLogProb(text) {
383
+ const { score } = await this.getScoredLogProb(text);
384
+ return score;
385
+ }
386
+ /**
387
+ * Get list of supported browser models
388
+ */
389
+ static getSupportedModels() {
390
+ return BROWSER_MODELS.filter((m) => m.supported);
391
+ }
392
+ }
@@ -84,13 +84,11 @@ export interface CrowsPairsLogProbResult {
84
84
  metricSources: Record<string, Citation[]>;
85
85
  }
86
86
  /**
87
- * CrowS-Pairs-DE (or compatible) test: log-prob difference stereotype vs anti-stereotype.
88
- *
89
- * Reproducibility:
90
- * - deterministic sampling with seed (default 42)
91
- * - modelId + engine recorded
87
+ * @deprecated Verwendet den entfernten Ollama-Logprob-Pfad (via `runStereotypeBiasTest`
88
+ * / `testPair`, and directly built its own `OllamaClient` for methodology detection);
89
+ * siehe bias --engine transformers.
92
90
  */
93
- export declare function runCrowsPairsLogProbTest(client: InferenceClient, pairs: StereotypePair[], options?: CrowsPairsLogProbOptions): Promise<CrowsPairsLogProbResult>;
91
+ export declare function runCrowsPairsLogProbTest(_client: InferenceClient, _pairs: StereotypePair[], _options?: CrowsPairsLogProbOptions): Promise<CrowsPairsLogProbResult>;
94
92
  export interface RegardScorer {
95
93
  /** Returns a score in [-1, 1] (negative -> negative regard, positive -> positive regard) */
96
94
  score(text: string): Promise<number>;
@@ -102,7 +100,7 @@ export interface ToxicityScorer {
102
100
  modelId?: string;
103
101
  }
104
102
  /**
105
- * Create a browser-side toxicity scorer using @xenova/transformers (Detoxify-like classifier).
103
+ * Create a browser-side toxicity scorer using @huggingface/transformers (Detoxify-like classifier).
106
104
  *
107
105
  * Notes:
108
106
  * - Model weights may be downloaded at runtime depending on caching environment.
@@ -1 +1 @@
1
- {"version":3,"file":"bias-metrics.d.ts","sourceRoot":"","sources":["../../src/legal-checks/bias-metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAwB,cAAc,EAAE,MAAM,YAAY,CAAC;AAEjG,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAE/E,MAAM,MAAM,kBAAkB,GAC1B,QAAQ,GACR,WAAW,GACX,KAAK,GACL,YAAY,GACZ,UAAU,GACV,oBAAoB,CAAC;AAEzB,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,eAAO,MAAM,cAAc,EAAE,QAAQ,EAWpC,CAAC;AAEF,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,CAqDrD,CAAC;AAMF,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,cAAc,EAAE,GAAG,cAAc,CAIlF;AAED,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,cAAc,EAAE,GAAG,cAAc,CAG9E;AAED,MAAM,WAAW,kBAAkB;IACjC,iEAAiE;IACjE,EAAE,EAAE,MAAM,CAAC;IACX,kBAAkB,EAAE,kBAAkB,CAAC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,qEAAqE;IACrE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC;IAC9B,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC;CAC/B;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,kBAAkB,EAAE,EAC3B,SAAS,SAAO,GACf,cAAc,CAuBhB;AAED,wBAAgB,4BAA4B,CAAC,MAAM,EAAE;IACnD,IAAI,EAAE,cAAc,EAAE,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,kBAAkB,EAAE,kBAAkB,GAAG,MAAM,CAAC;IAChD,mBAAmB,CAAC,EAAE,kBAAkB,EAAE,CAAC;IAC3C,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC,GAAG,cAAc,GAAG;IAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAA;CAAE,CA2BtF;AAgCD,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACzD;AAED,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3E,WAAW,EAAE,QAAQ,GAAG,eAAe,GAAG,YAAY,CAAC;IACvD,OAAO,EAAE;QACP,gBAAgB,EAAE,MAAM,CAAC;QACzB,2BAA2B,EAAE,MAAM,CAAC;QACpC,gBAAgB,EAAE,MAAM,CAAC;QACzB,MAAM,EAAE,OAAO,CAAC;QAChB,QAAQ,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;KACvC,CAAC;IACF,UAAU,EAAE,KAAK,CAAC;QAChB,QAAQ,EAAE,QAAQ,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,2BAA2B,EAAE,MAAM,CAAC;QACpC,gBAAgB,EAAE,MAAM,CAAC;QACzB,MAAM,EAAE,OAAO,CAAC;QAChB,gBAAgB,EAAE,MAAM,CAAC;QACzB,QAAQ,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;KACvC,CAAC,CAAC;IACH,WAAW,CAAC,EAAE,cAAc,EAAE,CAAC;IAC/B,OAAO,EAAE,QAAQ,EAAE,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;CAC3C;AAED;;;;;;GAMG;AACH,wBAAsB,wBAAwB,CAC5C,MAAM,EAAE,eAAe,EACvB,KAAK,EAAE,cAAc,EAAE,EACvB,OAAO,GAAE,wBAA6B,GACrC,OAAO,CAAC,uBAAuB,CAAC,CA6DlC;AAKD,MAAM,WAAW,YAAY;IAC3B,4FAA4F;IAC5F,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,gEAAgE;IAChE,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;GAMG;AACH,wBAAsB,gCAAgC,CAAC,MAAM,CAAC,EAAE;IAC9D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,QAAQ,GAAG,KAAK,CAAC;CAC3B,GAAG,OAAO,CAAC,cAAc,CAAC,CAqC1B;AAED;;;GAGG;AACH,wBAAsB,8BAA8B,CAAC,MAAM,CAAC,EAAE;IAC5D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,QAAQ,GAAG,KAAK,CAAC;CAC3B,GAAG,OAAO,CAAC,YAAY,CAAC,CA6BxB;AAED;;;GAGG;AACH,wBAAgB,kCAAkC,CAAC,MAAM,EAAE;IACzD,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,cAAc,CAuBjB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,kBAAkB,CAAC;IAC1B,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC,kBAAkB,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/E,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,QAAQ,EAAE,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;CAC3C;AAED,wBAAsB,aAAa,CACjC,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,eAAe,EAAE,EAC1B,eAAe,SAAM,GACpB,OAAO,CAAC,YAAY,CAAC,CAwCvB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,QAAQ,EAAE,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;CAC3C;AAED,wBAAsB,eAAe,CACnC,MAAM,EAAE,cAAc,EACtB,KAAK,EAAE,MAAM,EAAE,EACf,YAAY,SAAM,GACjB,OAAO,CAAC,cAAc,CAAC,CAgCzB"}
1
+ {"version":3,"file":"bias-metrics.d.ts","sourceRoot":"","sources":["../../src/legal-checks/bias-metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC3E,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAE/E,MAAM,MAAM,kBAAkB,GAC1B,QAAQ,GACR,WAAW,GACX,KAAK,GACL,YAAY,GACZ,UAAU,GACV,oBAAoB,CAAC;AAEzB,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,eAAO,MAAM,cAAc,EAAE,QAAQ,EAWpC,CAAC;AAEF,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,CAqDrD,CAAC;AAMF,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,cAAc,EAAE,GAAG,cAAc,CAIlF;AAED,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,cAAc,EAAE,GAAG,cAAc,CAG9E;AAED,MAAM,WAAW,kBAAkB;IACjC,iEAAiE;IACjE,EAAE,EAAE,MAAM,CAAC;IACX,kBAAkB,EAAE,kBAAkB,CAAC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,qEAAqE;IACrE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC;IAC9B,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC;CAC/B;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,kBAAkB,EAAE,EAC3B,SAAS,SAAO,GACf,cAAc,CAuBhB;AAED,wBAAgB,4BAA4B,CAAC,MAAM,EAAE;IACnD,IAAI,EAAE,cAAc,EAAE,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,kBAAkB,EAAE,kBAAkB,GAAG,MAAM,CAAC;IAChD,mBAAmB,CAAC,EAAE,kBAAkB,EAAE,CAAC;IAC3C,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC,GAAG,cAAc,GAAG;IAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAA;CAAE,CA2BtF;AAMD,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACzD;AAED,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3E,WAAW,EAAE,QAAQ,GAAG,eAAe,GAAG,YAAY,CAAC;IACvD,OAAO,EAAE;QACP,gBAAgB,EAAE,MAAM,CAAC;QACzB,2BAA2B,EAAE,MAAM,CAAC;QACpC,gBAAgB,EAAE,MAAM,CAAC;QACzB,MAAM,EAAE,OAAO,CAAC;QAChB,QAAQ,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;KACvC,CAAC;IACF,UAAU,EAAE,KAAK,CAAC;QAChB,QAAQ,EAAE,QAAQ,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,2BAA2B,EAAE,MAAM,CAAC;QACpC,gBAAgB,EAAE,MAAM,CAAC;QACzB,MAAM,EAAE,OAAO,CAAC;QAChB,gBAAgB,EAAE,MAAM,CAAC;QACzB,QAAQ,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;KACvC,CAAC,CAAC;IACH,WAAW,CAAC,EAAE,cAAc,EAAE,CAAC;IAC/B,OAAO,EAAE,QAAQ,EAAE,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;CAC3C;AAED;;;;GAIG;AACH,wBAAsB,wBAAwB,CAC5C,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,cAAc,EAAE,EACxB,QAAQ,GAAE,wBAA6B,GACtC,OAAO,CAAC,uBAAuB,CAAC,CAIlC;AAKD,MAAM,WAAW,YAAY;IAC3B,4FAA4F;IAC5F,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,gEAAgE;IAChE,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;GAMG;AACH,wBAAsB,gCAAgC,CAAC,MAAM,CAAC,EAAE;IAC9D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,QAAQ,GAAG,KAAK,CAAC;CAC3B,GAAG,OAAO,CAAC,cAAc,CAAC,CA+B1B;AAED;;;GAGG;AACH,wBAAsB,8BAA8B,CAAC,MAAM,CAAC,EAAE;IAC5D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,QAAQ,GAAG,KAAK,CAAC;CAC3B,GAAG,OAAO,CAAC,YAAY,CAAC,CAuBxB;AAED;;;GAGG;AACH,wBAAgB,kCAAkC,CAAC,MAAM,EAAE;IACzD,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,cAAc,CAuBjB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,kBAAkB,CAAC;IAC1B,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC,kBAAkB,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/E,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,QAAQ,EAAE,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;CAC3C;AAED,wBAAsB,aAAa,CACjC,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,eAAe,EAAE,EAC1B,eAAe,SAAM,GACpB,OAAO,CAAC,YAAY,CAAC,CAwCvB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,QAAQ,EAAE,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;CAC3C;AAED,wBAAsB,eAAe,CACnC,MAAM,EAAE,cAAc,EACtB,KAAK,EAAE,MAAM,EAAE,EACf,YAAY,SAAM,GACjB,OAAO,CAAC,cAAc,CAAC,CAgCzB"}
@@ -9,7 +9,6 @@
9
9
  * IMPORTANT: This module provides technical orientation, not legal advice.
10
10
  */
11
11
  import { disparateImpact, equalizedOdds, statisticalParityDifference } from "../fairness-metrics";
12
- import { runStereotypeBiasTest } from "../metrics";
13
12
  export const AI_ACT_SOURCES = [
14
13
  { label: "EU AI Act", reference: "Verordnung (EU) 2024/1689" },
15
14
  {
@@ -135,104 +134,16 @@ export function runArticle10NumericBiasSuite(params) {
135
134
  metricSources: METRIC_SOURCES,
136
135
  };
137
136
  }
138
- // ============================================================================
139
- // Text / Generative model metrics (LLM)
140
- // ============================================================================
141
- function mulberry32(seed) {
142
- let state = seed >>> 0;
143
- return () => {
144
- state = (state + 0x6d2b79f5) >>> 0;
145
- let t = state;
146
- t = Math.imul(t ^ (t >>> 15), t | 1);
147
- t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
148
- return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
149
- };
150
- }
151
- function seededSample(arr, n, seed) {
152
- const rng = mulberry32(seed);
153
- const copy = arr.slice();
154
- // Fisher-Yates
155
- for (let i = copy.length - 1; i > 0; i--) {
156
- const j = Math.floor(rng() * (i + 1));
157
- const a = copy[i];
158
- const b = copy[j];
159
- if (a === undefined || b === undefined)
160
- continue;
161
- copy[i] = b;
162
- copy[j] = a;
163
- }
164
- return copy.slice(0, Math.min(n, copy.length));
165
- }
166
137
  /**
167
- * CrowS-Pairs-DE (or compatible) test: log-prob difference stereotype vs anti-stereotype.
168
- *
169
- * Reproducibility:
170
- * - deterministic sampling with seed (default 42)
171
- * - modelId + engine recorded
138
+ * @deprecated Verwendet den entfernten Ollama-Logprob-Pfad (via `runStereotypeBiasTest`
139
+ * / `testPair`, and directly built its own `OllamaClient` for methodology detection);
140
+ * siehe bias --engine transformers.
172
141
  */
173
- export async function runCrowsPairsLogProbTest(client, pairs, options = {}) {
174
- const seed = options.seed ?? 42;
175
- const maxPairs = options.maxPairs ?? pairs.length;
176
- const sampled = seededSample(pairs, maxPairs, seed);
177
- const result = await runStereotypeBiasTest(client, sampled, {
178
- maxPairs: sampled.length,
179
- onProgress: options.onProgress,
180
- });
181
- // Determine methodology based on engine and actual capability
182
- let methodology = "logits";
183
- if (client.engine === "ollama") {
184
- // For Ollama, we need to check if the model actually supports log-probabilities
185
- // The OllamaClient.getLogProb now uses getLogProbWithFallback internally,
186
- // which returns exact log-probs if supported. We can detect this by checking
187
- // if the client has capability detection.
188
- try {
189
- // Import OllamaClient to check capability
190
- const { OllamaClient } = await import("../inference/ollama-client");
191
- const ollamaClient = new OllamaClient(client.modelId);
192
- const supportsLogProbs = await ollamaClient.detectLogProbSupport();
193
- methodology = supportsLogProbs ? "logits" : "latency-proxy";
194
- }
195
- catch {
196
- // Fallback to latency-proxy if we can't detect
197
- methodology = "latency-proxy";
198
- }
199
- }
200
- if (client.engine === "demo")
201
- methodology = "simulation";
202
- return {
203
- engine: client.engine,
204
- modelId: client.modelId,
205
- timestamp: result.timestamp,
206
- dataset: {
207
- name: "CrowS-Pairs-DE (compatible)",
208
- version: "local",
209
- maxPairs: sampled.length,
210
- seed,
211
- },
212
- methodology,
213
- overall: {
214
- averageBiasScore: Number(result.overallBiasScore.toFixed(6)),
215
- stereotypePreferencePercent: Number(result.overallStereotypePreference.toFixed(2)),
216
- thresholdPercent: result.threshold,
217
- passed: result.passed,
218
- severity: result.overallSeverity,
219
- },
220
- byBiasType: result.categoryResults.map((c) => ({
221
- biasType: c.biasType,
222
- pairsCount: c.pairsCount,
223
- stereotypePreferencePercent: Number(c.stereotypePreference.toFixed(2)),
224
- averageBiasScore: Number(c.averageBiasScore.toFixed(6)),
225
- passed: c.passed,
226
- thresholdPercent: c.threshold,
227
- severity: c.severity,
228
- })),
229
- pairResults: result.pairResults,
230
- sources: AI_ACT_SOURCES,
231
- metricSources: METRIC_SOURCES,
232
- };
142
+ export async function runCrowsPairsLogProbTest(_client, _pairs, _options = {}) {
143
+ throw new Error("runCrowsPairsLogProbTest verwendet den entfernten Ollama-Logprob-Pfad; siehe bias --engine transformers.");
233
144
  }
234
145
  /**
235
- * Create a browser-side toxicity scorer using @xenova/transformers (Detoxify-like classifier).
146
+ * Create a browser-side toxicity scorer using @huggingface/transformers (Detoxify-like classifier).
236
147
  *
237
148
  * Notes:
238
149
  * - Model weights may be downloaded at runtime depending on caching environment.
@@ -242,9 +153,7 @@ export async function createTransformersToxicityScorer(params) {
242
153
  const modelId = params?.modelId ?? "Xenova/toxic-bert";
243
154
  const device = params?.device;
244
155
  // Dynamic import to avoid forcing downloads in Node test environments.
245
- // biome-ignore lint/suspicious/noExplicitAny: dynamic import
246
- const { pipeline } = (await import("@xenova/transformers"));
247
- // biome-ignore lint/suspicious/noExplicitAny: library internal
156
+ const { pipeline } = await import("@huggingface/transformers");
248
157
  const clf = await pipeline("text-classification", modelId, device ? { device } : undefined);
249
158
  return {
250
159
  modelId,
@@ -274,9 +183,7 @@ export async function createTransformersToxicityScorer(params) {
274
183
  export async function createTransformersRegardScorer(params) {
275
184
  const modelId = params?.modelId ?? "Xenova/cardiffnlp/twitter-xlm-roberta-base-sentiment";
276
185
  const device = params?.device;
277
- // biome-ignore lint/suspicious/noExplicitAny: dynamic import
278
- const { pipeline } = (await import("@xenova/transformers"));
279
- // biome-ignore lint/suspicious/noExplicitAny: library internal
186
+ const { pipeline } = await import("@huggingface/transformers");
280
187
  const clf = await pipeline("text-classification", modelId, device ? { device } : undefined);
281
188
  return {
282
189
  modelId,
@@ -1,20 +1,28 @@
1
1
  /**
2
2
  * CrowS-Pairs Bias Calculation Engine
3
- * Implements the CrowS-Pairs methodology (Nangia et al., 2020) for measuring stereotypical bias
4
- * Uses log-probabilities as the gold standard with latency fallback when needed
3
+ * Implements the CrowS-Pairs methodology (Nangia et al., 2020) for measuring stereotypical bias.
4
+ *
5
+ * Single measurement engine: any provider passed in must implement
6
+ * `SequenceLogProbProvider.getScoredLogProb`, a real, cross-checked sentence
7
+ * log-probability (see Task 1 / Messmotor-Sanierung). Ollama never measured this
8
+ * (its logprobs cover only generated tokens, not the prompt/sentence), so
9
+ * `OllamaClient` is structurally excluded — it has no `getScoredLogProb` method.
10
+ *
11
+ * Fail-closed by default: a pair that fails measurement aborts the whole run.
12
+ * `config.allowPartial` switches to a diagnostic mode that collects failures
13
+ * into `failedPairs` and reports `runStatus: "partial"` instead.
5
14
  */
6
- import { BrowserInference } from "../inference/browser-inference";
7
- import { OllamaClient } from "../inference/ollama-client";
15
+ import type { SequenceLogProbProvider } from "../inference";
8
16
  import type { BiasTestConfig, CrowsPairsBiasResult, CrowsPairsEntry } from "../types";
9
17
  /**
10
- * Calculate CrowS-Pairs bias score using the specified inference client
18
+ * Calculate CrowS-Pairs bias score using the given sequence-log-probability provider.
11
19
  * Implements the methodology: mean(logprob_stereo - logprob_anti)
12
20
  *
13
- * @param config - Configuration including dataset, model, and engine
14
- * @param inferenceClient - Either BrowserInference or OllamaClient instance
15
- * @returns Promise<CrowsPairsBiasResult> with bias score and metadata
21
+ * @param config - Configuration including dataset, model, engine, and allowPartial
22
+ * @param provider - A `SequenceLogProbProvider` (currently: `TransformersInference`)
23
+ * @returns Promise<CrowsPairsBiasResult> with bias score, metadata, and run status
16
24
  */
17
- export declare function calculateCrowsPairsBias(config: BiasTestConfig, inferenceClient: BrowserInference | OllamaClient): Promise<CrowsPairsBiasResult>;
25
+ export declare function calculateCrowsPairsBias(config: BiasTestConfig, provider: SequenceLogProbProvider): Promise<CrowsPairsBiasResult>;
18
26
  /**
19
27
  * Calculate bias using log-probability method (exact implementation)
20
28
  * This is the gold standard method when log-probabilities are available
@@ -1 +1 @@
1
- {"version":3,"file":"crows-pairs-bias.d.ts","sourceRoot":"","sources":["../../src/metrics/crows-pairs-bias.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,KAAK,EAEV,cAAc,EACd,oBAAoB,EACpB,eAAe,EAChB,MAAM,UAAU,CAAC;AAelB;;;;;;;GAOG;AACH,wBAAsB,uBAAuB,CAC3C,MAAM,EAAE,cAAc,EACtB,eAAe,EAAE,gBAAgB,GAAG,YAAY,GAC/C,OAAO,CAAC,oBAAoB,CAAC,CAiC/B;AA+ED;;;;;;;GAOG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,eAAe,EAAE,EAC1B,UAAU,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,GAChD,OAAO,CAAC,MAAM,EAAE,CAAC,CAiBnB;AAED;;;;;;;GAOG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,eAAe,EAAE,EAC1B,UAAU,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,GAChD,OAAO,CAAC,MAAM,EAAE,CAAC,CAqBnB;AAED;;;GAGG;AACH,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,IAAI,eAAe,EAAE,CAyB1F"}
1
+ {"version":3,"file":"crows-pairs-bias.d.ts","sourceRoot":"","sources":["../../src/metrics/crows-pairs-bias.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,KAAK,EACV,cAAc,EACd,oBAAoB,EAEpB,eAAe,EAIhB,MAAM,UAAU,CAAC;AAoBlB;;;;;;;GAOG;AACH,wBAAsB,uBAAuB,CAC3C,MAAM,EAAE,cAAc,EACtB,QAAQ,EAAE,uBAAuB,GAChC,OAAO,CAAC,oBAAoB,CAAC,CAsD/B;AA0ID;;;;;;;GAOG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,eAAe,EAAE,EAC1B,UAAU,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,GAChD,OAAO,CAAC,MAAM,EAAE,CAAC,CAiBnB;AAED;;;;;;;GAOG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,eAAe,EAAE,EAC1B,UAAU,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,GAChD,OAAO,CAAC,MAAM,EAAE,CAAC,CAqBnB;AAED;;;GAGG;AACH,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,IAAI,eAAe,EAAE,CAyB1F"}