@euconform/core 1.4.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/CHANGELOG.md +46 -0
- package/dist/datasets/crows-pairs.d.ts +18 -0
- package/dist/datasets/crows-pairs.d.ts.map +1 -1
- package/dist/datasets/crows-pairs.js +58 -0
- package/dist/evidence/bias-comparison-markdown.d.ts +11 -0
- package/dist/evidence/bias-comparison-markdown.d.ts.map +1 -0
- package/dist/evidence/bias-comparison-markdown.js +120 -0
- package/dist/evidence/bias-comparison.d.ts +133 -0
- package/dist/evidence/bias-comparison.d.ts.map +1 -0
- package/dist/evidence/bias-comparison.js +274 -0
- package/dist/evidence/bias-statistics.d.ts +72 -0
- package/dist/evidence/bias-statistics.d.ts.map +1 -0
- package/dist/evidence/bias-statistics.js +122 -0
- package/dist/evidence/bias-v2-semantics.d.ts +19 -0
- package/dist/evidence/bias-v2-semantics.d.ts.map +1 -0
- package/dist/evidence/bias-v2-semantics.js +133 -0
- package/dist/evidence/browser.d.ts +18 -0
- package/dist/evidence/browser.d.ts.map +1 -0
- package/dist/evidence/browser.js +14 -0
- package/dist/evidence/bundle.d.ts +1 -0
- package/dist/evidence/bundle.d.ts.map +1 -1
- package/dist/evidence/bundle.js +2 -0
- package/dist/evidence/index.d.ts +9 -2
- package/dist/evidence/index.d.ts.map +1 -1
- package/dist/evidence/index.js +5 -1
- package/dist/evidence/output.d.ts.map +1 -1
- package/dist/evidence/output.js +1 -0
- package/dist/evidence/types.d.ts +183 -3
- package/dist/evidence/types.d.ts.map +1 -1
- package/dist/evidence/validate-bias-v2.d.ts +31 -0
- package/dist/evidence/validate-bias-v2.d.ts.map +1 -0
- package/dist/evidence/validate-bias-v2.js +40 -0
- package/dist/evidence/validate.d.ts +29 -2
- package/dist/evidence/validate.d.ts.map +1 -1
- package/dist/evidence/validate.js +96 -11
- package/dist/fairness-metrics.d.ts +15 -11
- package/dist/fairness-metrics.d.ts.map +1 -1
- package/dist/fairness-metrics.js +59 -183
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/inference/browser-inference.d.ts +6 -57
- package/dist/inference/browser-inference.d.ts.map +1 -1
- package/dist/inference/browser-inference.js +6 -219
- package/dist/inference/capability-detection.d.ts +9 -2
- package/dist/inference/capability-detection.d.ts.map +1 -1
- package/dist/inference/capability-detection.js +18 -16
- package/dist/inference/index.d.ts +15 -2
- package/dist/inference/index.d.ts.map +1 -1
- package/dist/inference/index.js +12 -3
- package/dist/inference/inference-factory.d.ts +6 -3
- package/dist/inference/inference-factory.d.ts.map +1 -1
- package/dist/inference/inference-factory.js +7 -26
- package/dist/inference/ollama-client.d.ts +29 -0
- package/dist/inference/ollama-client.d.ts.map +1 -1
- package/dist/inference/ollama-client.js +29 -0
- package/dist/inference/scored-log-prob-types.d.ts +64 -0
- package/dist/inference/scored-log-prob-types.d.ts.map +1 -0
- package/dist/inference/scored-log-prob-types.js +12 -0
- package/dist/inference/sequence-logprob.d.ts +34 -0
- package/dist/inference/sequence-logprob.d.ts.map +1 -0
- package/dist/inference/sequence-logprob.js +87 -0
- package/dist/inference/template-scoring.d.ts +124 -0
- package/dist/inference/template-scoring.d.ts.map +1 -0
- package/dist/inference/template-scoring.js +164 -0
- package/dist/inference/transformers-inference.d.ts +124 -0
- package/dist/inference/transformers-inference.d.ts.map +1 -0
- package/dist/inference/transformers-inference.js +392 -0
- package/dist/legal-checks/bias-metrics.d.ts +5 -7
- package/dist/legal-checks/bias-metrics.d.ts.map +1 -1
- package/dist/legal-checks/bias-metrics.js +8 -101
- package/dist/metrics/crows-pairs-bias.d.ts +17 -9
- package/dist/metrics/crows-pairs-bias.d.ts.map +1 -1
- package/dist/metrics/crows-pairs-bias.js +147 -59
- package/dist/metrics/stereotype-bias.d.ts +13 -5
- package/dist/metrics/stereotype-bias.d.ts.map +1 -1
- package/dist/metrics/stereotype-bias.js +16 -99
- package/dist/sbom/component-mapper.d.ts +26 -0
- package/dist/sbom/component-mapper.d.ts.map +1 -0
- package/dist/sbom/component-mapper.js +83 -0
- package/dist/sbom/cyclonedx-import.d.ts +34 -0
- package/dist/sbom/cyclonedx-import.d.ts.map +1 -0
- package/dist/sbom/cyclonedx-import.js +131 -0
- package/dist/sbom/index.d.ts +6 -0
- package/dist/sbom/index.d.ts.map +1 -0
- package/dist/sbom/index.js +4 -0
- package/dist/sbom/known-packages.d.ts +13 -0
- package/dist/sbom/known-packages.d.ts.map +1 -0
- package/dist/sbom/known-packages.js +129 -0
- package/dist/sbom/purl.d.ts +14 -0
- package/dist/sbom/purl.d.ts.map +1 -0
- package/dist/sbom/purl.js +56 -0
- package/dist/sbom/types.d.ts +39 -0
- package/dist/sbom/types.d.ts.map +1 -0
- package/dist/sbom/types.js +5 -0
- package/dist/scanner/detectors/ai-framework-detector.js +1 -1
- package/dist/scanner/detectors/local-inference-detector.d.ts.map +1 -1
- package/dist/scanner/detectors/local-inference-detector.js +1 -1
- package/dist/scanner/detectors/package-json-detector.d.ts.map +1 -1
- package/dist/scanner/detectors/package-json-detector.js +1 -0
- package/dist/types.d.ts +103 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/validation/index.d.ts +2 -0
- package/dist/validation/index.d.ts.map +1 -0
- package/dist/validation/index.js +1 -0
- package/dist/validation/schema-validator.d.ts +18 -0
- package/dist/validation/schema-validator.d.ts.map +1 -0
- package/dist/validation/schema-validator.js +105 -0
- package/package.json +9 -3
- package/scripts/seed-logprob-fixture.mjs +179 -0
- package/scripts/ts-extensionless-resolve-hook.mjs +26 -0
- package/src/datasets/crows-pairs.ts +74 -0
- package/src/evidence/bias-comparison-markdown.ts +162 -0
- package/src/evidence/bias-comparison.ts +432 -0
- package/src/evidence/bias-statistics.ts +180 -0
- package/src/evidence/bias-v2-semantics.ts +214 -0
- package/src/evidence/browser.ts +35 -0
- package/src/evidence/bundle.ts +2 -0
- package/src/evidence/index.ts +31 -0
- package/src/evidence/output.ts +1 -0
- package/src/evidence/types.ts +213 -3
- package/src/evidence/validate-bias-v2.ts +47 -0
- package/src/evidence/validate.ts +151 -17
- package/src/fairness-metrics.ts +79 -222
- package/src/index.ts +7 -0
- package/src/inference/browser-inference.ts +14 -256
- package/src/inference/capability-detection.ts +32 -18
- package/src/inference/index.ts +33 -5
- package/src/inference/inference-factory.ts +9 -30
- package/src/inference/ollama-client.ts +29 -0
- package/src/inference/scored-log-prob-types.ts +66 -0
- package/src/inference/sequence-logprob.ts +136 -0
- package/src/inference/template-scoring.ts +248 -0
- package/src/inference/transformers-inference.ts +494 -0
- package/src/legal-checks/bias-metrics.ts +20 -118
- package/src/metrics/crows-pairs-bias.ts +180 -83
- package/src/metrics/stereotype-bias.ts +27 -119
- package/src/sbom/component-mapper.ts +107 -0
- package/src/sbom/cyclonedx-import.ts +198 -0
- package/src/sbom/index.ts +12 -0
- package/src/sbom/known-packages.ts +142 -0
- package/src/sbom/purl.ts +66 -0
- package/src/sbom/types.ts +34 -0
- package/src/scanner/detectors/ai-framework-detector.ts +1 -1
- package/src/scanner/detectors/local-inference-detector.ts +2 -1
- package/src/scanner/detectors/package-json-detector.ts +1 -0
- package/src/types.ts +108 -2
- package/src/validation/index.ts +7 -0
- package/src/validation/schema-validator.ts +147 -0
- package/tests/annex-iv-report.test.ts +1 -1
- package/tests/capability-detection.test.ts +57 -0
- package/tests/crows-pairs-bias.test.ts +286 -186
- package/tests/custom-dataset.test.ts +80 -0
- package/tests/evidence/bias-artifact-v2.test.ts +135 -0
- package/tests/evidence/bias-artifact.test.ts +78 -0
- package/tests/evidence/bias-comparison.test.ts +397 -0
- package/tests/evidence/bias-statistics.test.ts +191 -0
- package/tests/evidence/external-observation.test.ts +167 -0
- package/tests/evidence/spec-schemas.test.ts +35 -2
- package/tests/evidence/validate-bias-v2.test.ts +180 -0
- package/tests/fairness-metrics.test.ts +77 -1
- package/tests/fixtures/cyclonedx-minimal.json +13 -0
- package/tests/fixtures/cyclonedx-mixed.json +17 -0
- package/tests/fixtures/cyclonedx-ml-bom.json +25 -0
- package/tests/inference/browser-inference-scoring.test.ts +159 -0
- package/tests/inference/fixtures/logprob-golden.json +69 -0
- package/tests/inference/gemma-template.integration.test.ts +81 -0
- package/tests/inference/logprob-golden.integration.test.ts +128 -0
- package/tests/inference/synthetic-logits.test.ts +188 -0
- package/tests/inference/template-scoring.test.ts +249 -0
- package/tests/inference/transformers-inference-template-mode.test.ts +214 -0
- package/tests/integration/bias-calculation.integration.test.ts +119 -599
- package/tests/sbom/component-mapper.test.ts +258 -0
- package/tests/sbom/cyclonedx-import.test.ts +329 -0
- package/tests/sbom/purl.test.ts +102 -0
- package/tests/stilled-ollama-paths.test.ts +76 -0
- package/tests/validation/schema-validator.test.ts +219 -0
- package/src/inference/transformers.d.ts +0 -62
|
@@ -1,221 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
2
|
+
* @deprecated Kept for backward compatibility (apps/web, existing tests import
|
|
3
|
+
* `BrowserInference` from this path). The canonical implementation now lives in
|
|
4
|
+
* `transformers-inference.ts` under the name `TransformersInference` — see
|
|
5
|
+
* Messmotor-Sanierung Task 2. New code should import `TransformersInference`
|
|
6
|
+
* from "./transformers-inference" directly.
|
|
6
7
|
*/
|
|
7
|
-
|
|
8
|
-
export const BROWSER_MODELS = [
|
|
9
|
-
{
|
|
10
|
-
id: "Xenova/phi-3-mini-4k-instruct",
|
|
11
|
-
name: "Phi-3 Mini",
|
|
12
|
-
size: "3.8B",
|
|
13
|
-
supported: true,
|
|
14
|
-
},
|
|
15
|
-
{
|
|
16
|
-
id: "Xenova/Phi-3.5-mini-instruct",
|
|
17
|
-
name: "Phi-3.5 Mini",
|
|
18
|
-
size: "3.8B",
|
|
19
|
-
supported: true,
|
|
20
|
-
},
|
|
21
|
-
{
|
|
22
|
-
id: "Xenova/gpt2",
|
|
23
|
-
name: "GPT-2",
|
|
24
|
-
size: "124M",
|
|
25
|
-
supported: true,
|
|
26
|
-
},
|
|
27
|
-
{
|
|
28
|
-
id: "Xenova/distilgpt2",
|
|
29
|
-
name: "DistilGPT-2",
|
|
30
|
-
size: "82M",
|
|
31
|
-
supported: true,
|
|
32
|
-
},
|
|
33
|
-
];
|
|
34
|
-
/**
|
|
35
|
-
* Check if WebGPU is available in the browser
|
|
36
|
-
*/
|
|
37
|
-
export async function checkWebGPU() {
|
|
38
|
-
if (typeof navigator === "undefined")
|
|
39
|
-
return false;
|
|
40
|
-
if (typeof window === "undefined")
|
|
41
|
-
return false;
|
|
42
|
-
try {
|
|
43
|
-
// Check for WebGPU support
|
|
44
|
-
const gpu = navigator.gpu;
|
|
45
|
-
if (!gpu)
|
|
46
|
-
return false;
|
|
47
|
-
// Try to get adapter
|
|
48
|
-
const adapter = await gpu.requestAdapter();
|
|
49
|
-
return adapter !== null;
|
|
50
|
-
}
|
|
51
|
-
catch {
|
|
52
|
-
return false;
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
/**
|
|
56
|
-
* Check if transformers.js is available
|
|
57
|
-
*/
|
|
58
|
-
export async function checkTransformersAvailable() {
|
|
59
|
-
try {
|
|
60
|
-
// Try dynamic import - this will fail if package not installed
|
|
61
|
-
await import("@xenova/transformers");
|
|
62
|
-
return true;
|
|
63
|
-
}
|
|
64
|
-
catch {
|
|
65
|
-
return false;
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
/**
|
|
69
|
-
* Browser inference engine using transformers.js
|
|
70
|
-
* STRICT MODE: Throws error if package not installed.
|
|
71
|
-
*/
|
|
72
|
-
export class BrowserInference {
|
|
73
|
-
modelId;
|
|
74
|
-
model = null;
|
|
75
|
-
tokenizer = null;
|
|
76
|
-
pipeline = null; // Keep pipeline for generation
|
|
77
|
-
loadingPromise = null;
|
|
78
|
-
loadError = null;
|
|
79
|
-
statusCallback = null;
|
|
80
|
-
constructor(modelId = "Xenova/distilgpt2", statusCallback = null) {
|
|
81
|
-
this.modelId = modelId;
|
|
82
|
-
this.statusCallback = statusCallback;
|
|
83
|
-
}
|
|
84
|
-
/**
|
|
85
|
-
* Check if browser inference is available
|
|
86
|
-
*/
|
|
87
|
-
async isAvailable() {
|
|
88
|
-
const hasTransformers = await checkTransformersAvailable();
|
|
89
|
-
return hasTransformers;
|
|
90
|
-
}
|
|
91
|
-
/**
|
|
92
|
-
* Load the model (lazy loading)
|
|
93
|
-
* Handles concurrency with loadingPromise
|
|
94
|
-
*/
|
|
95
|
-
async load() {
|
|
96
|
-
if (this.model && this.tokenizer)
|
|
97
|
-
return;
|
|
98
|
-
if (this.loadingPromise)
|
|
99
|
-
return this.loadingPromise;
|
|
100
|
-
if (this.loadError)
|
|
101
|
-
throw this.loadError;
|
|
102
|
-
this.loadingPromise = this.performLoad().finally(() => {
|
|
103
|
-
this.loadingPromise = null;
|
|
104
|
-
});
|
|
105
|
-
return this.loadingPromise;
|
|
106
|
-
}
|
|
107
|
-
async performLoad() {
|
|
108
|
-
if (this.statusCallback)
|
|
109
|
-
this.statusCallback("loading");
|
|
110
|
-
try {
|
|
111
|
-
const { AutoTokenizer, AutoModelForCausalLM, pipeline } = await import("@xenova/transformers");
|
|
112
|
-
const hasWebGPU = await checkWebGPU();
|
|
113
|
-
const device = hasWebGPU ? "webgpu" : "cpu";
|
|
114
|
-
if (this.statusCallback)
|
|
115
|
-
this.statusCallback("loading-tokenizer");
|
|
116
|
-
this.tokenizer = await AutoTokenizer.from_pretrained(this.modelId);
|
|
117
|
-
if (this.statusCallback)
|
|
118
|
-
this.statusCallback("loading-model");
|
|
119
|
-
// biome-ignore lint/suspicious/noExplicitAny: library internal
|
|
120
|
-
this.model = await AutoModelForCausalLM.from_pretrained(this.modelId, {
|
|
121
|
-
device,
|
|
122
|
-
quantized: true,
|
|
123
|
-
});
|
|
124
|
-
if (this.statusCallback)
|
|
125
|
-
this.statusCallback("loading-pipeline");
|
|
126
|
-
// biome-ignore lint/suspicious/noExplicitAny: library internal
|
|
127
|
-
this.pipeline = await pipeline("text-generation", this.modelId, {
|
|
128
|
-
device,
|
|
129
|
-
model: this.model,
|
|
130
|
-
tokenizer: this.tokenizer,
|
|
131
|
-
});
|
|
132
|
-
if (this.statusCallback)
|
|
133
|
-
this.statusCallback("ready");
|
|
134
|
-
}
|
|
135
|
-
catch (err) {
|
|
136
|
-
this.loadError = err instanceof Error ? err : new Error(String(err));
|
|
137
|
-
if (this.statusCallback)
|
|
138
|
-
this.statusCallback("error");
|
|
139
|
-
throw this.loadError;
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
/**
|
|
143
|
-
* Generate text
|
|
144
|
-
*/
|
|
145
|
-
async generate(prompt, maxTokens = 50) {
|
|
146
|
-
await this.load();
|
|
147
|
-
if (!this.pipeline) {
|
|
148
|
-
throw new Error("Model not loaded");
|
|
149
|
-
}
|
|
150
|
-
// biome-ignore lint/suspicious/noExplicitAny: library internal
|
|
151
|
-
const result = await this.pipeline(prompt, {
|
|
152
|
-
max_new_tokens: maxTokens,
|
|
153
|
-
temperature: 0.7,
|
|
154
|
-
do_sample: true,
|
|
155
|
-
});
|
|
156
|
-
return result[0]?.generated_text || "";
|
|
157
|
-
}
|
|
158
|
-
/**
|
|
159
|
-
* Calculate Log-Probability for a given text
|
|
160
|
-
* This is the "Gold Standard" for bias testing.
|
|
161
|
-
* We calculate the loss (negative log likelihood) of the text.
|
|
162
|
-
*/
|
|
163
|
-
async getLogProb(text) {
|
|
164
|
-
await this.load();
|
|
165
|
-
if (!this.tokenizer || !this.model) {
|
|
166
|
-
throw new Error("Model not loaded");
|
|
167
|
-
}
|
|
168
|
-
// biome-ignore lint/suspicious/noExplicitAny: internal transformers.js types
|
|
169
|
-
const inputs = await this.tokenizer(text, { return_tensors: "pt" });
|
|
170
|
-
const inputIds = inputs.input_ids;
|
|
171
|
-
// biome-ignore lint/suspicious/noExplicitAny: internal transformers.js types
|
|
172
|
-
const outputs = await this.model(inputs);
|
|
173
|
-
const logits = outputs.logits;
|
|
174
|
-
// 3. Compute Log Likelihood
|
|
175
|
-
// We want to calculate P(text) = P(t1) * P(t2|t1) * ...
|
|
176
|
-
// LogProb = sum( log( P(ti | t_<i) ) )
|
|
177
|
-
// logits shape: [batch_size, seq_len, vocab_size]
|
|
178
|
-
// inputIds shape: [batch_size, seq_len]
|
|
179
|
-
// We are interested in the probability of the *next* token given previous tokens.
|
|
180
|
-
// So we look at logits[0, i, :] to predict inputIds[0, i+1]
|
|
181
|
-
const data = logits.data; // Float32Array
|
|
182
|
-
const dims = logits.dims; // [batch, seq, vocab]
|
|
183
|
-
const vocabSize = dims[2];
|
|
184
|
-
const seqLen = dims[1];
|
|
185
|
-
let totalLogProb = 0;
|
|
186
|
-
// Helper to get logit value
|
|
187
|
-
const getLogit = (seqIdx, vocabIdx) => {
|
|
188
|
-
return data[seqIdx * vocabSize + vocabIdx];
|
|
189
|
-
};
|
|
190
|
-
// Helper for softmax/logsoftmax
|
|
191
|
-
// We only need the log_prob of the target token
|
|
192
|
-
// log_softmax(x_i) = x_i - log(sum(exp(x_j)))
|
|
193
|
-
// To avoid overflow: x_i - max(x) - log(sum(exp(x_j - max(x))))
|
|
194
|
-
for (let i = 0; i < seqLen - 1; i++) {
|
|
195
|
-
const targetTokenId = Number(inputIds.data[i + 1]);
|
|
196
|
-
// Calculate max logit for numerical stability
|
|
197
|
-
let maxLogit = Number.NEGATIVE_INFINITY;
|
|
198
|
-
for (let j = 0; j < vocabSize; j++) {
|
|
199
|
-
const val = getLogit(i, j);
|
|
200
|
-
if (val > maxLogit)
|
|
201
|
-
maxLogit = val;
|
|
202
|
-
}
|
|
203
|
-
// Calculate sum of exp
|
|
204
|
-
let sumExp = 0;
|
|
205
|
-
for (let j = 0; j < vocabSize; j++) {
|
|
206
|
-
sumExp += Math.exp(getLogit(i, j) - maxLogit);
|
|
207
|
-
}
|
|
208
|
-
const logSumExp = maxLogit + Math.log(sumExp);
|
|
209
|
-
const targetLogit = getLogit(i, targetTokenId);
|
|
210
|
-
const logProb = targetLogit - logSumExp;
|
|
211
|
-
totalLogProb += logProb;
|
|
212
|
-
}
|
|
213
|
-
return totalLogProb;
|
|
214
|
-
}
|
|
215
|
-
/**
|
|
216
|
-
* Get list of supported browser models
|
|
217
|
-
*/
|
|
218
|
-
static getSupportedModels() {
|
|
219
|
-
return BROWSER_MODELS.filter((m) => m.supported);
|
|
220
|
-
}
|
|
221
|
-
}
|
|
8
|
+
export { TransformersInference as BrowserInference, BROWSER_MODELS, checkWebGPU, checkTransformersAvailable, } from "./transformers-inference";
|
|
@@ -4,9 +4,16 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import type { BiasCalculationMethod } from "../types";
|
|
6
6
|
/**
|
|
7
|
-
* Detection status for model capabilities
|
|
7
|
+
* Detection status for model capabilities.
|
|
8
|
+
*
|
|
9
|
+
* "unverified": the model is declared supported (e.g. it's in the static
|
|
10
|
+
* BROWSER_MODELS list) but has NOT actually been loaded/tested — no request
|
|
11
|
+
* was made, no capability was measured. Distinct from "available", which
|
|
12
|
+
* means a real detection/measurement round confirmed it. Do not report
|
|
13
|
+
* "available" for a model that was never tested (see Messmotor-Sanierung
|
|
14
|
+
* Task 2): the runtime status is what a consumer reads, not source comments.
|
|
8
15
|
*/
|
|
9
|
-
export type CapabilityDetectionStatus = "detecting" | "available" | "unavailable" | "error";
|
|
16
|
+
export type CapabilityDetectionStatus = "detecting" | "available" | "unverified" | "unavailable" | "error";
|
|
10
17
|
/**
|
|
11
18
|
* Model capability information
|
|
12
19
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"capability-detection.d.ts","sourceRoot":"","sources":["../../src/inference/capability-detection.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAEtD
|
|
1
|
+
{"version":3,"file":"capability-detection.d.ts","sourceRoot":"","sources":["../../src/inference/capability-detection.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAEtD;;;;;;;;;GASG;AACH,MAAM,MAAM,yBAAyB,GACjC,WAAW,GACX,WAAW,GACX,YAAY,GACZ,aAAa,GACb,OAAO,CAAC;AAEZ;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,SAAS,GAAG,QAAQ,CAAC;IAC7B,MAAM,EAAE,qBAAqB,CAAC;IAC9B,MAAM,EAAE,yBAAyB,CAAC;IAClC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,OAAO,CAAC;IACnB,gBAAgB,EAAE,OAAO,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,eAAe,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,EAAE,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,gBAAgB,EAAE,OAAO,CAAC;IAC1B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,qBAAqB,EAAE,OAAO,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,2BAA4B,SAAQ,mBAAmB;IACtE,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,eAAe,EAAE,eAAe,CAAC;IACjC,WAAW,EAAE,MAAM,CAAC;CACrB;AACD;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,qBAAqB,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;IACpD,qBAAqB,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAC/F,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,oBAAoB,GAAG,IAAI,CAAC;IAClE,mBAAmB,CAAC,UAAU,EAAE,eAAe,GAAG,IAAI,CAAC;IACvD,UAAU,IAAI,IAAI,CAAC;IACnB,mBAAmB,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;CACnD;AAED;;GAEG;AACH,qBAAa,iCAAkC,YAAW,0BAA0B;IAClF,OAAO,CAAC,KAAK,CAAgD;IAC7D,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAuB;IACzD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAA2B;;IAM5D;;;OAGG;IACG,qBAAqB,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;IAgFzD;;;OAGG;IACG,qBAAqB,CACzB,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,SAAS,GAAG,QAAQ,GAC3B,OAAO,CAAC,eAAe,CAAC;IAuD3B;;OAEG;IACH,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,oBAAoB,GAAG,IAAI;IAQjE;;OAEG;IACH,mBAAmB,CAAC,UAAU,EAAE,eAAe,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI;IAmB3E;;OAEG;IACH,UAAU,IAAI,IAAI;IAgBlB;;OAEG;IACG,mBAAmB,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;IAKvD;;OAEG;IACH,SAAS,CAAC,YAAY,CAAC,UAAU,EAAE,oBAAoB,GAAG,OAAO;IAMjE;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAuB5B;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAS1B;;OAEG;IACH,SAAS,CAAC,oBAAoB,CAAC,YAAY,EAAE,eAAe,EAAE,GAAG,eAAe,EAAE;IAQlF;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,UAAU,EAAE,eAAe,GAAG,MAAM;CAkBtE;AAED;;GAEG;AACH,qBAAa,kCAAmC,SAAQ,iCAAiC;IACvF,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAQ;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAK;IACjC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAQ;IAE5C;;OAEG;IACG,qBAAqB,CACzB,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,SAAS,GAAG,QAAQ,GAC3B,OAAO,CAAC,eAAe,CAAC;IAyB3B;;OAEG;YACW,+BAA+B;IAsC7C;;OAEG;YACW,iCAAiC;IA0C/C;;OAEG;IACG,qBAAqB,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;IA6FzD;;OAEG;IACH,OAAO,CAAC,WAAW;IAkBnB;;OAEG;IACH,OAAO,CAAC,KAAK;IAIb;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAU3B;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAWhC;;OAEG;IACH,OAAO,CAAC,eAAe;IA6BvB;;OAEG;IACH,SAAS,CAAC,YAAY,CAAC,UAAU,EAAE,oBAAoB,GAAG,OAAO;CAclE;AAED;;GAEG;AACH,wBAAgB,gCAAgC,IAAI,0BAA0B,CAE7E"}
|
|
@@ -21,6 +21,11 @@ export class DefaultCapabilityDetectionService {
|
|
|
21
21
|
try {
|
|
22
22
|
// Import browser models dynamically
|
|
23
23
|
const { BROWSER_MODELS } = await import("./browser-inference");
|
|
24
|
+
// These entries are declared from the static BROWSER_MODELS list, not
|
|
25
|
+
// measured — no model has actually been loaded/tested at this point.
|
|
26
|
+
// status: "unverified" (not "available") reflects that honestly; no
|
|
27
|
+
// lastTested since no test ran. A real per-model verification step is
|
|
28
|
+
// future work, not this task's scope.
|
|
24
29
|
for (const model of BROWSER_MODELS) {
|
|
25
30
|
if (!model.supported)
|
|
26
31
|
continue;
|
|
@@ -28,9 +33,8 @@ export class DefaultCapabilityDetectionService {
|
|
|
28
33
|
modelId: model.id,
|
|
29
34
|
engine: "browser",
|
|
30
35
|
method: "logprobs_exact",
|
|
31
|
-
status: "
|
|
36
|
+
status: "unverified",
|
|
32
37
|
recommended: true,
|
|
33
|
-
lastTested: new Date().toISOString(),
|
|
34
38
|
});
|
|
35
39
|
}
|
|
36
40
|
}
|
|
@@ -41,9 +45,8 @@ export class DefaultCapabilityDetectionService {
|
|
|
41
45
|
modelId: "Xenova/distilgpt2",
|
|
42
46
|
engine: "browser",
|
|
43
47
|
method: "logprobs_exact",
|
|
44
|
-
status: "
|
|
48
|
+
status: "unverified",
|
|
45
49
|
recommended: true,
|
|
46
|
-
lastTested: new Date().toISOString(),
|
|
47
50
|
});
|
|
48
51
|
}
|
|
49
52
|
try {
|
|
@@ -95,15 +98,14 @@ export class DefaultCapabilityDetectionService {
|
|
|
95
98
|
if (cached && this.isCacheValid(cached)) {
|
|
96
99
|
return cached.capability;
|
|
97
100
|
}
|
|
98
|
-
//
|
|
101
|
+
// Declared, not measured — no model has been loaded/tested here.
|
|
99
102
|
if (engine === "browser") {
|
|
100
103
|
const capability = {
|
|
101
104
|
modelId,
|
|
102
105
|
engine: "browser",
|
|
103
106
|
method: "logprobs_exact",
|
|
104
|
-
status: "
|
|
107
|
+
status: "unverified",
|
|
105
108
|
recommended: true,
|
|
106
|
-
lastTested: new Date().toISOString(),
|
|
107
109
|
};
|
|
108
110
|
this.setCachedCapability(capability);
|
|
109
111
|
return capability;
|
|
@@ -252,7 +254,9 @@ export class DefaultCapabilityDetectionService {
|
|
|
252
254
|
* Calculate recommendation score for a capability
|
|
253
255
|
*/
|
|
254
256
|
getRecommendationScore(capability) {
|
|
255
|
-
|
|
257
|
+
// "unverified" is a usable, declared-supported model (see the type doc) —
|
|
258
|
+
// only truly broken/unknown states (unavailable, error, detecting) score 0.
|
|
259
|
+
if (capability.status !== "available" && capability.status !== "unverified")
|
|
256
260
|
return 0;
|
|
257
261
|
let score = 0;
|
|
258
262
|
// Browser inference gets highest score
|
|
@@ -283,15 +287,14 @@ export class EnhancedCapabilityDetectionService extends DefaultCapabilityDetecti
|
|
|
283
287
|
if (cached && this.isCacheValid(cached)) {
|
|
284
288
|
return cached.capability;
|
|
285
289
|
}
|
|
286
|
-
//
|
|
290
|
+
// Declared, not measured — no model has been loaded/tested here.
|
|
287
291
|
if (engine === "browser") {
|
|
288
292
|
const capability = {
|
|
289
293
|
modelId,
|
|
290
294
|
engine: "browser",
|
|
291
295
|
method: "logprobs_exact",
|
|
292
|
-
status: "
|
|
296
|
+
status: "unverified",
|
|
293
297
|
recommended: true,
|
|
294
|
-
lastTested: new Date().toISOString(),
|
|
295
298
|
};
|
|
296
299
|
this.setCachedCapability(capability);
|
|
297
300
|
return capability;
|
|
@@ -378,7 +381,8 @@ export class EnhancedCapabilityDetectionService extends DefaultCapabilityDetecti
|
|
|
378
381
|
*/
|
|
379
382
|
async detectAllCapabilities() {
|
|
380
383
|
const capabilities = [];
|
|
381
|
-
// Add all supported browser models dynamically
|
|
384
|
+
// Add all supported browser models dynamically. Declared, not measured —
|
|
385
|
+
// no model has actually been loaded/tested at this point.
|
|
382
386
|
try {
|
|
383
387
|
const { BROWSER_MODELS } = await import("./browser-inference");
|
|
384
388
|
for (const model of BROWSER_MODELS) {
|
|
@@ -388,9 +392,8 @@ export class EnhancedCapabilityDetectionService extends DefaultCapabilityDetecti
|
|
|
388
392
|
modelId: model.id,
|
|
389
393
|
engine: "browser",
|
|
390
394
|
method: "logprobs_exact",
|
|
391
|
-
status: "
|
|
395
|
+
status: "unverified",
|
|
392
396
|
recommended: true,
|
|
393
|
-
lastTested: new Date().toISOString(),
|
|
394
397
|
});
|
|
395
398
|
}
|
|
396
399
|
}
|
|
@@ -401,9 +404,8 @@ export class EnhancedCapabilityDetectionService extends DefaultCapabilityDetecti
|
|
|
401
404
|
modelId: "Xenova/distilgpt2",
|
|
402
405
|
engine: "browser",
|
|
403
406
|
method: "logprobs_exact",
|
|
404
|
-
status: "
|
|
407
|
+
status: "unverified",
|
|
405
408
|
recommended: true,
|
|
406
|
-
lastTested: new Date().toISOString(),
|
|
407
409
|
});
|
|
408
410
|
}
|
|
409
411
|
try {
|
|
@@ -2,8 +2,19 @@
|
|
|
2
2
|
* Inference Factory
|
|
3
3
|
* Auto-detects available inference engines and provides unified interface
|
|
4
4
|
*/
|
|
5
|
-
import { type BrowserModel } from "./browser-inference";
|
|
6
5
|
import { type OllamaModel } from "./ollama-client";
|
|
6
|
+
import { type BrowserModel, type ScoredLogProb } from "./transformers-inference";
|
|
7
|
+
/**
|
|
8
|
+
* The one contract a bias-measurement engine must satisfy: a real, cross-checked
|
|
9
|
+
* sentence log-probability (see Task 1 / Messmotor-Sanierung). `OllamaClient`
|
|
10
|
+
* deliberately does NOT implement this — Ollama's logprobs only cover generated
|
|
11
|
+
* tokens, never prompt/sentence log-probability, so it cannot honestly produce a
|
|
12
|
+
* `ScoredLogProb`. TypeScript's structural typing then locks it out of any API
|
|
13
|
+
* that requires this interface (e.g. `calculateCrowsPairsBias`).
|
|
14
|
+
*/
|
|
15
|
+
export interface SequenceLogProbProvider {
|
|
16
|
+
getScoredLogProb(text: string): Promise<ScoredLogProb>;
|
|
17
|
+
}
|
|
7
18
|
export type InferenceEngine = "ollama" | "browser" | "demo";
|
|
8
19
|
export interface InferenceCapabilities {
|
|
9
20
|
ollama: {
|
|
@@ -36,7 +47,9 @@ export declare function createInferenceClient(engine: InferenceEngine, modelId:
|
|
|
36
47
|
*/
|
|
37
48
|
export declare function createAutoInferenceClient(preferredModel?: string): Promise<InferenceClient>;
|
|
38
49
|
export { detectOllama, listOllamaModels, OllamaClient, type CapabilityCache, } from "./ollama-client";
|
|
39
|
-
export {
|
|
50
|
+
export { TransformersInference, checkWebGPU, BROWSER_MODELS, type ScoredLogProb, type ScoredLogProbProvenance, } from "./transformers-inference";
|
|
51
|
+
export { isConditionalGenerationConfig, type MinimalModelConfig } from "./template-scoring";
|
|
52
|
+
export { BrowserInference } from "./browser-inference";
|
|
40
53
|
export { InferenceFactory, createInferenceFactory, detectBestMethodForConfig, type InferenceResult, } from "./inference-factory";
|
|
41
54
|
export { createCapabilityDetectionService, DefaultCapabilityDetectionService, EnhancedCapabilityDetectionService, type CapabilityDetectionService, type ModelCapability, type ModelSelectionState, type CapabilityCacheEntry, type DetectionProgress, type UserPreferences, type EnhancedModelSelectionState, type CapabilityDetectionStatus, } from "./capability-detection";
|
|
42
55
|
export type { OllamaModel, BrowserModel };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/inference/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/inference/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAgB,KAAK,WAAW,EAAkC,MAAM,iBAAiB,CAAC;AACjG,OAAO,EAEL,KAAK,YAAY,EACjB,KAAK,aAAa,EAGnB,MAAM,0BAA0B,CAAC;AAElC;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;CACxD;AAED,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG,SAAS,GAAG,MAAM,CAAC;AAE5D,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE;QACN,SAAS,EAAE,OAAO,CAAC;QACnB,MAAM,EAAE,WAAW,EAAE,CAAC;KACvB,CAAC;IACF,OAAO,EAAE;QACP,SAAS,EAAE,OAAO,CAAC;QACnB,MAAM,EAAE,OAAO,CAAC;QAChB,MAAM,EAAE,YAAY,EAAE,CAAC;KACxB,CAAC;IACF,WAAW,EAAE,eAAe,CAAC;CAC9B;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,eAAe,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1C,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC7C;AAED;;GAEG;AACH,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,qBAAqB,CAAC,CA2BzE;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,eAAe,EACvB,OAAO,EAAE,MAAM,EACf,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,GACxC,eAAe,CAqCjB;AAED;;GAEG;AACH,wBAAsB,yBAAyB,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAejG;AAGD,OAAO,EACL,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACZ,KAAK,eAAe,GACrB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,qBAAqB,EACrB,WAAW,EACX,cAAc,EACd,KAAK,aAAa,EAClB,KAAK,uBAAuB,GAC7B,MAAM,0BAA0B,CAAC;AAOlC,OAAO,EAAE,6BAA6B,EAAE,KAAK,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAE5F,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EACL,gBAAgB,EAChB,sBAAsB,EACtB,yBAAyB,EACzB,KAAK,eAAe,GACrB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,gCAAgC,EAChC,iCAAiC,EACjC,kCAAkC,EAClC,KAAK,0BAA0B,EAC/B,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,oBAAoB,EACzB,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,2BAA2B,EAChC,KAAK,yBAAyB,GAC/B,MAAM,wBAAwB,CAAC;AAChC,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC"}
|
package/dist/inference/index.js
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
* Inference Factory
|
|
3
3
|
* Auto-detects available inference engines and provides unified interface
|
|
4
4
|
*/
|
|
5
|
-
import { BROWSER_MODELS, BrowserInference, checkWebGPU, } from "./browser-inference";
|
|
6
5
|
import { OllamaClient, detectOllama, listOllamaModels } from "./ollama-client";
|
|
6
|
+
import { BROWSER_MODELS, TransformersInference, checkWebGPU, } from "./transformers-inference";
|
|
7
7
|
/**
|
|
8
8
|
* Detect all available inference capabilities
|
|
9
9
|
*/
|
|
@@ -49,7 +49,7 @@ export function createInferenceClient(engine, modelId, statusCallback) {
|
|
|
49
49
|
};
|
|
50
50
|
}
|
|
51
51
|
case "browser": {
|
|
52
|
-
const browserClient = new
|
|
52
|
+
const browserClient = new TransformersInference(modelId, statusCallback);
|
|
53
53
|
return {
|
|
54
54
|
engine: "browser",
|
|
55
55
|
modelId,
|
|
@@ -92,6 +92,15 @@ export async function createAutoInferenceClient(preferredModel) {
|
|
|
92
92
|
}
|
|
93
93
|
// Re-export types and utilities
|
|
94
94
|
export { detectOllama, listOllamaModels, OllamaClient, } from "./ollama-client";
|
|
95
|
-
export {
|
|
95
|
+
export { TransformersInference, checkWebGPU, BROWSER_MODELS, } from "./transformers-inference";
|
|
96
|
+
// Architecture detection for the Gemma-4 template-mode scorer, re-exported so callers that
|
|
97
|
+
// must pick a dtype BEFORE constructing `TransformersInference` (e.g. the CLI's
|
|
98
|
+
// `run-bias-test.ts`, which needs to choose "q4" over "fp32" for conditional-generation
|
|
99
|
+
// models — fp32 Gemma-4 weights were measured and rejected as impractically large in
|
|
100
|
+
// Messmotor-Sanierung Task 5) can reuse the same detection `TransformersInference` uses
|
|
101
|
+
// internally, instead of duplicating the architecture check.
|
|
102
|
+
export { isConditionalGenerationConfig } from "./template-scoring";
|
|
103
|
+
// Backward-compat alias: existing consumers (apps/web, tests) import `BrowserInference`.
|
|
104
|
+
export { BrowserInference } from "./browser-inference";
|
|
96
105
|
export { InferenceFactory, createInferenceFactory, detectBestMethodForConfig, } from "./inference-factory";
|
|
97
106
|
export { createCapabilityDetectionService, DefaultCapabilityDetectionService, EnhancedCapabilityDetectionService, } from "./capability-detection";
|
|
@@ -31,10 +31,13 @@ export declare class InferenceFactory {
|
|
|
31
31
|
*/
|
|
32
32
|
setSelectedCapability(capability: ModelCapability): void;
|
|
33
33
|
/**
|
|
34
|
-
*
|
|
35
|
-
*
|
|
34
|
+
* @deprecated Uses the removed Ollama logprob path: this method auto-selected
|
|
35
|
+
* between browser and Ollama engines via `getInferenceClient`, which no longer
|
|
36
|
+
* measures anything honest for Ollama (see Messmotor-Sanierung Task 2). The
|
|
37
|
+
* auto-selection logic itself needs a real redesign, not a throw-or-convert
|
|
38
|
+
* fix — see bias --engine transformers for the supported path.
|
|
36
39
|
*/
|
|
37
|
-
createBiasCalculation(
|
|
40
|
+
createBiasCalculation(_config: BiasTestConfig): Promise<InferenceResult<CrowsPairsBiasResult>>;
|
|
38
41
|
/**
|
|
39
42
|
* Detect the best available calculation method for the given client
|
|
40
43
|
* Returns the method that should be used for bias calculations
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"inference-factory.d.ts","sourceRoot":"","sources":["../../src/inference/inference-factory.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"inference-factory.d.ts","sourceRoot":"","sources":["../../src/inference/inference-factory.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,qBAAqB,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAC5F,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,CAAC;IAChC,IAAI,EAAE,CAAC,CAAC;IACR,MAAM,EAAE,qBAAqB,CAAC;IAC9B,MAAM,EAAE,SAAS,GAAG,QAAQ,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,eAAe,CAAC;CAC9B;AAED;;;GAGG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,aAAa,CAAiC;IACtD,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,cAAc,CAA2C;IACjE,OAAO,CAAC,kBAAkB,CAAgC;gBAE9C,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI;IAIrD;;OAEG;IACH,qBAAqB,CAAC,UAAU,EAAE,eAAe,GAAG,IAAI;IAIxD;;;;;;OAMG;IACG,qBAAqB,CACzB,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAC;IAMjD;;;OAGG;YACW,gBAAgB;IAe9B;;OAEG;IACH,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB;IAOjD;;OAEG;IACH,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY;IAO5C,OAAO,CAAC,qBAAqB;YAgBf,kBAAkB;IAUhC;;;OAGG;IACG,qBAAqB,CAAC,CAAC,EAC3B,IAAI,EAAE,CAAC,EACP,MAAM,EAAE,SAAS,GAAG,QAAQ,EAC5B,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,gBAAgB,GAAG,YAAY,GACvC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IA8B9B;;OAEG;IACH,OAAO,IAAI,IAAI;CAMhB;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CACpC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,GACxC,gBAAgB,CAElB;AAED;;;GAGG;AACH,wBAAsB,yBAAyB,CAC7C,MAAM,EAAE,cAAc,GACrB,OAAO,CAAC,qBAAqB,CAAC,CAYhC"}
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
* Inference Factory with Method Detection
|
|
3
3
|
* Provides unified interface for bias calculations across engines with automatic method detection
|
|
4
4
|
*/
|
|
5
|
-
import { calculateCrowsPairsBias } from "../metrics/crows-pairs-bias";
|
|
6
5
|
import { BrowserInference } from "./browser-inference";
|
|
7
6
|
import { OllamaClient } from "./ollama-client";
|
|
8
7
|
/**
|
|
@@ -24,32 +23,14 @@ export class InferenceFactory {
|
|
|
24
23
|
this.selectedCapability = capability;
|
|
25
24
|
}
|
|
26
25
|
/**
|
|
27
|
-
*
|
|
28
|
-
*
|
|
26
|
+
* @deprecated Uses the removed Ollama logprob path: this method auto-selected
|
|
27
|
+
* between browser and Ollama engines via `getInferenceClient`, which no longer
|
|
28
|
+
* measures anything honest for Ollama (see Messmotor-Sanierung Task 2). The
|
|
29
|
+
* auto-selection logic itself needs a real redesign, not a throw-or-convert
|
|
30
|
+
* fix — see bias --engine transformers for the supported path.
|
|
29
31
|
*/
|
|
30
|
-
async createBiasCalculation(
|
|
31
|
-
|
|
32
|
-
this.validateConfiguration(config);
|
|
33
|
-
const inferenceClient = await this.getInferenceClient(engine, model);
|
|
34
|
-
let _detectedMethod;
|
|
35
|
-
if (engine === "browser") {
|
|
36
|
-
_detectedMethod = "logprobs_exact";
|
|
37
|
-
}
|
|
38
|
-
else {
|
|
39
|
-
// Use capability method if available, otherwise detect
|
|
40
|
-
_detectedMethod =
|
|
41
|
-
this.selectedCapability?.method ?? (await this.detectBestMethod(inferenceClient));
|
|
42
|
-
}
|
|
43
|
-
// Perform bias calculation
|
|
44
|
-
const biasResult = await calculateCrowsPairsBias(config, inferenceClient);
|
|
45
|
-
// Return result with metadata including capability information
|
|
46
|
-
return {
|
|
47
|
-
data: biasResult,
|
|
48
|
-
method: biasResult.method, // Use the method determined during calculation
|
|
49
|
-
engine,
|
|
50
|
-
model,
|
|
51
|
-
capability: this.selectedCapability || undefined,
|
|
52
|
-
};
|
|
32
|
+
async createBiasCalculation(_config) {
|
|
33
|
+
throw new Error("InferenceFactory.createBiasCalculation verwendet den entfernten Ollama-Logprob-Pfad; siehe bias --engine transformers.");
|
|
53
34
|
}
|
|
54
35
|
/**
|
|
55
36
|
* Detect the best available calculation method for the given client
|
|
@@ -89,9 +89,18 @@ export declare class OllamaClient {
|
|
|
89
89
|
ensureModelLoaded(): Promise<void>;
|
|
90
90
|
isAvailable(): Promise<boolean>;
|
|
91
91
|
generate(prompt: string): Promise<string>;
|
|
92
|
+
/**
|
|
93
|
+
* @deprecated Not a sentence log-probability — see `getLogProbWithFallback`.
|
|
94
|
+
*/
|
|
92
95
|
getLogProb(prompt: string): Promise<number>;
|
|
93
96
|
getPerplexity(text: string): Promise<number>;
|
|
94
97
|
/**
|
|
98
|
+
* @deprecated Part of the removed Ollama bias-measurement path (Messmotor-Sanierung
|
|
99
|
+
* Task 2): Ollama's logprobs cover only generated tokens, never the prompt/sentence,
|
|
100
|
+
* so this "capability" check never verified a real measurement. Kept only because
|
|
101
|
+
* `scan`'s non-bias paths and the capability-cache tests still reference it — not
|
|
102
|
+
* a foundation to build on. See bias --engine transformers for the real path.
|
|
103
|
+
*
|
|
95
104
|
* Test if the current model supports log-probabilities
|
|
96
105
|
* Sends a test request with "Der" prompt to detect capability
|
|
97
106
|
*/
|
|
@@ -105,13 +114,33 @@ export declare class OllamaClient {
|
|
|
105
114
|
*/
|
|
106
115
|
private setCachedCapability;
|
|
107
116
|
/**
|
|
117
|
+
* @deprecated Part of the removed Ollama bias-measurement path (see
|
|
118
|
+
* `testLogProbCapability`); no longer used by `calculateCrowsPairsBias`.
|
|
119
|
+
*
|
|
108
120
|
* Detect if the current model supports log-probabilities
|
|
109
121
|
* Checks cache first, then runs test if needed
|
|
110
122
|
*/
|
|
111
123
|
detectLogProbSupport(): Promise<boolean>;
|
|
124
|
+
/**
|
|
125
|
+
* @deprecated Part of the removed Ollama bias-measurement path; see
|
|
126
|
+
* `testLogProbCapability`. Not a real sentence log-probability.
|
|
127
|
+
*/
|
|
112
128
|
private tryExactLogProbs;
|
|
129
|
+
/**
|
|
130
|
+
* @deprecated Part of the removed Ollama bias-measurement path; this is a
|
|
131
|
+
* latency proxy, not a probability — the exact thing Messmotor-Sanierung
|
|
132
|
+
* removes from the bias measurement path. Do not build on this.
|
|
133
|
+
*/
|
|
113
134
|
private latencyFallback;
|
|
114
135
|
/**
|
|
136
|
+
* @deprecated Removed from the bias-measurement path (Messmotor-Sanierung Task 2):
|
|
137
|
+
* Ollama returns logprobs only for generated tokens, never for the prompt/sentence
|
|
138
|
+
* being scored, so this never measured a sentence log-probability — even the
|
|
139
|
+
* "exact" branch (`tryExactLogProbs`) is not what its name implies, and the
|
|
140
|
+
* fallback is a latency proxy. Verified: r(sentence length, logP)=0.019 where a
|
|
141
|
+
* real sentence-probability measure shows −0.96. Use `TransformersInference.
|
|
142
|
+
* getScoredLogProb` (via `bias --engine transformers`) instead.
|
|
143
|
+
*
|
|
115
144
|
* Get log-probability with automatic fallback to latency method
|
|
116
145
|
* Returns both the calculated value and the method used
|
|
117
146
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ollama-client.d.ts","sourceRoot":"","sources":["../../src/inference/ollama-client.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,qBAAqB,EAAwB,MAAM,UAAU,CAAC;AAE5E,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC5B,OAAO,CAAC,EAAE;QACR,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,KAAK,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,sBAAuB,SAAQ,sBAAsB;IACpE,QAAQ,CAAC,EAAE,KAAK,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;CACJ;AAED,sFAAsF;AACtF,MAAM,WAAW,eAAe;IAC9B,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IAChC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACvC;AAID;;GAEG;AACH,wBAAsB,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC,CAcrD;AAED;;GAEG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAc/D;AAED;;GAEG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,qBAAqB,GAC7B,OAAO,CAAC,sBAAsB,CAAC,CAkBjC;AAED;;;GAGG;AACH,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CA+C/E;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAiChF;AAED;;GAEG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,OAAO,CAAS;IACxB,SAAgB,KAAK,EAAE,MAAM,CAAC;IAC9B,OAAO,CAAC,eAAe,CAAoC;IAC3D,OAAO,CAAC,aAAa,CAAyB;gBAElC,KAAK,EAAE,MAAM,EAAE,OAAO,SAAkB,EAAE,KAAK,CAAC,EAAE,eAAe;IAO7E,OAAO,CAAC,YAAY;IAIpB;;;OAGG;IACG,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC;IAiBlC,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAI/B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"ollama-client.d.ts","sourceRoot":"","sources":["../../src/inference/ollama-client.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,qBAAqB,EAAwB,MAAM,UAAU,CAAC;AAE5E,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC5B,OAAO,CAAC,EAAE;QACR,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,KAAK,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,sBAAuB,SAAQ,sBAAsB;IACpE,QAAQ,CAAC,EAAE,KAAK,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;CACJ;AAED,sFAAsF;AACtF,MAAM,WAAW,eAAe;IAC9B,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IAChC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACvC;AAID;;GAEG;AACH,wBAAsB,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC,CAcrD;AAED;;GAEG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAc/D;AAED;;GAEG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,qBAAqB,GAC7B,OAAO,CAAC,sBAAsB,CAAC,CAkBjC;AAED;;;GAGG;AACH,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CA+C/E;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAiChF;AAED;;GAEG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,OAAO,CAAS;IACxB,SAAgB,KAAK,EAAE,MAAM,CAAC;IAC9B,OAAO,CAAC,eAAe,CAAoC;IAC3D,OAAO,CAAC,aAAa,CAAyB;gBAElC,KAAK,EAAE,MAAM,EAAE,OAAO,SAAkB,EAAE,KAAK,CAAC,EAAE,eAAe;IAO7E,OAAO,CAAC,YAAY;IAIpB;;;OAGG;IACG,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC;IAiBlC,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAI/B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAS/C;;OAEG;IACG,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAM3C,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAIlD;;;;;;;;;OASG;YACW,qBAAqB;IAmEnC;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA0B3B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAqB3B;;;;;;OAMG;IACG,oBAAoB,IAAI,OAAO,CAAC,OAAO,CAAC;IAqB9C;;;OAGG;YACW,gBAAgB;IAgD9B;;;;OAIG;YACW,eAAe;IAyC7B;;;;;;;;;;;OAWG;IACG,sBAAsB,CAC1B,MAAM,EAAE,MAAM,GACb,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,qBAAqB,CAAA;KAAE,CAAC;CAU/D"}
|