@euconform/core 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +4 -0
- package/LICENSE +21 -0
- package/dist/fairness-metrics.d.ts +65 -0
- package/dist/fairness-metrics.d.ts.map +1 -0
- package/dist/fairness-metrics.js +394 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +17 -0
- package/dist/inference/browser-inference.d.ts +60 -0
- package/dist/inference/browser-inference.d.ts.map +1 -0
- package/dist/inference/browser-inference.js +221 -0
- package/dist/inference/capability-detection.d.ts +183 -0
- package/dist/inference/capability-detection.d.ts.map +1 -0
- package/dist/inference/capability-detection.js +552 -0
- package/dist/inference/index.d.ts +43 -0
- package/dist/inference/index.d.ts.map +1 -0
- package/dist/inference/index.js +97 -0
- package/dist/inference/inference-factory.d.ts +73 -0
- package/dist/inference/inference-factory.d.ts.map +1 -0
- package/dist/inference/inference-factory.js +175 -0
- package/dist/inference/ollama-client.d.ts +114 -0
- package/dist/inference/ollama-client.d.ts.map +1 -0
- package/dist/inference/ollama-client.js +386 -0
- package/dist/legal-checks/annex-iv-report.d.ts +131 -0
- package/dist/legal-checks/annex-iv-report.d.ts.map +1 -0
- package/dist/legal-checks/annex-iv-report.js +62 -0
- package/dist/legal-checks/bias-metrics.d.ts +162 -0
- package/dist/legal-checks/bias-metrics.d.ts.map +1 -0
- package/dist/legal-checks/bias-metrics.js +408 -0
- package/dist/legal-checks/data-governance.d.ts +30 -0
- package/dist/legal-checks/data-governance.d.ts.map +1 -0
- package/dist/legal-checks/data-governance.js +99 -0
- package/dist/legal-checks/human-oversight-logging.d.ts +27 -0
- package/dist/legal-checks/human-oversight-logging.d.ts.map +1 -0
- package/dist/legal-checks/human-oversight-logging.js +106 -0
- package/dist/legal-checks/index.d.ts +13 -0
- package/dist/legal-checks/index.d.ts.map +1 -0
- package/dist/legal-checks/index.js +6 -0
- package/dist/legal-checks/risk-classifier.d.ts +35 -0
- package/dist/legal-checks/risk-classifier.d.ts.map +1 -0
- package/dist/legal-checks/risk-classifier.js +234 -0
- package/dist/legal-checks/robustness-transparency.d.ts +45 -0
- package/dist/legal-checks/robustness-transparency.d.ts.map +1 -0
- package/dist/legal-checks/robustness-transparency.js +111 -0
- package/dist/metrics/crows-pairs-bias.d.ts +41 -0
- package/dist/metrics/crows-pairs-bias.d.ts.map +1 -0
- package/dist/metrics/crows-pairs-bias.js +172 -0
- package/dist/metrics/index.d.ts +6 -0
- package/dist/metrics/index.d.ts.map +1 -0
- package/dist/metrics/index.js +5 -0
- package/dist/metrics/stereotype-bias.d.ts +89 -0
- package/dist/metrics/stereotype-bias.d.ts.map +1 -0
- package/dist/metrics/stereotype-bias.js +157 -0
- package/dist/risk-engine.d.ts +23 -0
- package/dist/risk-engine.d.ts.map +1 -0
- package/dist/risk-engine.js +451 -0
- package/dist/types.d.ts +131 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +5 -0
- package/package.json +28 -0
- package/src/fairness-metrics.ts +498 -0
- package/src/index.ts +150 -0
- package/src/inference/browser-inference.ts +258 -0
- package/src/inference/capability-detection.ts +714 -0
- package/src/inference/index.ts +156 -0
- package/src/inference/inference-factory.ts +222 -0
- package/src/inference/ollama-client.ts +514 -0
- package/src/inference/transformers.d.ts +62 -0
- package/src/legal-checks/annex-iv-report.ts +157 -0
- package/src/legal-checks/bias-metrics.ts +590 -0
- package/src/legal-checks/data-governance.ts +148 -0
- package/src/legal-checks/human-oversight-logging.ts +135 -0
- package/src/legal-checks/index.ts +55 -0
- package/src/legal-checks/risk-classifier.ts +285 -0
- package/src/legal-checks/robustness-transparency.ts +162 -0
- package/src/metrics/crows-pairs-bias.ts +245 -0
- package/src/metrics/index.ts +22 -0
- package/src/metrics/stereotype-bias.ts +267 -0
- package/src/risk-engine.ts +525 -0
- package/src/types.ts +154 -0
- package/tests/annex-iv-report.test.ts +323 -0
- package/tests/crows-pairs-bias.test.ts +342 -0
- package/tests/fairness-metrics.test.ts +187 -0
- package/tests/integration/bias-calculation.integration.test.ts +688 -0
- package/tests/legal-checks.test.ts +53 -0
- package/tests/ollama-client.test.ts +427 -0
- package/tests/risk-engine.test.ts +149 -0
- package/tsconfig.json +11 -0
- package/vitest.config.ts +17 -0
|
@@ -0,0 +1,386 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ollama Client for local LLM inference
|
|
3
|
+
* Connects to Ollama running on localhost:11434
|
|
4
|
+
*/
|
|
5
|
+
const OLLAMA_BASE_URL = "http://localhost:11434";
|
|
6
|
+
/**
|
|
7
|
+
* Check if Ollama is running and accessible
|
|
8
|
+
*/
|
|
9
|
+
export async function detectOllama() {
|
|
10
|
+
try {
|
|
11
|
+
const controller = new AbortController();
|
|
12
|
+
const timeoutId = setTimeout(() => controller.abort(), 2000);
|
|
13
|
+
const response = await fetch(`${OLLAMA_BASE_URL}/api/tags`, {
|
|
14
|
+
signal: controller.signal,
|
|
15
|
+
});
|
|
16
|
+
clearTimeout(timeoutId);
|
|
17
|
+
return response.ok;
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Get list of available models from Ollama
|
|
25
|
+
*/
|
|
26
|
+
export async function listOllamaModels() {
|
|
27
|
+
try {
|
|
28
|
+
const response = await fetch(`${OLLAMA_BASE_URL}/api/tags`);
|
|
29
|
+
if (!response.ok) {
|
|
30
|
+
throw new Error(`Failed to list models: ${response.statusText}`);
|
|
31
|
+
}
|
|
32
|
+
const data = await response.json();
|
|
33
|
+
return data.models || [];
|
|
34
|
+
}
|
|
35
|
+
catch (error) {
|
|
36
|
+
console.error("Failed to list Ollama models:", error);
|
|
37
|
+
return [];
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Generate text with Ollama
|
|
42
|
+
*/
|
|
43
|
+
export async function ollamaGenerate(options) {
|
|
44
|
+
const response = await fetch(`${OLLAMA_BASE_URL}/api/generate`, {
|
|
45
|
+
method: "POST",
|
|
46
|
+
headers: { "Content-Type": "application/json" },
|
|
47
|
+
body: JSON.stringify({
|
|
48
|
+
...options,
|
|
49
|
+
stream: false,
|
|
50
|
+
}),
|
|
51
|
+
});
|
|
52
|
+
if (!response.ok) {
|
|
53
|
+
const errorText = await response.text();
|
|
54
|
+
throw new Error(`Ollama generate failed (${response.status}): ${errorText || response.statusText}`);
|
|
55
|
+
}
|
|
56
|
+
return response.json();
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Get log probability for a prompt (for bias scoring)
|
|
60
|
+
* Uses the /api/generate endpoint with num_predict: 0 to get prompt logprobs
|
|
61
|
+
*/
|
|
62
|
+
export async function getLogProb(model, prompt) {
|
|
63
|
+
try {
|
|
64
|
+
// Use generate with num_predict: 0 to ONLY evaluate the prompt
|
|
65
|
+
// This forces the model to process the input tokens, giving us prompt_eval metrics
|
|
66
|
+
const response = await fetch(`${OLLAMA_BASE_URL}/api/generate`, {
|
|
67
|
+
method: "POST",
|
|
68
|
+
headers: { "Content-Type": "application/json" },
|
|
69
|
+
body: JSON.stringify({
|
|
70
|
+
model,
|
|
71
|
+
prompt,
|
|
72
|
+
stream: false,
|
|
73
|
+
options: {
|
|
74
|
+
num_predict: 0, // Don't generate, just evaluate prompt
|
|
75
|
+
temperature: 0,
|
|
76
|
+
},
|
|
77
|
+
}),
|
|
78
|
+
});
|
|
79
|
+
if (!response.ok) {
|
|
80
|
+
const errorText = await response.text();
|
|
81
|
+
throw new Error(`Failed to get logprob for model '${model}': ${errorText || response.statusText}`);
|
|
82
|
+
}
|
|
83
|
+
const data = await response.json();
|
|
84
|
+
// Optimization: Use prompt_eval metrics (processing time of the input)
|
|
85
|
+
// This is a better proxy for "perplexity" or "likelihood" of the sentence than generation speed
|
|
86
|
+
if (data.prompt_eval_duration && data.prompt_eval_count) {
|
|
87
|
+
const timePerToken = data.prompt_eval_duration / data.prompt_eval_count / 1e9; // ns to s
|
|
88
|
+
// Normalize: faster processing = higher probability (lower surprise)
|
|
89
|
+
// We return negative log of time to mimic log-prob scale (higher is better)
|
|
90
|
+
return -Math.log(timePerToken + 0.00001); // Smaller epsilon for precision
|
|
91
|
+
}
|
|
92
|
+
// Fallback: use eval metrics if prompt metrics missing (rare)
|
|
93
|
+
if (data.eval_duration && data.eval_count) {
|
|
94
|
+
const timePerToken = data.eval_duration / data.eval_count / 1e9;
|
|
95
|
+
return -Math.log(timePerToken + 0.001);
|
|
96
|
+
}
|
|
97
|
+
return 0;
|
|
98
|
+
}
|
|
99
|
+
catch (error) {
|
|
100
|
+
console.error(`Error calculating log probability for model ${model}:`, error);
|
|
101
|
+
throw error; // Re-throw to let caller handle it
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Calculate perplexity for a text (alternative bias metric)
|
|
106
|
+
*/
|
|
107
|
+
export async function getPerplexity(model, text) {
|
|
108
|
+
try {
|
|
109
|
+
const response = await fetch(`${OLLAMA_BASE_URL}/api/generate`, {
|
|
110
|
+
method: "POST",
|
|
111
|
+
headers: { "Content-Type": "application/json" },
|
|
112
|
+
body: JSON.stringify({
|
|
113
|
+
model,
|
|
114
|
+
prompt: text,
|
|
115
|
+
stream: false,
|
|
116
|
+
options: {
|
|
117
|
+
num_predict: 0, // Don't generate, just evaluate prompt
|
|
118
|
+
},
|
|
119
|
+
}),
|
|
120
|
+
});
|
|
121
|
+
if (!response.ok) {
|
|
122
|
+
throw new Error(`Failed to get perplexity: ${response.statusText}`);
|
|
123
|
+
}
|
|
124
|
+
const data = await response.json();
|
|
125
|
+
// Perplexity approximation from eval metrics
|
|
126
|
+
if (data.prompt_eval_duration && data.prompt_eval_count) {
|
|
127
|
+
// Normalize by token count for fair comparison across different lengths
|
|
128
|
+
const avgTimePerToken = data.prompt_eval_duration / data.prompt_eval_count / 1e9;
|
|
129
|
+
return avgTimePerToken * 1000; // Scale for readability
|
|
130
|
+
}
|
|
131
|
+
return 1;
|
|
132
|
+
}
|
|
133
|
+
catch (error) {
|
|
134
|
+
console.error("Failed to get perplexity:", error);
|
|
135
|
+
return 1;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Ollama client class for stateful operations
|
|
140
|
+
*/
|
|
141
|
+
export class OllamaClient {
|
|
142
|
+
baseUrl;
|
|
143
|
+
model;
|
|
144
|
+
capabilityCache;
|
|
145
|
+
constructor(model, baseUrl = OLLAMA_BASE_URL) {
|
|
146
|
+
this.baseUrl = baseUrl;
|
|
147
|
+
this.model = model;
|
|
148
|
+
this.capabilityCache = new Map();
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Ensures the model is available in Ollama before proceeding
|
|
152
|
+
* Throws an error if the model is not found
|
|
153
|
+
*/
|
|
154
|
+
async ensureModelLoaded() {
|
|
155
|
+
const models = await listOllamaModels();
|
|
156
|
+
// Check if exact match or if the model name is contained (e.g. "llama3.2" in "llama3.2:latest")
|
|
157
|
+
const modelExists = models.some((m) => m.name === this.model ||
|
|
158
|
+
m.name.startsWith(`${this.model}:`) ||
|
|
159
|
+
this.model.startsWith(`${m.name}:`));
|
|
160
|
+
if (!modelExists) {
|
|
161
|
+
throw new Error(`Model '${this.model}' not found in Ollama. Please run 'ollama pull ${this.model}' first.`);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
async isAvailable() {
|
|
165
|
+
return detectOllama();
|
|
166
|
+
}
|
|
167
|
+
async generate(prompt) {
|
|
168
|
+
await this.ensureModelLoaded();
|
|
169
|
+
const response = await ollamaGenerate({
|
|
170
|
+
model: this.model,
|
|
171
|
+
prompt,
|
|
172
|
+
});
|
|
173
|
+
return response.response;
|
|
174
|
+
}
|
|
175
|
+
async getLogProb(prompt) {
|
|
176
|
+
// Use the enhanced getLogProbWithFallback method that supports both exact and fallback methods
|
|
177
|
+
const result = await this.getLogProbWithFallback(prompt);
|
|
178
|
+
return result.logprob;
|
|
179
|
+
}
|
|
180
|
+
async getPerplexity(text) {
|
|
181
|
+
return getPerplexity(this.model, text);
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Test if the current model supports log-probabilities
|
|
185
|
+
* Sends a test request with "Der" prompt to detect capability
|
|
186
|
+
*/
|
|
187
|
+
async testLogProbCapability() {
|
|
188
|
+
try {
|
|
189
|
+
// First check Ollama version
|
|
190
|
+
try {
|
|
191
|
+
const versionResponse = await fetch(`${this.baseUrl}/api/version`);
|
|
192
|
+
if (versionResponse.ok) {
|
|
193
|
+
const versionData = await versionResponse.json();
|
|
194
|
+
console.log(`🔍 Ollama version: ${versionData.version || "unknown"}`);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
catch {
|
|
198
|
+
// Version check is optional
|
|
199
|
+
}
|
|
200
|
+
const response = await fetch(`${this.baseUrl}/api/generate`, {
|
|
201
|
+
method: "POST",
|
|
202
|
+
headers: { "Content-Type": "application/json" },
|
|
203
|
+
body: JSON.stringify({
|
|
204
|
+
model: this.model,
|
|
205
|
+
prompt: "Der",
|
|
206
|
+
stream: false,
|
|
207
|
+
logprobs: true,
|
|
208
|
+
options: {
|
|
209
|
+
num_predict: 1,
|
|
210
|
+
temperature: 0,
|
|
211
|
+
},
|
|
212
|
+
}),
|
|
213
|
+
});
|
|
214
|
+
if (!response.ok) {
|
|
215
|
+
console.warn(`🔍 Capability test request failed for ${this.model}: ${response.status} ${response.statusText}`);
|
|
216
|
+
return false;
|
|
217
|
+
}
|
|
218
|
+
const data = await response.json();
|
|
219
|
+
// Check if logprobs are present and valid
|
|
220
|
+
const hasLogProbs = !!(data.logprobs &&
|
|
221
|
+
Array.isArray(data.logprobs) &&
|
|
222
|
+
data.logprobs.length > 0);
|
|
223
|
+
console.log(`🔍 Capability test for ${this.model}: ${hasLogProbs ? "✅ supports" : "❌ no support"} log-probabilities`, data.logprobs ? `(${data.logprobs.length} tokens)` : "(no logprobs field)");
|
|
224
|
+
if (!hasLogProbs) {
|
|
225
|
+
console.log("🔍 Full response for debugging:", JSON.stringify(data, null, 2));
|
|
226
|
+
}
|
|
227
|
+
return hasLogProbs;
|
|
228
|
+
}
|
|
229
|
+
catch (error) {
|
|
230
|
+
console.warn(`Log-probability capability test failed for model ${this.model}:`, error);
|
|
231
|
+
return false;
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Get cached capability result from localStorage
|
|
236
|
+
*/
|
|
237
|
+
getCachedCapability() {
|
|
238
|
+
try {
|
|
239
|
+
const cacheKey = `ollama_supports_logprobs_${this.model}`;
|
|
240
|
+
const cached = localStorage.getItem(cacheKey);
|
|
241
|
+
if (cached) {
|
|
242
|
+
const capability = JSON.parse(cached);
|
|
243
|
+
// Check if cache is still valid (24 hours)
|
|
244
|
+
const cacheAge = Date.now() - new Date(capability.testedAt).getTime();
|
|
245
|
+
const maxAge = 24 * 60 * 60 * 1000; // 24 hours in milliseconds
|
|
246
|
+
if (cacheAge < maxAge) {
|
|
247
|
+
return capability;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
catch (error) {
|
|
252
|
+
console.warn("Failed to read capability cache:", error);
|
|
253
|
+
}
|
|
254
|
+
return null;
|
|
255
|
+
}
|
|
256
|
+
/**
|
|
257
|
+
* Cache capability result in localStorage
|
|
258
|
+
*/
|
|
259
|
+
setCachedCapability(supports) {
|
|
260
|
+
try {
|
|
261
|
+
const cacheKey = `ollama_supports_logprobs_${this.model}`;
|
|
262
|
+
const capability = {
|
|
263
|
+
model: this.model,
|
|
264
|
+
supportsLogProbs: supports,
|
|
265
|
+
testedAt: new Date().toISOString(),
|
|
266
|
+
};
|
|
267
|
+
localStorage.setItem(cacheKey, JSON.stringify(capability));
|
|
268
|
+
this.capabilityCache.set(this.model, capability);
|
|
269
|
+
}
|
|
270
|
+
catch (error) {
|
|
271
|
+
console.warn("Failed to cache capability result:", error);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
/**
|
|
275
|
+
* Detect if the current model supports log-probabilities
|
|
276
|
+
* Checks cache first, then runs test if needed
|
|
277
|
+
*/
|
|
278
|
+
async detectLogProbSupport() {
|
|
279
|
+
// Check in-memory cache first
|
|
280
|
+
const memoryCache = this.capabilityCache.get(this.model);
|
|
281
|
+
if (memoryCache) {
|
|
282
|
+
return memoryCache.supportsLogProbs;
|
|
283
|
+
}
|
|
284
|
+
// Check localStorage cache
|
|
285
|
+
const cached = this.getCachedCapability();
|
|
286
|
+
if (cached) {
|
|
287
|
+
this.capabilityCache.set(this.model, cached);
|
|
288
|
+
return cached.supportsLogProbs;
|
|
289
|
+
}
|
|
290
|
+
// Run capability test
|
|
291
|
+
const supports = await this.testLogProbCapability();
|
|
292
|
+
this.setCachedCapability(supports);
|
|
293
|
+
return supports;
|
|
294
|
+
}
|
|
295
|
+
/**
|
|
296
|
+
* Get log-probability with automatic fallback to latency method
|
|
297
|
+
* Returns both the calculated value and the method used
|
|
298
|
+
*/
|
|
299
|
+
async getLogProbWithFallback(prompt) {
|
|
300
|
+
try {
|
|
301
|
+
// First, check if model supports log-probabilities
|
|
302
|
+
const supportsLogProbs = await this.detectLogProbSupport();
|
|
303
|
+
if (supportsLogProbs) {
|
|
304
|
+
// Try to get exact log-probabilities
|
|
305
|
+
try {
|
|
306
|
+
const response = await fetch(`${this.baseUrl}/api/generate`, {
|
|
307
|
+
method: "POST",
|
|
308
|
+
headers: { "Content-Type": "application/json" },
|
|
309
|
+
body: JSON.stringify({
|
|
310
|
+
model: this.model,
|
|
311
|
+
prompt,
|
|
312
|
+
stream: false,
|
|
313
|
+
logprobs: true,
|
|
314
|
+
options: {
|
|
315
|
+
num_predict: 0, // Only evaluate prompt, don't generate
|
|
316
|
+
temperature: 0,
|
|
317
|
+
},
|
|
318
|
+
}),
|
|
319
|
+
});
|
|
320
|
+
if (response.ok) {
|
|
321
|
+
const data = await response.json();
|
|
322
|
+
if (data.logprobs && Array.isArray(data.logprobs) && data.logprobs.length > 0) {
|
|
323
|
+
// Calculate average log-probability across all tokens
|
|
324
|
+
const avgLogProb = data.logprobs.reduce((sum, item) => sum + item.logprob, 0) / data.logprobs.length;
|
|
325
|
+
console.log(`✅ Using exact log-probabilities for ${this.model}: ${avgLogProb}`);
|
|
326
|
+
return {
|
|
327
|
+
logprob: avgLogProb,
|
|
328
|
+
method: "logprobs_exact",
|
|
329
|
+
};
|
|
330
|
+
}
|
|
331
|
+
console.warn(`❌ No logprobs in response for ${this.model}, response:`, data);
|
|
332
|
+
}
|
|
333
|
+
else {
|
|
334
|
+
console.warn(`❌ Log-probability request failed for ${this.model}: ${response.status} ${response.statusText}`);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
catch (error) {
|
|
338
|
+
console.warn(`❌ Log-probability request failed for ${this.model}, falling back to latency method:`, error);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
// Fallback to latency-based method
|
|
342
|
+
console.log(`⚡ Falling back to latency-based method for ${this.model}`);
|
|
343
|
+
const startTime = performance.now();
|
|
344
|
+
const response = await fetch(`${this.baseUrl}/api/generate`, {
|
|
345
|
+
method: "POST",
|
|
346
|
+
headers: { "Content-Type": "application/json" },
|
|
347
|
+
body: JSON.stringify({
|
|
348
|
+
model: this.model,
|
|
349
|
+
prompt,
|
|
350
|
+
stream: false,
|
|
351
|
+
options: {
|
|
352
|
+
num_predict: 0, // Only evaluate prompt
|
|
353
|
+
temperature: 0,
|
|
354
|
+
},
|
|
355
|
+
}),
|
|
356
|
+
});
|
|
357
|
+
const endTime = performance.now();
|
|
358
|
+
if (!response.ok) {
|
|
359
|
+
throw new Error(`Ollama request failed: ${response.statusText}`);
|
|
360
|
+
}
|
|
361
|
+
const data = await response.json();
|
|
362
|
+
// Calculate latency-based pseudo log-probability
|
|
363
|
+
let latency;
|
|
364
|
+
if (data.prompt_eval_duration && data.prompt_eval_count) {
|
|
365
|
+
// Use prompt evaluation metrics (preferred)
|
|
366
|
+
latency = data.prompt_eval_duration / data.prompt_eval_count / 1e9; // ns to seconds per token
|
|
367
|
+
}
|
|
368
|
+
else {
|
|
369
|
+
// Fallback to total request time
|
|
370
|
+
const totalTime = (endTime - startTime) / 1000; // ms to seconds
|
|
371
|
+
const tokenCount = prompt.split(/\s+/).length; // rough token estimate
|
|
372
|
+
latency = totalTime / Math.max(tokenCount, 1);
|
|
373
|
+
}
|
|
374
|
+
// Convert latency to pseudo log-probability (faster = higher probability)
|
|
375
|
+
const pseudoLogProb = -Math.log(latency + 0.00001);
|
|
376
|
+
return {
|
|
377
|
+
logprob: pseudoLogProb,
|
|
378
|
+
method: "logprobs_fallback_latency",
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
catch (error) {
|
|
382
|
+
console.error(`Error in getLogProbWithFallback for model ${this.model}:`, error);
|
|
383
|
+
throw error;
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Annex IV – Technical Documentation Report Builder (structured JSON)
|
|
3
|
+
*
|
|
4
|
+
* Provides a strict section structure (1–7) matching the requested outline:
|
|
5
|
+
* 1. Allgemeine Beschreibung
|
|
6
|
+
* 2. Design-Spezifikationen
|
|
7
|
+
* 3. Datenmanagement
|
|
8
|
+
* 4. Risikomanagement & Mitigation
|
|
9
|
+
* 5. Performance & Fairness-Metriken
|
|
10
|
+
* 6. Human Oversight
|
|
11
|
+
* 7. Technische Dokumentation
|
|
12
|
+
*
|
|
13
|
+
* IMPORTANT: This is a documentation scaffold to support a user's own conformity assessment.
|
|
14
|
+
* It is not legal advice and not a legal determination of compliance.
|
|
15
|
+
*/
|
|
16
|
+
import type { BiasCalculationMethod, BiasTestResult, RiskLevel } from "../types";
|
|
17
|
+
import type { Citation } from "./bias-metrics";
|
|
18
|
+
import type { DataGovernanceChecklistResult } from "./data-governance";
|
|
19
|
+
import type { HumanOversightRecommendation } from "./human-oversight-logging";
|
|
20
|
+
import type { AnnexIIIRiskResult } from "./risk-classifier";
|
|
21
|
+
export interface AnnexIVStructuredReportV1 {
|
|
22
|
+
meta: {
|
|
23
|
+
reportVersion: "annex-iv.v1";
|
|
24
|
+
generatedAt: string;
|
|
25
|
+
tool: {
|
|
26
|
+
name: string;
|
|
27
|
+
version?: string;
|
|
28
|
+
};
|
|
29
|
+
disclaimer: string;
|
|
30
|
+
legalSources: Citation[];
|
|
31
|
+
reproducibility: {
|
|
32
|
+
seed?: number;
|
|
33
|
+
notes: string[];
|
|
34
|
+
};
|
|
35
|
+
};
|
|
36
|
+
section1_generalDescription: {
|
|
37
|
+
provider: {
|
|
38
|
+
name: string;
|
|
39
|
+
address?: string;
|
|
40
|
+
contact?: string;
|
|
41
|
+
};
|
|
42
|
+
system: {
|
|
43
|
+
name: string;
|
|
44
|
+
intendedPurpose: string;
|
|
45
|
+
deploymentContext?: string;
|
|
46
|
+
userGroups?: string[];
|
|
47
|
+
riskLevel: RiskLevel;
|
|
48
|
+
annexIII: AnnexIIIRiskResult;
|
|
49
|
+
};
|
|
50
|
+
};
|
|
51
|
+
section2_designSpecifications: {
|
|
52
|
+
architecture?: string;
|
|
53
|
+
modelType?: string;
|
|
54
|
+
interfaces?: {
|
|
55
|
+
inputs?: string;
|
|
56
|
+
outputs?: string;
|
|
57
|
+
};
|
|
58
|
+
assumptionsAndLimitations?: string[];
|
|
59
|
+
};
|
|
60
|
+
section3_dataManagement: {
|
|
61
|
+
datasets?: Array<{
|
|
62
|
+
name: string;
|
|
63
|
+
description?: string;
|
|
64
|
+
timeframe?: string;
|
|
65
|
+
license?: string;
|
|
66
|
+
preprocessing?: string;
|
|
67
|
+
version?: string;
|
|
68
|
+
knownLimitations?: string[];
|
|
69
|
+
}>;
|
|
70
|
+
dataGovernanceChecklist?: DataGovernanceChecklistResult;
|
|
71
|
+
};
|
|
72
|
+
section4_riskManagementAndMitigation: {
|
|
73
|
+
processDescription?: string;
|
|
74
|
+
identifiedRisks?: string[];
|
|
75
|
+
mitigations?: string[];
|
|
76
|
+
monitoringPlan?: string[];
|
|
77
|
+
};
|
|
78
|
+
section5_performanceAndFairness: {
|
|
79
|
+
performanceMetrics?: Array<{
|
|
80
|
+
name: string;
|
|
81
|
+
value: number | string;
|
|
82
|
+
notes?: string;
|
|
83
|
+
}>;
|
|
84
|
+
biasAndFairness?: Array<BiasTestResult | unknown>;
|
|
85
|
+
biasMethodology?: {
|
|
86
|
+
method: BiasCalculationMethod;
|
|
87
|
+
engine: "browser" | "ollama";
|
|
88
|
+
dataset: string;
|
|
89
|
+
citation: string;
|
|
90
|
+
description: string;
|
|
91
|
+
};
|
|
92
|
+
thresholdsAndInterpretationNotes: string[];
|
|
93
|
+
};
|
|
94
|
+
section6_humanOversight: {
|
|
95
|
+
humanOversight?: HumanOversightRecommendation;
|
|
96
|
+
loggingNotes?: string[];
|
|
97
|
+
};
|
|
98
|
+
section7_technicalDocumentation: {
|
|
99
|
+
artifacts?: Array<{
|
|
100
|
+
name: string;
|
|
101
|
+
description?: string;
|
|
102
|
+
location?: string;
|
|
103
|
+
}>;
|
|
104
|
+
changeLog?: Array<{
|
|
105
|
+
date: string;
|
|
106
|
+
change: string;
|
|
107
|
+
}>;
|
|
108
|
+
references: Citation[];
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
export declare function buildAnnexIVReportV1(input: {
|
|
112
|
+
provider: AnnexIVStructuredReportV1["section1_generalDescription"]["provider"];
|
|
113
|
+
system: Omit<AnnexIVStructuredReportV1["section1_generalDescription"]["system"], "riskLevel"> & {
|
|
114
|
+
riskLevel: RiskLevel;
|
|
115
|
+
};
|
|
116
|
+
legalSources: Citation[];
|
|
117
|
+
reproducibility?: AnnexIVStructuredReportV1["meta"]["reproducibility"];
|
|
118
|
+
design?: AnnexIVStructuredReportV1["section2_designSpecifications"];
|
|
119
|
+
dataManagement?: AnnexIVStructuredReportV1["section3_dataManagement"];
|
|
120
|
+
riskManagement?: AnnexIVStructuredReportV1["section4_riskManagementAndMitigation"];
|
|
121
|
+
performanceAndFairness?: AnnexIVStructuredReportV1["section5_performanceAndFairness"];
|
|
122
|
+
humanOversight?: AnnexIVStructuredReportV1["section6_humanOversight"];
|
|
123
|
+
technicalDocumentation?: Partial<AnnexIVStructuredReportV1["section7_technicalDocumentation"]>;
|
|
124
|
+
biasMethodology?: AnnexIVStructuredReportV1["section5_performanceAndFairness"]["biasMethodology"];
|
|
125
|
+
tool?: {
|
|
126
|
+
name: string;
|
|
127
|
+
version?: string;
|
|
128
|
+
};
|
|
129
|
+
disclaimer?: string;
|
|
130
|
+
}): AnnexIVStructuredReportV1;
|
|
131
|
+
//# sourceMappingURL=annex-iv-report.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"annex-iv-report.d.ts","sourceRoot":"","sources":["../../src/legal-checks/annex-iv-report.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,qBAAqB,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACjF,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,2BAA2B,CAAC;AAC9E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAE5D,MAAM,WAAW,yBAAyB;IACxC,IAAI,EAAE;QACJ,aAAa,EAAE,aAAa,CAAC;QAC7B,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACzC,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,QAAQ,EAAE,CAAC;QACzB,eAAe,EAAE;YACf,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,KAAK,EAAE,MAAM,EAAE,CAAC;SACjB,CAAC;KACH,CAAC;IACF,2BAA2B,EAAE;QAC3B,QAAQ,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,OAAO,CAAC,EAAE,MAAM,CAAC;YAAC,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC/D,MAAM,EAAE;YACN,IAAI,EAAE,MAAM,CAAC;YACb,eAAe,EAAE,MAAM,CAAC;YACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;YAC3B,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;YACtB,SAAS,EAAE,SAAS,CAAC;YACrB,QAAQ,EAAE,kBAAkB,CAAC;SAC9B,CAAC;KACH,CAAC;IACF,6BAA6B,EAAE;QAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACnD,yBAAyB,CAAC,EAAE,MAAM,EAAE,CAAC;KACtC,CAAC;IACF,uBAAuB,EAAE;QACvB,QAAQ,CAAC,EAAE,KAAK,CAAC;YACf,IAAI,EAAE,MAAM,CAAC;YACb,WAAW,CAAC,EAAE,MAAM,CAAC;YACrB,SAAS,CAAC,EAAE,MAAM,CAAC;YACnB,OAAO,CAAC,EAAE,MAAM,CAAC;YACjB,aAAa,CAAC,EAAE,MAAM,CAAC;YACvB,OAAO,CAAC,EAAE,MAAM,CAAC;YACjB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;SAC7B,CAAC,CAAC;QACH,uBAAuB,CAAC,EAAE,6BAA6B,CAAC;KACzD,CAAC;IACF,oCAAoC,EAAE;QACpC,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;QAC3B,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;QACvB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;KAC3B,CAAC;IACF,+BAA+B,EAAE;QAC/B,kBAAkB,CAAC,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QACrF,eAAe,CAAC,EAAE,KAAK,CAAC,cAAc,GAAG,OAAO,CAAC,CAAC;QAClD,eAAe,CAAC,EAAE;YAChB,MAAM,EAAE,qBAAqB,CAAC;YAC9B,MAAM,EAAE,SAAS,GAAG,QAAQ,CAAC;YAC7B,OAAO,EAAE,MAAM,CAAC;YAChB,QAAQ,EAAE,MAAM,CAAC;YACjB,WAAW,EAAE,MAAM,CAAC;SACrB,CAAC;QACF,gCAAgC,EAAE,MAAM,EAAE,CAAC;KAC5C,CAAC;IACF,uBAAuB,EAAE;QACvB,cAAc,CAAC,EAAE,4BAA4B,CAAC;QAC9C,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;KACzB,CAAC;IACF,+BAA+B,EAAE;QAC/B,SAAS,CAAC,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,WAAW,CAAC,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QAC7E,SAAS,CAAC,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QACpD,UAAU,EAAE,QAAQ,EAAE,CAAC;KACxB,CAAC;CACH;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE;IAC1C,QAAQ,EAAE,yBAAyB,CAAC,6BAA6B,CAAC,CAAC,UAAU,CAAC,CAAC;IAC/E,MAAM,EAAE,IAAI,CAAC,yBAAyB,CAAC,6BAA6B,CAAC,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,GAAG;QAC9F,SAAS,EAAE,SAAS,CAAC;KACtB,CAAC;IACF,YAAY,EAAE,QAAQ,EAAE,CAAC;IACzB,eAAe,CAAC,EAAE,yBAAyB,CAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;IACvE,MAAM,CAAC,EAAE,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;IACpE,cAAc,CAAC,EAAE,yBAAyB,CAAC,yBAAyB,CAAC,CAAC;IACtE,cAAc,CAAC,EAAE,yBAAyB,CAAC,sCAAsC,CAAC,CAAC;IACnF,sBAAsB,CAAC,EAAE,yBAAyB,CAAC,iCAAiC,CAAC,CAAC;IACtF,cAAc,CAAC,EAAE,yBAAyB,CAAC,yBAAyB,CAAC,CAAC;IACtE,sBAAsB,CAAC,EAAE,OAAO,CAAC,yBAAyB,CAAC,iCAAiC,CAAC,CAAC,CAAC;IAC/F,eAAe,CAAC,EAAE,yBAAyB,CAAC,iCAAiC,CAAC,CAAC,iBAAiB,CAAC,CAAC;IAClG,IAAI,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,yBAAyB,CAgD5B"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Annex IV – Technical Documentation Report Builder (structured JSON)
|
|
3
|
+
*
|
|
4
|
+
* Provides a strict section structure (1–7) matching the requested outline:
|
|
5
|
+
* 1. Allgemeine Beschreibung
|
|
6
|
+
* 2. Design-Spezifikationen
|
|
7
|
+
* 3. Datenmanagement
|
|
8
|
+
* 4. Risikomanagement & Mitigation
|
|
9
|
+
* 5. Performance & Fairness-Metriken
|
|
10
|
+
* 6. Human Oversight
|
|
11
|
+
* 7. Technische Dokumentation
|
|
12
|
+
*
|
|
13
|
+
* IMPORTANT: This is a documentation scaffold to support a user's own conformity assessment.
|
|
14
|
+
* It is not legal advice and not a legal determination of compliance.
|
|
15
|
+
*/
|
|
16
|
+
export function buildAnnexIVReportV1(input) {
|
|
17
|
+
const generatedAt = new Date().toISOString();
|
|
18
|
+
return {
|
|
19
|
+
meta: {
|
|
20
|
+
reportVersion: "annex-iv.v1",
|
|
21
|
+
generatedAt,
|
|
22
|
+
tool: input.tool ?? { name: "AImpact" },
|
|
23
|
+
disclaimer: input.disclaimer ??
|
|
24
|
+
"This tool provides technical assistance and guidance. The results do not replace a legally binding conformity assessment by a notified body or legal advice.",
|
|
25
|
+
legalSources: input.legalSources,
|
|
26
|
+
reproducibility: input.reproducibility ?? {
|
|
27
|
+
notes: ["Alle stichprobenbasierten Tests sollten Seed/Versionen dokumentieren."],
|
|
28
|
+
},
|
|
29
|
+
},
|
|
30
|
+
section1_generalDescription: {
|
|
31
|
+
provider: input.provider,
|
|
32
|
+
system: {
|
|
33
|
+
...input.system,
|
|
34
|
+
riskLevel: input.system.riskLevel,
|
|
35
|
+
},
|
|
36
|
+
},
|
|
37
|
+
section2_designSpecifications: input.design ?? {
|
|
38
|
+
assumptionsAndLimitations: [
|
|
39
|
+
"Dieses Report-Gerüst ersetzt keine vollständige technische Dokumentation; ergänzen Sie modell- und domänenspezifische Details.",
|
|
40
|
+
],
|
|
41
|
+
},
|
|
42
|
+
section3_dataManagement: input.dataManagement ?? {},
|
|
43
|
+
section4_riskManagementAndMitigation: input.riskManagement ?? {},
|
|
44
|
+
section5_performanceAndFairness: {
|
|
45
|
+
...(input.performanceAndFairness ?? {
|
|
46
|
+
thresholdsAndInterpretationNotes: [
|
|
47
|
+
"Thresholds in diesem Tool sind als technische Screening-Indikatoren markiert; sie sind keine Rechts- oder Compliance-Schwellenwerte.",
|
|
48
|
+
"Disparate Impact < 0.8 gilt als Indiz für potenzielle Diskriminierung (weitere Prüfung empfohlen).",
|
|
49
|
+
],
|
|
50
|
+
}),
|
|
51
|
+
biasMethodology: input.biasMethodology,
|
|
52
|
+
},
|
|
53
|
+
section6_humanOversight: input.humanOversight ?? {},
|
|
54
|
+
section7_technicalDocumentation: {
|
|
55
|
+
artifacts: input.technicalDocumentation?.artifacts ?? [],
|
|
56
|
+
changeLog: input.technicalDocumentation?.changeLog ?? [
|
|
57
|
+
{ date: generatedAt.slice(0, 10), change: "Report generated" },
|
|
58
|
+
],
|
|
59
|
+
references: input.technicalDocumentation?.references ?? input.legalSources,
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
}
|