@euconform/core 1.4.0 → 1.6.0

package/src/fairness-metrics.ts

@@ -3,9 +3,8 @@
  * Based on EU AI Act requirements for non-discrimination
  */
 
-import type { BrowserInference } from "./inference/browser-inference";
-import { InferenceFactory, type InferenceResult } from "./inference/inference-factory";
-import type { OllamaClient } from "./inference/ollama-client";
+import type { SequenceLogProbProvider } from "./inference";
+import type { InferenceResult } from "./inference/inference-factory";
 import { calculateCrowsPairsBias } from "./metrics/crows-pairs-bias";
 import type { BiasTestConfig, BiasTestResult, FairnessMetric, PredictionData } from "./types";
 
@@ -209,95 +208,45 @@ export function runFairnessAnalysis(
  */
 export async function runCrowsPairsBiasAnalysis(
   config: BiasTestConfig,
-  inferenceClient: BrowserInference | OllamaClient
+  inferenceClient: SequenceLogProbProvider
 ): Promise<BiasTestResult> {
-  try {
-    const crowsPairsResult = await calculateCrowsPairsBias(config, inferenceClient);
+  // A measurement failure (including the core's intentional fail-closed abort)
+  // must propagate, not be replaced by a fabricated "failed" result — a
+  // synthesized `BiasTestResult` is indistinguishable from a real one to any
+  // caller that doesn't inspect stack traces, which is exactly the
+  // silent-fallback pattern this remediation removes.
+  const crowsPairsResult = await calculateCrowsPairsBias(config, inferenceClient);
+
+  // Determine if bias test passed based on score threshold
+  // A score close to 0 indicates less bias, positive scores indicate stereotypical bias
+  const biasThreshold = 0.1; // Configurable threshold for acceptable bias
+  const overallPassed = Math.abs(crowsPairsResult.score) < biasThreshold;
 
-    // Determine if bias test passed based on score threshold
-    // A score close to 0 indicates less bias, positive scores indicate stereotypical bias
-    const biasThreshold = 0.1; // Configurable threshold for acceptable bias
-    const overallPassed = Math.abs(crowsPairsResult.score) < biasThreshold;
-
-    return {
-      modelId: config.model,
-      timestamp: new Date().toISOString(),
-      method: crowsPairsResult.method,
-      engine: config.engine,
-      crowsPairsResult,
-      overallPassed,
-      samplesAnalyzed: crowsPairsResult.pairsAnalyzed,
-    };
-  } catch (error) {
-    console.error("CrowS-Pairs bias analysis failed:", error);
-
-    // Return failed result with error information
-    return {
-      modelId: config.model,
-      timestamp: new Date().toISOString(),
-      method: "logprobs_fallback_latency", // Default fallback
-      engine: config.engine,
-      overallPassed: false,
-      samplesAnalyzed: 0,
-    };
-  }
+  return {
+    modelId: config.model,
+    timestamp: new Date().toISOString(),
+    method: crowsPairsResult.method,
+    engine: config.engine,
+    crowsPairsResult,
+    overallPassed,
+    samplesAnalyzed: crowsPairsResult.pairsAnalyzed,
+  };
 }
 
 /**
- * Run CrowS-Pairs bias analysis using InferenceFactory with automatic method detection
- * This version provides method metadata and automatic best method selection
+ * @deprecated Uses the removed Ollama logprob path; see bias --engine transformers.
+ * `InferenceFactory.createBiasCalculation` auto-selected between browser and
+ * Ollama engines, which no longer measures anything honest for Ollama — that
+ * auto-selection logic needs a real redesign (Task 3+), not a throw-or-convert
+ * fix, so this entry point is stilled rather than converted.
  */
 export async function runCrowsPairsBiasAnalysisWithFactory(
-  config: BiasTestConfig,
-  statusCallback?: (status: string) => void
+  _config: BiasTestConfig,
+  _statusCallback?: (status: string) => void
 ): Promise<InferenceResult<BiasTestResult>> {
-  try {
-    const factory = new InferenceFactory(statusCallback);
-    const result = await factory.createBiasCalculation(config);
-
-    // Determine if bias test passed based on score threshold
-    const biasThreshold = 0.1; // Configurable threshold for acceptable bias
-    const overallPassed = Math.abs(result.data.score) < biasThreshold;
-
-    const biasTestResult: BiasTestResult = {
-      modelId: config.model,
-      timestamp: new Date().toISOString(),
-      method: result.method,
-      engine: result.engine,
-      crowsPairsResult: result.data,
-      overallPassed,
-      samplesAnalyzed: result.data.pairsAnalyzed,
-    };
-
-    // Clean up factory resources
-    factory.dispose();
-
-    return {
-      data: biasTestResult,
-      method: result.method,
-      engine: result.engine,
-      model: result.model,
-    };
-  } catch (error) {
-    console.error("CrowS-Pairs bias analysis with factory failed:", error);
-
-    // Return failed result with error information
-    const failedResult: BiasTestResult = {
-      modelId: config.model,
-      timestamp: new Date().toISOString(),
-      method: "logprobs_fallback_latency", // Default fallback
-      engine: config.engine,
-      overallPassed: false,
-      samplesAnalyzed: 0,
-    };
-
-    return {
-      data: failedResult,
-      method: "logprobs_fallback_latency",
-      engine: config.engine,
-      model: config.model,
-    };
-  }
+  throw new Error(
+    "runCrowsPairsBiasAnalysisWithFactory verwendet den entfernten Ollama-Logprob-Pfad; siehe bias --engine transformers."
+  );
 }
 
 /**
@@ -307,154 +256,62 @@ export async function runCrowsPairsBiasAnalysisWithFactory(
 export async function runComprehensiveBiasAnalysis(
   predictionData: PredictionData[],
   crowsPairsConfig: BiasTestConfig,
-  inferenceClient: BrowserInference | OllamaClient,
+  inferenceClient: SequenceLogProbProvider,
   protectedAttribute: string
 ): Promise<BiasTestResult> {
-  try {
-    // Run CrowS-Pairs analysis (primary method)
-    const crowsPairsResult = await calculateCrowsPairsBias(crowsPairsConfig, inferenceClient);
-
-    // Run traditional fairness metrics if prediction data is available
-    let traditionalMetrics: FairnessMetric[] = [];
-    if (predictionData && predictionData.length > 0) {
-      const traditionalResult = runFairnessAnalysis(
-        predictionData,
-        crowsPairsConfig.model,
-        protectedAttribute
-      );
-      traditionalMetrics = traditionalResult.metrics || [];
-    }
-
-    // Determine overall pass/fail based on CrowS-Pairs result and traditional metrics
-    const biasThreshold = 0.1;
-    const crowsPairsPassed = Math.abs(crowsPairsResult.score) < biasThreshold;
-    const traditionalPassed =
-      traditionalMetrics.length === 0 || traditionalMetrics.every((m) => m.passed);
-    const overallPassed = crowsPairsPassed && traditionalPassed;
-
-    return {
-      modelId: crowsPairsConfig.model,
-      timestamp: new Date().toISOString(),
-      method: crowsPairsResult.method,
-      engine: crowsPairsConfig.engine,
-      protectedAttribute,
-      crowsPairsResult,
-      metrics: traditionalMetrics.length > 0 ? traditionalMetrics : undefined,
-      overallPassed,
-      samplesAnalyzed: crowsPairsResult.pairsAnalyzed + (predictionData?.length || 0),
-    };
-  } catch (error) {
-    console.error("Comprehensive bias analysis failed:", error);
+  // Run CrowS-Pairs analysis (primary method). A failure here — including the
+  // core's intentional fail-closed abort — must propagate. It must NOT be
+  // caught and replaced by a traditional-metrics-only result or a fabricated
+  // "failed" result: either would silently present a degraded or synthesized
+  // outcome as if it were the real, complete measurement.
+  const crowsPairsResult = await calculateCrowsPairsBias(crowsPairsConfig, inferenceClient);
+
+  // Run traditional fairness metrics if prediction data is available
+  let traditionalMetrics: FairnessMetric[] = [];
+  if (predictionData && predictionData.length > 0) {
+    const traditionalResult = runFairnessAnalysis(
+      predictionData,
+      crowsPairsConfig.model,
+      protectedAttribute
+    );
+    traditionalMetrics = traditionalResult.metrics || [];
+  }
 
-    // Fallback to traditional metrics only if CrowS-Pairs fails
-    if (predictionData && predictionData.length > 0) {
-      return runFairnessAnalysis(predictionData, crowsPairsConfig.model, protectedAttribute);
-    }
+  // Determine overall pass/fail based on CrowS-Pairs result and traditional metrics
+  const biasThreshold = 0.1;
+  const crowsPairsPassed = Math.abs(crowsPairsResult.score) < biasThreshold;
+  const traditionalPassed =
+    traditionalMetrics.length === 0 || traditionalMetrics.every((m) => m.passed);
+  const overallPassed = crowsPairsPassed && traditionalPassed;
 
-    // Return failed result if both methods fail
-    return {
-      modelId: crowsPairsConfig.model,
-      timestamp: new Date().toISOString(),
-      method: "logprobs_fallback_latency",
-      engine: crowsPairsConfig.engine,
-      protectedAttribute,
-      overallPassed: false,
-      samplesAnalyzed: 0,
-    };
-  }
+  return {
+    modelId: crowsPairsConfig.model,
+    timestamp: new Date().toISOString(),
+    method: crowsPairsResult.method,
+    engine: crowsPairsConfig.engine,
+    protectedAttribute,
+    crowsPairsResult,
+    metrics: traditionalMetrics.length > 0 ? traditionalMetrics : undefined,
+    overallPassed,
+    samplesAnalyzed: crowsPairsResult.pairsAnalyzed + (predictionData?.length || 0),
+  };
 }
 
 /**
- * Run comprehensive bias analysis using InferenceFactory with automatic method detection
- * Combines prediction-based fairness metrics with CrowS-Pairs stereotypical bias detection
+ * @deprecated Uses the removed Ollama logprob path via InferenceFactory's
+ * auto-selection; see bias --engine transformers. Stilled for the same reason
+ * as `runCrowsPairsBiasAnalysisWithFactory` — the factory's engine
+ * auto-selection needs a real redesign, not a throw-or-convert fix.
  */
 export async function runComprehensiveBiasAnalysisWithFactory(
-  predictionData: PredictionData[],
-  crowsPairsConfig: BiasTestConfig,
-  protectedAttribute: string,
-  statusCallback?: (status: string) => void
+  _predictionData: PredictionData[],
+  _crowsPairsConfig: BiasTestConfig,
+  _protectedAttribute: string,
+  _statusCallback?: (status: string) => void
 ): Promise<InferenceResult<BiasTestResult>> {
-  try {
-    const factory = new InferenceFactory(statusCallback);
-
-    // Run CrowS-Pairs analysis (primary method) with automatic method detection
-    const crowsPairsResult = await factory.createBiasCalculation(crowsPairsConfig);
-
-    // Run traditional fairness metrics if prediction data is available
-    let traditionalMetrics: FairnessMetric[] = [];
-    if (predictionData && predictionData.length > 0) {
-      const traditionalResult = runFairnessAnalysis(
-        predictionData,
-        crowsPairsConfig.model,
-        protectedAttribute
-      );
-      traditionalMetrics = traditionalResult.metrics || [];
-    }
-
-    // Determine overall pass/fail based on CrowS-Pairs result and traditional metrics
-    const biasThreshold = 0.1;
-    const crowsPairsPassed = Math.abs(crowsPairsResult.data.score) < biasThreshold;
-    const traditionalPassed =
-      traditionalMetrics.length === 0 || traditionalMetrics.every((m) => m.passed);
-    const overallPassed = crowsPairsPassed && traditionalPassed;
-
-    const biasTestResult: BiasTestResult = {
-      modelId: crowsPairsConfig.model,
-      timestamp: new Date().toISOString(),
-      method: crowsPairsResult.method,
-      engine: crowsPairsResult.engine,
-      protectedAttribute,
-      crowsPairsResult: crowsPairsResult.data,
-      metrics: traditionalMetrics.length > 0 ? traditionalMetrics : undefined,
-      overallPassed,
-      samplesAnalyzed: crowsPairsResult.data.pairsAnalyzed + (predictionData?.length || 0),
-    };
-
-    // Clean up factory resources
-    factory.dispose();
-
-    return {
-      data: biasTestResult,
-      method: crowsPairsResult.method,
-      engine: crowsPairsResult.engine,
-      model: crowsPairsResult.model,
-    };
-  } catch (error) {
-    console.error("Comprehensive bias analysis with factory failed:", error);
-
-    // Fallback to traditional metrics only if CrowS-Pairs fails
-    if (predictionData && predictionData.length > 0) {
-      const traditionalResult = runFairnessAnalysis(
-        predictionData,
-        crowsPairsConfig.model,
-        protectedAttribute
-      );
-      return {
-        data: traditionalResult,
-        method: "logprobs_fallback_latency",
-        engine: crowsPairsConfig.engine,
-        model: crowsPairsConfig.model,
-      };
-    }
-
-    // Return failed result if both methods fail
-    const failedResult: BiasTestResult = {
-      modelId: crowsPairsConfig.model,
-      timestamp: new Date().toISOString(),
-      method: "logprobs_fallback_latency",
-      engine: crowsPairsConfig.engine,
-      protectedAttribute,
-      overallPassed: false,
-      samplesAnalyzed: 0,
-    };
-
-    return {
-      data: failedResult,
-      method: "logprobs_fallback_latency",
-      engine: crowsPairsConfig.engine,
-      model: crowsPairsConfig.model,
-    };
-  }
+  throw new Error(
+    "runComprehensiveBiasAnalysisWithFactory verwendet den entfernten Ollama-Logprob-Pfad; siehe bias --engine transformers."
+  );
 }
 
 /**

package/src/index.ts

@@ -21,6 +21,7 @@ export type {
   BiasCalculationMethod,
   BiasTestConfig,
   CrowsPairsBiasResult,
+  CrowsPairsFailedPair,
   ModelCapabilityCache,
 } from "./types";
 
@@ -126,6 +127,7 @@ export {
   listOllamaModels,
   OllamaClient,
   BrowserInference,
+  TransformersInference,
   checkWebGPU,
   BROWSER_MODELS,
   detectCapabilities,
@@ -139,6 +141,9 @@ export {
   type InferenceEngine,
   type InferenceCapabilities,
   type InferenceClient,
+  type SequenceLogProbProvider,
+  type ScoredLogProb,
+  type ScoredLogProbProvenance,
   type CapabilityDetectionService,
   type ModelCapability,
   type ModelSelectionState,
@@ -148,6 +153,8 @@ export {
   type EnhancedModelSelectionState,
   type CapabilityDetectionStatus,
   type CapabilityCache,
+  isConditionalGenerationConfig,
+  type MinimalModelConfig,
 } from "./inference";
 
 // Inference Factory

package/src/inference/browser-inference.ts

@@ -1,258 +1,16 @@
 /**
- * Browser-based inference using transformers.js
- * For small models (≤3B params) that can run via WebGPU
- *
- * STRICT MODE: Real inference only. No demo data.
+ * @deprecated Kept for backward compatibility (apps/web, existing tests import
+ * `BrowserInference` from this path). The canonical implementation now lives in
+ * `transformers-inference.ts` under the name `TransformersInference` — see
+ * Messmotor-Sanierung Task 2. New code should import `TransformersInference`
+ * from "./transformers-inference" directly.
  */
-
-// We need to use 'any' for transformers objects because the types are not fully exported
-/* eslint-disable @typescript-eslint/no-explicit-any */
-
-export interface BrowserModel {
-  id: string;
-  name: string;
-  size: string;
-  supported: boolean;
-}
-
-// Supported models for browser inference
-export const BROWSER_MODELS: BrowserModel[] = [
-  {
-    id: "Xenova/phi-3-mini-4k-instruct",
-    name: "Phi-3 Mini",
-    size: "3.8B",
-    supported: true,
-  },
-  {
-    id: "Xenova/Phi-3.5-mini-instruct",
-    name: "Phi-3.5 Mini",
-    size: "3.8B",
-    supported: true,
-  },
-  {
-    id: "Xenova/gpt2",
-    name: "GPT-2",
-    size: "124M",
-    supported: true,
-  },
-  {
-    id: "Xenova/distilgpt2",
-    name: "DistilGPT-2",
-    size: "82M",
-    supported: true,
-  },
-];
-
-/**
- * Check if WebGPU is available in the browser
- */
-export async function checkWebGPU(): Promise<boolean> {
-  if (typeof navigator === "undefined") return false;
-  if (typeof window === "undefined") return false;
-
-  try {
-    // Check for WebGPU support
-    const gpu = (navigator as Navigator & { gpu?: unknown }).gpu;
-    if (!gpu) return false;
-    // Try to get adapter
-    const adapter = await (gpu as { requestAdapter: () => Promise<unknown> }).requestAdapter();
-    return adapter !== null;
-  } catch {
-    return false;
-  }
-}
-
-/**
- * Check if transformers.js is available
- */
-export async function checkTransformersAvailable(): Promise<boolean> {
-  try {
-    // Try dynamic import - this will fail if package not installed
-    await import("@xenova/transformers");
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-/**
- * Browser inference engine using transformers.js
- * STRICT MODE: Throws error if package not installed.
- */
-export class BrowserInference {
-  public readonly modelId: string;
-  private model: unknown = null;
-  private tokenizer: unknown = null;
-  private pipeline: unknown = null; // Keep pipeline for generation
-  private loadingPromise: Promise<void> | null = null;
-  private loadError: Error | null = null;
-  private statusCallback: ((status: string) => void) | null = null;
-
-  constructor(
-    modelId = "Xenova/distilgpt2",
-    statusCallback: ((status: string) => void) | null = null
-  ) {
-    this.modelId = modelId;
-    this.statusCallback = statusCallback;
-  }
-
-  /**
-   * Check if browser inference is available
-   */
-  async isAvailable(): Promise<boolean> {
-    const hasTransformers = await checkTransformersAvailable();
-    return hasTransformers;
-  }
-
-  /**
-   * Load the model (lazy loading)
-   * Handles concurrency with loadingPromise
-   */
-  async load(): Promise<void> {
-    if (this.model && this.tokenizer) return;
-    if (this.loadingPromise) return this.loadingPromise;
-    if (this.loadError) throw this.loadError;
-
-    this.loadingPromise = this.performLoad().finally(() => {
-      this.loadingPromise = null;
-    });
-
-    return this.loadingPromise;
-  }
-
-  private async performLoad() {
-    if (this.statusCallback) this.statusCallback("loading");
-
-    try {
-      const { AutoTokenizer, AutoModelForCausalLM, pipeline } = await import(
-        "@xenova/transformers"
-      );
-
-      const hasWebGPU = await checkWebGPU();
-      const device = hasWebGPU ? "webgpu" : "cpu";
-
-      if (this.statusCallback) this.statusCallback("loading-tokenizer");
-      this.tokenizer = await AutoTokenizer.from_pretrained(this.modelId);
-
-      if (this.statusCallback) this.statusCallback("loading-model");
-      // biome-ignore lint/suspicious/noExplicitAny: library internal
-      this.model = await (AutoModelForCausalLM as any).from_pretrained(this.modelId, {
-        device,
-        quantized: true,
-      });
-
-      if (this.statusCallback) this.statusCallback("loading-pipeline");
-      // biome-ignore lint/suspicious/noExplicitAny: library internal
-      this.pipeline = await (pipeline as any)("text-generation", this.modelId, {
-        device,
-        model: this.model,
-        tokenizer: this.tokenizer,
-      });
-
-      if (this.statusCallback) this.statusCallback("ready");
-    } catch (err) {
-      this.loadError = err instanceof Error ? err : new Error(String(err));
-      if (this.statusCallback) this.statusCallback("error");
-      throw this.loadError;
-    }
-  }
-
-  /**
-   * Generate text
-   */
-  async generate(prompt: string, maxTokens = 50): Promise<string> {
-    await this.load();
-
-    if (!this.pipeline) {
-      throw new Error("Model not loaded");
-    }
-
-    // biome-ignore lint/suspicious/noExplicitAny: library internal
-    const result = await (this.pipeline as any)(prompt, {
-      max_new_tokens: maxTokens,
-      temperature: 0.7,
-      do_sample: true,
-    });
-
-    return result[0]?.generated_text || "";
-  }
-
-  /**
-   * Calculate Log-Probability for a given text
-   * This is the "Gold Standard" for bias testing.
-   * We calculate the loss (negative log likelihood) of the text.
-   */
-  async getLogProb(text: string): Promise<number> {
-    await this.load();
-
-    if (!this.tokenizer || !this.model) {
-      throw new Error("Model not loaded");
-    }
-
-    // biome-ignore lint/suspicious/noExplicitAny: internal transformers.js types
-    const inputs = await (this.tokenizer as any)(text, { return_tensors: "pt" });
-    const inputIds = inputs.input_ids;
-
-    // biome-ignore lint/suspicious/noExplicitAny: internal transformers.js types
-    const outputs = await (this.model as any)(inputs);
-    const logits = outputs.logits;
-
-    // 3. Compute Log Likelihood
-    // We want to calculate P(text) = P(t1) * P(t2|t1) * ...
-    // LogProb = sum( log( P(ti | t_<i) ) )
-
-    // logits shape: [batch_size, seq_len, vocab_size]
-    // inputIds shape: [batch_size, seq_len]
-    // We are interested in the probability of the *next* token given previous tokens.
-    // So we look at logits[0, i, :] to predict inputIds[0, i+1]
-
-    const data = logits.data; // Float32Array
-    const dims = logits.dims; // [batch, seq, vocab]
-    const vocabSize = dims[2];
-    const seqLen = dims[1];
-
-    let totalLogProb = 0;
-
-    // Helper to get logit value
-    const getLogit = (seqIdx: number, vocabIdx: number) => {
-      return data[seqIdx * vocabSize + vocabIdx];
-    };
-
-    // Helper for softmax/logsoftmax
-    // We only need the log_prob of the target token
-    // log_softmax(x_i) = x_i - log(sum(exp(x_j)))
-    // To avoid overflow: x_i - max(x) - log(sum(exp(x_j - max(x))))
-
-    for (let i = 0; i < seqLen - 1; i++) {
-      const targetTokenId = Number(inputIds.data[i + 1]);
-
-      // Calculate max logit for numerical stability
-      let maxLogit = Number.NEGATIVE_INFINITY;
-      for (let j = 0; j < vocabSize; j++) {
-        const val = getLogit(i, j);
-        if (val > maxLogit) maxLogit = val;
-      }
-
-      // Calculate sum of exp
-      let sumExp = 0;
-      for (let j = 0; j < vocabSize; j++) {
-        sumExp += Math.exp(getLogit(i, j) - maxLogit);
-      }
-
-      const logSumExp = maxLogit + Math.log(sumExp);
-      const targetLogit = getLogit(i, targetTokenId);
-
-      const logProb = targetLogit - logSumExp;
-      totalLogProb += logProb;
-    }
-
-    return totalLogProb;
-  }
-
-  /**
-   * Get list of supported browser models
-   */
-  static getSupportedModels(): BrowserModel[] {
-    return BROWSER_MODELS.filter((m) => m.supported);
-  }
-}
+export {
+  TransformersInference as BrowserInference,
+  type BrowserModel,
+  BROWSER_MODELS,
+  checkWebGPU,
+  checkTransformersAvailable,
+  type ScoredLogProb,
+  type ScoredLogProbProvenance,
+} from "./transformers-inference";