@euconform/core 1.5.0 → 1.6.0

package/src/inference/browser-inference.ts

@@ -1,258 +1,16 @@
 /**
- * Browser-based inference using transformers.js
- * For small models (≤3B params) that can run via WebGPU
- *
- * STRICT MODE: Real inference only. No demo data.
+ * @deprecated Kept for backward compatibility (apps/web, existing tests import
+ * `BrowserInference` from this path). The canonical implementation now lives in
+ * `transformers-inference.ts` under the name `TransformersInference` — see
+ * Messmotor-Sanierung Task 2. New code should import `TransformersInference`
+ * from "./transformers-inference" directly.
  */
-
-// We need to use 'any' for transformers objects because the types are not fully exported
-/* eslint-disable @typescript-eslint/no-explicit-any */
-
-export interface BrowserModel {
-  id: string;
-  name: string;
-  size: string;
-  supported: boolean;
-}
-
-// Supported models for browser inference
-export const BROWSER_MODELS: BrowserModel[] = [
-  {
-    id: "Xenova/phi-3-mini-4k-instruct",
-    name: "Phi-3 Mini",
-    size: "3.8B",
-    supported: true,
-  },
-  {
-    id: "Xenova/Phi-3.5-mini-instruct",
-    name: "Phi-3.5 Mini",
-    size: "3.8B",
-    supported: true,
-  },
-  {
-    id: "Xenova/gpt2",
-    name: "GPT-2",
-    size: "124M",
-    supported: true,
-  },
-  {
-    id: "Xenova/distilgpt2",
-    name: "DistilGPT-2",
-    size: "82M",
-    supported: true,
-  },
-];
-
-/**
- * Check if WebGPU is available in the browser
- */
-export async function checkWebGPU(): Promise<boolean> {
-  if (typeof navigator === "undefined") return false;
-  if (typeof window === "undefined") return false;
-
-  try {
-    // Check for WebGPU support
-    const gpu = (navigator as Navigator & { gpu?: unknown }).gpu;
-    if (!gpu) return false;
-    // Try to get adapter
-    const adapter = await (gpu as { requestAdapter: () => Promise<unknown> }).requestAdapter();
-    return adapter !== null;
-  } catch {
-    return false;
-  }
-}
-
-/**
- * Check if transformers.js is available
- */
-export async function checkTransformersAvailable(): Promise<boolean> {
-  try {
-    // Try dynamic import - this will fail if package not installed
-    await import("@xenova/transformers");
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-/**
- * Browser inference engine using transformers.js
- * STRICT MODE: Throws error if package not installed.
- */
-export class BrowserInference {
-  public readonly modelId: string;
-  private model: unknown = null;
-  private tokenizer: unknown = null;
-  private pipeline: unknown = null; // Keep pipeline for generation
-  private loadingPromise: Promise<void> | null = null;
-  private loadError: Error | null = null;
-  private statusCallback: ((status: string) => void) | null = null;
-
-  constructor(
-    modelId = "Xenova/distilgpt2",
-    statusCallback: ((status: string) => void) | null = null
-  ) {
-    this.modelId = modelId;
-    this.statusCallback = statusCallback;
-  }
-
-  /**
-   * Check if browser inference is available
-   */
-  async isAvailable(): Promise<boolean> {
-    const hasTransformers = await checkTransformersAvailable();
-    return hasTransformers;
-  }
-
-  /**
-   * Load the model (lazy loading)
-   * Handles concurrency with loadingPromise
-   */
-  async load(): Promise<void> {
-    if (this.model && this.tokenizer) return;
-    if (this.loadingPromise) return this.loadingPromise;
-    if (this.loadError) throw this.loadError;
-
-    this.loadingPromise = this.performLoad().finally(() => {
-      this.loadingPromise = null;
-    });
-
-    return this.loadingPromise;
-  }
-
-  private async performLoad() {
-    if (this.statusCallback) this.statusCallback("loading");
-
-    try {
-      const { AutoTokenizer, AutoModelForCausalLM, pipeline } = await import(
-        "@xenova/transformers"
-      );
-
-      const hasWebGPU = await checkWebGPU();
-      const device = hasWebGPU ? "webgpu" : "cpu";
-
-      if (this.statusCallback) this.statusCallback("loading-tokenizer");
-      this.tokenizer = await AutoTokenizer.from_pretrained(this.modelId);
-
-      if (this.statusCallback) this.statusCallback("loading-model");
-      // biome-ignore lint/suspicious/noExplicitAny: library internal
-      this.model = await (AutoModelForCausalLM as any).from_pretrained(this.modelId, {
-        device,
-        quantized: true,
-      });
-
-      if (this.statusCallback) this.statusCallback("loading-pipeline");
-      // biome-ignore lint/suspicious/noExplicitAny: library internal
-      this.pipeline = await (pipeline as any)("text-generation", this.modelId, {
-        device,
-        model: this.model,
-        tokenizer: this.tokenizer,
-      });
-
-      if (this.statusCallback) this.statusCallback("ready");
-    } catch (err) {
-      this.loadError = err instanceof Error ? err : new Error(String(err));
-      if (this.statusCallback) this.statusCallback("error");
-      throw this.loadError;
-    }
-  }
-
-  /**
-   * Generate text
-   */
-  async generate(prompt: string, maxTokens = 50): Promise<string> {
-    await this.load();
-
-    if (!this.pipeline) {
-      throw new Error("Model not loaded");
-    }
-
-    // biome-ignore lint/suspicious/noExplicitAny: library internal
-    const result = await (this.pipeline as any)(prompt, {
-      max_new_tokens: maxTokens,
-      temperature: 0.7,
-      do_sample: true,
-    });
-
-    return result[0]?.generated_text || "";
-  }
-
-  /**
-   * Calculate Log-Probability for a given text
-   * This is the "Gold Standard" for bias testing.
-   * We calculate the loss (negative log likelihood) of the text.
-   */
-  async getLogProb(text: string): Promise<number> {
-    await this.load();
-
-    if (!this.tokenizer || !this.model) {
-      throw new Error("Model not loaded");
-    }
-
-    // biome-ignore lint/suspicious/noExplicitAny: internal transformers.js types
-    const inputs = await (this.tokenizer as any)(text, { return_tensors: "pt" });
-    const inputIds = inputs.input_ids;
-
-    // biome-ignore lint/suspicious/noExplicitAny: internal transformers.js types
-    const outputs = await (this.model as any)(inputs);
-    const logits = outputs.logits;
-
-    // 3. Compute Log Likelihood
-    // We want to calculate P(text) = P(t1) * P(t2|t1) * ...
-    // LogProb = sum( log( P(ti | t_<i) ) )
-
-    // logits shape: [batch_size, seq_len, vocab_size]
-    // inputIds shape: [batch_size, seq_len]
-    // We are interested in the probability of the *next* token given previous tokens.
-    // So we look at logits[0, i, :] to predict inputIds[0, i+1]
-
-    const data = logits.data; // Float32Array
-    const dims = logits.dims; // [batch, seq, vocab]
-    const vocabSize = dims[2];
-    const seqLen = dims[1];
-
-    let totalLogProb = 0;
-
-    // Helper to get logit value
-    const getLogit = (seqIdx: number, vocabIdx: number) => {
-      return data[seqIdx * vocabSize + vocabIdx];
-    };
-
-    // Helper for softmax/logsoftmax
-    // We only need the log_prob of the target token
-    // log_softmax(x_i) = x_i - log(sum(exp(x_j)))
-    // To avoid overflow: x_i - max(x) - log(sum(exp(x_j - max(x))))
-
-    for (let i = 0; i < seqLen - 1; i++) {
-      const targetTokenId = Number(inputIds.data[i + 1]);
-
-      // Calculate max logit for numerical stability
-      let maxLogit = Number.NEGATIVE_INFINITY;
-      for (let j = 0; j < vocabSize; j++) {
-        const val = getLogit(i, j);
-        if (val > maxLogit) maxLogit = val;
-      }
-
-      // Calculate sum of exp
-      let sumExp = 0;
-      for (let j = 0; j < vocabSize; j++) {
-        sumExp += Math.exp(getLogit(i, j) - maxLogit);
-      }
-
-      const logSumExp = maxLogit + Math.log(sumExp);
-      const targetLogit = getLogit(i, targetTokenId);
-
-      const logProb = targetLogit - logSumExp;
-      totalLogProb += logProb;
-    }
-
-    return totalLogProb;
-  }
-
-  /**
-   * Get list of supported browser models
-   */
-  static getSupportedModels(): BrowserModel[] {
-    return BROWSER_MODELS.filter((m) => m.supported);
-  }
-}
+export {
+  TransformersInference as BrowserInference,
+  type BrowserModel,
+  BROWSER_MODELS,
+  checkWebGPU,
+  checkTransformersAvailable,
+  type ScoredLogProb,
+  type ScoredLogProbProvenance,
+} from "./transformers-inference";

package/src/inference/capability-detection.ts

@@ -6,9 +6,21 @@
 import type { BiasCalculationMethod } from "../types";
 
 /**
- * Detection status for model capabilities
+ * Detection status for model capabilities.
+ *
+ * "unverified": the model is declared supported (e.g. it's in the static
+ * BROWSER_MODELS list) but has NOT actually been loaded/tested — no request
+ * was made, no capability was measured. Distinct from "available", which
+ * means a real detection/measurement round confirmed it. Do not report
+ * "available" for a model that was never tested (see Messmotor-Sanierung
+ * Task 2): the runtime status is what a consumer reads, not source comments.
  */
-export type CapabilityDetectionStatus = "detecting" | "available" | "unavailable" | "error";
+export type CapabilityDetectionStatus =
+  | "detecting"
+  | "available"
+  | "unverified"
+  | "unavailable"
+  | "error";
 
 /**
  * Model capability information
@@ -105,15 +117,19 @@ export class DefaultCapabilityDetectionService implements CapabilityDetectionSer
       // Import browser models dynamically
       const { BROWSER_MODELS } = await import("./browser-inference");
 
+      // These entries are declared from the static BROWSER_MODELS list, not
+      // measured — no model has actually been loaded/tested at this point.
+      // status: "unverified" (not "available") reflects that honestly; no
+      // lastTested since no test ran. A real per-model verification step is
+      // future work, not this task's scope.
       for (const model of BROWSER_MODELS) {
         if (!model.supported) continue;
         capabilities.push({
           modelId: model.id,
           engine: "browser",
           method: "logprobs_exact",
-          status: "available",
+          status: "unverified",
           recommended: true,
-          lastTested: new Date().toISOString(),
         });
       }
     } catch (error) {
@@ -123,9 +139,8 @@ export class DefaultCapabilityDetectionService implements CapabilityDetectionSer
         modelId: "Xenova/distilgpt2",
         engine: "browser",
         method: "logprobs_exact",
-        status: "available",
+        status: "unverified",
         recommended: true,
-        lastTested: new Date().toISOString(),
       });
     }
 
@@ -189,15 +204,14 @@ export class DefaultCapabilityDetectionService implements CapabilityDetectionSer
       return cached.capability;
     }
 
-    // Browser inference always supports exact log-probabilities
+    // Declared, not measured — no model has been loaded/tested here.
     if (engine === "browser") {
       const capability: ModelCapability = {
         modelId,
         engine: "browser",
         method: "logprobs_exact",
-        status: "available",
+        status: "unverified",
         recommended: true,
-        lastTested: new Date().toISOString(),
       };
 
       this.setCachedCapability(capability);
@@ -360,7 +374,9 @@ export class DefaultCapabilityDetectionService implements CapabilityDetectionSer
    * Calculate recommendation score for a capability
    */
   protected getRecommendationScore(capability: ModelCapability): number {
-    if (capability.status !== "available") return 0;
+    // "unverified" is a usable, declared-supported model (see the type doc) —
+    // only truly broken/unknown states (unavailable, error, detecting) score 0.
+    if (capability.status !== "available" && capability.status !== "unverified") return 0;
 
     let score = 0;
 
@@ -398,15 +414,14 @@ export class EnhancedCapabilityDetectionService extends DefaultCapabilityDetecti
       return cached.capability;
     }
 
-    // Browser inference always supports exact log-probabilities
+    // Declared, not measured — no model has been loaded/tested here.
     if (engine === "browser") {
       const capability: ModelCapability = {
         modelId,
         engine: "browser",
         method: "logprobs_exact",
-        status: "available",
+        status: "unverified",
         recommended: true,
-        lastTested: new Date().toISOString(),
       };
 
       this.setCachedCapability(capability);
@@ -509,7 +524,8 @@ export class EnhancedCapabilityDetectionService extends DefaultCapabilityDetecti
   async detectAllCapabilities(): Promise<ModelCapability[]> {
     const capabilities: ModelCapability[] = [];
 
-    // Add all supported browser models dynamically
+    // Add all supported browser models dynamically. Declared, not measured —
+    // no model has actually been loaded/tested at this point.
     try {
       const { BROWSER_MODELS } = await import("./browser-inference");
 
@@ -519,9 +535,8 @@ export class EnhancedCapabilityDetectionService extends DefaultCapabilityDetecti
           modelId: model.id,
           engine: "browser",
           method: "logprobs_exact",
-          status: "available",
+          status: "unverified",
           recommended: true,
-          lastTested: new Date().toISOString(),
         });
       }
     } catch (error) {
@@ -531,9 +546,8 @@ export class EnhancedCapabilityDetectionService extends DefaultCapabilityDetecti
         modelId: "Xenova/distilgpt2",
         engine: "browser",
         method: "logprobs_exact",
-        status: "available",
+        status: "unverified",
         recommended: true,
-        lastTested: new Date().toISOString(),
       });
     }
 

package/src/inference/index.ts

@@ -3,13 +3,26 @@
  * Auto-detects available inference engines and provides unified interface
  */
 
+import { OllamaClient, type OllamaModel, detectOllama, listOllamaModels } from "./ollama-client";
 import {
   BROWSER_MODELS,
-  BrowserInference,
   type BrowserModel,
+  type ScoredLogProb,
+  TransformersInference,
   checkWebGPU,
-} from "./browser-inference";
-import { OllamaClient, type OllamaModel, detectOllama, listOllamaModels } from "./ollama-client";
+} from "./transformers-inference";
+
+/**
+ * The one contract a bias-measurement engine must satisfy: a real, cross-checked
+ * sentence log-probability (see Task 1 / Messmotor-Sanierung). `OllamaClient`
+ * deliberately does NOT implement this — Ollama's logprobs only cover generated
+ * tokens, never prompt/sentence log-probability, so it cannot honestly produce a
+ * `ScoredLogProb`. TypeScript's structural typing then locks it out of any API
+ * that requires this interface (e.g. `calculateCrowsPairsBias`).
+ */
+export interface SequenceLogProbProvider {
+  getScoredLogProb(text: string): Promise<ScoredLogProb>;
+}
 
 export type InferenceEngine = "ollama" | "browser" | "demo";
 
@@ -85,7 +98,7 @@ export function createInferenceClient(
     }
 
     case "browser": {
-      const browserClient = new BrowserInference(modelId, statusCallback);
+      const browserClient = new TransformersInference(modelId, statusCallback);
       return {
         engine: "browser",
         modelId,
@@ -138,7 +151,22 @@ export {
   OllamaClient,
   type CapabilityCache,
 } from "./ollama-client";
-export { BrowserInference, checkWebGPU, BROWSER_MODELS } from "./browser-inference";
+export {
+  TransformersInference,
+  checkWebGPU,
+  BROWSER_MODELS,
+  type ScoredLogProb,
+  type ScoredLogProbProvenance,
+} from "./transformers-inference";
+// Architecture detection for the Gemma-4 template-mode scorer, re-exported so callers that
+// must pick a dtype BEFORE constructing `TransformersInference` (e.g. the CLI's
+// `run-bias-test.ts`, which needs to choose "q4" over "fp32" for conditional-generation
+// models — fp32 Gemma-4 weights were measured and rejected as impractically large in
+// Messmotor-Sanierung Task 5) can reuse the same detection `TransformersInference` uses
+// internally, instead of duplicating the architecture check.
+export { isConditionalGenerationConfig, type MinimalModelConfig } from "./template-scoring";
+// Backward-compat alias: existing consumers (apps/web, tests) import `BrowserInference`.
+export { BrowserInference } from "./browser-inference";
 export {
   InferenceFactory,
   createInferenceFactory,

package/src/inference/inference-factory.ts

@@ -3,7 +3,6 @@
  * Provides unified interface for bias calculations across engines with automatic method detection
  */
 
-import { calculateCrowsPairsBias } from "../metrics/crows-pairs-bias";
 import type { BiasCalculationMethod, BiasTestConfig, CrowsPairsBiasResult } from "../types";
 import { BrowserInference } from "./browser-inference";
 import type { ModelCapability } from "./capability-detection";
@@ -42,38 +41,18 @@ export class InferenceFactory {
   }
 
   /**
-   * Create a bias calculation with automatic method detection
-   * Returns InferenceResult with both data and method metadata
+   * @deprecated Uses the removed Ollama logprob path: this method auto-selected
+   * between browser and Ollama engines via `getInferenceClient`, which no longer
+   * measures anything honest for Ollama (see Messmotor-Sanierung Task 2). The
+   * auto-selection logic itself needs a real redesign, not a throw-or-convert
+   * fix — see bias --engine transformers for the supported path.
    */
   async createBiasCalculation(
-    config: BiasTestConfig
+    _config: BiasTestConfig
   ): Promise<InferenceResult<CrowsPairsBiasResult>> {
-    const { engine, model } = config;
-
-    this.validateConfiguration(config);
-
-    const inferenceClient = await this.getInferenceClient(engine, model);
-    let _detectedMethod: BiasCalculationMethod;
-
-    if (engine === "browser") {
-      _detectedMethod = "logprobs_exact";
-    } else {
-      // Use capability method if available, otherwise detect
-      _detectedMethod =
-        this.selectedCapability?.method ?? (await this.detectBestMethod(inferenceClient));
-    }
-
-    // Perform bias calculation
-    const biasResult = await calculateCrowsPairsBias(config, inferenceClient);
-
-    // Return result with metadata including capability information
-    return {
-      data: biasResult,
-      method: biasResult.method, // Use the method determined during calculation
-      engine,
-      model,
-      capability: this.selectedCapability || undefined,
-    };
+    throw new Error(
+      "InferenceFactory.createBiasCalculation verwendet den entfernten Ollama-Logprob-Pfad; siehe bias --engine transformers."
+    );
   }
 
   /**

package/src/inference/ollama-client.ts

@@ -270,6 +270,9 @@ export class OllamaClient {
     return response.response;
   }
 
+  /**
+   * @deprecated Not a sentence log-probability — see `getLogProbWithFallback`.
+   */
   async getLogProb(prompt: string): Promise<number> {
     // Use the enhanced getLogProbWithFallback method that supports both exact and fallback methods
     const result = await this.getLogProbWithFallback(prompt);
@@ -281,6 +284,12 @@ export class OllamaClient {
   }
 
   /**
+   * @deprecated Part of the removed Ollama bias-measurement path (Messmotor-Sanierung
+   * Task 2): Ollama's logprobs cover only generated tokens, never the prompt/sentence,
+   * so this "capability" check never verified a real measurement. Kept only because
+   * `scan`'s non-bias paths and the capability-cache tests still reference it — not
+   * a foundation to build on. See bias --engine transformers for the real path.
+   *
    * Test if the current model supports log-probabilities
    * Sends a test request with "Der" prompt to detect capability
    */
@@ -405,6 +414,9 @@ export class OllamaClient {
   }
 
   /**
+   * @deprecated Part of the removed Ollama bias-measurement path (see
+   * `testLogProbCapability`); no longer used by `calculateCrowsPairsBias`.
+   *
    * Detect if the current model supports log-probabilities
    * Checks cache first, then runs test if needed
    */
@@ -429,6 +441,10 @@ export class OllamaClient {
     return supports;
   }
 
+  /**
+   * @deprecated Part of the removed Ollama bias-measurement path; see
+   * `testLogProbCapability`. Not a real sentence log-probability.
+   */
   private async tryExactLogProbs(
     prompt: string
   ): Promise<{ logprob: number; method: BiasCalculationMethod } | null> {
@@ -477,6 +493,11 @@ export class OllamaClient {
     }
   }
 
+  /**
+   * @deprecated Part of the removed Ollama bias-measurement path; this is a
+   * latency proxy, not a probability — the exact thing Messmotor-Sanierung
+   * removes from the bias measurement path. Do not build on this.
+   */
   private async latencyFallback(
     prompt: string
   ): Promise<{ logprob: number; method: BiasCalculationMethod }> {
@@ -519,6 +540,14 @@ export class OllamaClient {
   }
 
   /**
+   * @deprecated Removed from the bias-measurement path (Messmotor-Sanierung Task 2):
+   * Ollama returns logprobs only for generated tokens, never for the prompt/sentence
+   * being scored, so this never measured a sentence log-probability — even the
+   * "exact" branch (`tryExactLogProbs`) is not what its name implies, and the
+   * fallback is a latency proxy. Verified: r(sentence length, logP)=0.019 where a
+   * real sentence-probability measure shows −0.96. Use `TransformersInference.
+   * getScoredLogProb` (via `bias --engine transformers`) instead.
+   *
    * Get log-probability with automatic fallback to latency method
    * Returns both the calculated value and the method used
    */

package/src/inference/scored-log-prob-types.ts

@@ -0,0 +1,66 @@
+/**
+ * Shared result types for `SequenceLogProbProvider.getScoredLogProb`.
+ *
+ * Pulled out of `transformers-inference.ts` (Messmotor-Sanierung Task 5) so that
+ * `template-scoring.ts` (the Gemma-4 template-mode scorer) and `transformers-inference.ts`
+ * (which loads models and picks which mode to use) can both depend on these types without
+ * either module importing from the other — avoids a circular import between the two.
+ *
+ * `transformers-inference.ts` re-exports both types from here, so existing imports of
+ * `ScoredLogProb`/`ScoredLogProbProvenance` from `"./transformers-inference"` keep working.
+ */
+
+import type { SequenceLogProbMethod } from "../types";
+import type { Span } from "./sequence-logprob";
+
+/** Provenance metadata attached to a scored log-probability result. */
+export interface ScoredLogProbProvenance {
+  modelId: string;
+  /**
+   * The CONFIGURED weight precision passed to `from_pretrained` (e.g. "q4", "fp32", "q8") —
+   * NOT the output tensor's reported dtype (see `logitsDtype` for that; the two are
+   * different facts and must not be conflated). Task 1 measured 1.3-11.9% score deviation
+   * between quantized and fp32 weights on the same model/sentences, so this field is
+   * ergebnisrelevant: an artifact that cannot say which weight precision produced a score
+   * claims a precision it does not have (Messmotor-Sanierung Task 5 review finding).
+   */
+  dtype: string;
+  /**
+   * The OUTPUT LOGITS TENSOR's reported `.type` (e.g. "float32"). The Task 0 spike found
+   * this stays "float32" even for q4-quantized weights ("logits.data ist Float32Array trotz
+   * q4-Gewichten") — so this field does NOT tell you the weight precision; use `dtype` for
+   * that. Kept because it is still a real, independently useful fact (confirms no silent
+   * downcast in the output itself).
+   */
+  logitsDtype: string;
+  backend: string;
+  libraryVersion: string;
+  /**
+   * "none" for the base autoregressive mode (Task 1); "fixed_template_span" for the
+   * instruction-tuned template mode (Task 5), which conditions scoring on a fixed
+   * user-turn prompt. Optional so pre-Task-5 test doubles that predate this field
+   * (e.g. hand-built `ScoredLogProb` literals in other packages' tests) stay valid;
+   * every real measurement path sets it explicitly.
+   */
+  templateStrategy?: "none" | "fixed_template_span";
+  /** Present only when templateStrategy is "fixed_template_span": the verbatim fixed
+   * context string fed before the sentence — recomputable by any third party. */
+  promptText?: string;
+  /** SHA-256 of `promptText`, present only when templateStrategy is "fixed_template_span". */
+  promptSha256?: string;
+}
+
+/** Full result of scoring a sentence with `getScoredLogProb`. */
+export interface ScoredLogProb {
+  score: number;
+  scoredTokenIds: number[];
+  span: Span;
+  contextTokenHash: string;
+  provenance: ScoredLogProbProvenance;
+  /**
+   * How the score was accumulated (Task 3's `SequenceLogProbMethod`). Optional for the
+   * same backward-compatibility reason as `provenance.templateStrategy`; every real
+   * measurement path sets it explicitly.
+   */
+  method?: SequenceLogProbMethod;
+}