@euconform/core 1.4.0 → 1.6.0

package/src/inference/capability-detection.ts

@@ -6,9 +6,21 @@
 import type { BiasCalculationMethod } from "../types";
 
 /**
- * Detection status for model capabilities
+ * Detection status for model capabilities.
+ *
+ * "unverified": the model is declared supported (e.g. it's in the static
+ * BROWSER_MODELS list) but has NOT actually been loaded/tested — no request
+ * was made, no capability was measured. Distinct from "available", which
+ * means a real detection/measurement round confirmed it. Do not report
+ * "available" for a model that was never tested (see Messmotor-Sanierung
+ * Task 2): the runtime status is what a consumer reads, not source comments.
  */
-export type CapabilityDetectionStatus = "detecting" | "available" | "unavailable" | "error";
+export type CapabilityDetectionStatus =
+  | "detecting"
+  | "available"
+  | "unverified"
+  | "unavailable"
+  | "error";
 
 /**
  * Model capability information
@@ -105,15 +117,19 @@ export class DefaultCapabilityDetectionService implements CapabilityDetectionSer
       // Import browser models dynamically
       const { BROWSER_MODELS } = await import("./browser-inference");
 
+      // These entries are declared from the static BROWSER_MODELS list, not
+      // measured — no model has actually been loaded/tested at this point.
+      // status: "unverified" (not "available") reflects that honestly; no
+      // lastTested since no test ran. A real per-model verification step is
+      // future work, not this task's scope.
       for (const model of BROWSER_MODELS) {
         if (!model.supported) continue;
         capabilities.push({
           modelId: model.id,
           engine: "browser",
           method: "logprobs_exact",
-          status: "available",
+          status: "unverified",
           recommended: true,
-          lastTested: new Date().toISOString(),
         });
       }
     } catch (error) {
@@ -123,9 +139,8 @@ export class DefaultCapabilityDetectionService implements CapabilityDetectionSer
         modelId: "Xenova/distilgpt2",
         engine: "browser",
         method: "logprobs_exact",
-        status: "available",
+        status: "unverified",
         recommended: true,
-        lastTested: new Date().toISOString(),
       });
     }
 
@@ -189,15 +204,14 @@ export class DefaultCapabilityDetectionService implements CapabilityDetectionSer
       return cached.capability;
     }
 
-    // Browser inference always supports exact log-probabilities
+    // Declared, not measured — no model has been loaded/tested here.
     if (engine === "browser") {
       const capability: ModelCapability = {
         modelId,
         engine: "browser",
         method: "logprobs_exact",
-        status: "available",
+        status: "unverified",
         recommended: true,
-        lastTested: new Date().toISOString(),
       };
 
       this.setCachedCapability(capability);
@@ -360,7 +374,9 @@ export class DefaultCapabilityDetectionService implements CapabilityDetectionSer
    * Calculate recommendation score for a capability
    */
   protected getRecommendationScore(capability: ModelCapability): number {
-    if (capability.status !== "available") return 0;
+    // "unverified" is a usable, declared-supported model (see the type doc) —
+    // only truly broken/unknown states (unavailable, error, detecting) score 0.
+    if (capability.status !== "available" && capability.status !== "unverified") return 0;
 
     let score = 0;
 
@@ -398,15 +414,14 @@ export class EnhancedCapabilityDetectionService extends DefaultCapabilityDetecti
       return cached.capability;
     }
 
-    // Browser inference always supports exact log-probabilities
+    // Declared, not measured — no model has been loaded/tested here.
     if (engine === "browser") {
       const capability: ModelCapability = {
         modelId,
         engine: "browser",
         method: "logprobs_exact",
-        status: "available",
+        status: "unverified",
         recommended: true,
-        lastTested: new Date().toISOString(),
       };
 
       this.setCachedCapability(capability);
@@ -509,7 +524,8 @@ export class EnhancedCapabilityDetectionService extends DefaultCapabilityDetecti
   async detectAllCapabilities(): Promise<ModelCapability[]> {
     const capabilities: ModelCapability[] = [];
 
-    // Add all supported browser models dynamically
+    // Add all supported browser models dynamically. Declared, not measured —
+    // no model has actually been loaded/tested at this point.
     try {
       const { BROWSER_MODELS } = await import("./browser-inference");
 
@@ -519,9 +535,8 @@ export class EnhancedCapabilityDetectionService extends DefaultCapabilityDetecti
           modelId: model.id,
           engine: "browser",
           method: "logprobs_exact",
-          status: "available",
+          status: "unverified",
           recommended: true,
-          lastTested: new Date().toISOString(),
         });
       }
     } catch (error) {
@@ -531,9 +546,8 @@ export class EnhancedCapabilityDetectionService extends DefaultCapabilityDetecti
         modelId: "Xenova/distilgpt2",
         engine: "browser",
         method: "logprobs_exact",
-        status: "available",
+        status: "unverified",
         recommended: true,
-        lastTested: new Date().toISOString(),
       });
     }
 

package/src/inference/index.ts

@@ -3,13 +3,26 @@
  * Auto-detects available inference engines and provides unified interface
  */
 
+import { OllamaClient, type OllamaModel, detectOllama, listOllamaModels } from "./ollama-client";
 import {
   BROWSER_MODELS,
-  BrowserInference,
   type BrowserModel,
+  type ScoredLogProb,
+  TransformersInference,
   checkWebGPU,
-} from "./browser-inference";
-import { OllamaClient, type OllamaModel, detectOllama, listOllamaModels } from "./ollama-client";
+} from "./transformers-inference";
+
+/**
+ * The one contract a bias-measurement engine must satisfy: a real, cross-checked
+ * sentence log-probability (see Task 1 / Messmotor-Sanierung). `OllamaClient`
+ * deliberately does NOT implement this — Ollama's logprobs only cover generated
+ * tokens, never prompt/sentence log-probability, so it cannot honestly produce a
+ * `ScoredLogProb`. TypeScript's structural typing then locks it out of any API
+ * that requires this interface (e.g. `calculateCrowsPairsBias`).
+ */
+export interface SequenceLogProbProvider {
+  getScoredLogProb(text: string): Promise<ScoredLogProb>;
+}
 
 export type InferenceEngine = "ollama" | "browser" | "demo";
 
@@ -85,7 +98,7 @@ export function createInferenceClient(
     }
 
     case "browser": {
-      const browserClient = new BrowserInference(modelId, statusCallback);
+      const browserClient = new TransformersInference(modelId, statusCallback);
       return {
         engine: "browser",
         modelId,
@@ -138,7 +151,22 @@ export {
   OllamaClient,
   type CapabilityCache,
 } from "./ollama-client";
-export { BrowserInference, checkWebGPU, BROWSER_MODELS } from "./browser-inference";
+export {
+  TransformersInference,
+  checkWebGPU,
+  BROWSER_MODELS,
+  type ScoredLogProb,
+  type ScoredLogProbProvenance,
+} from "./transformers-inference";
+// Architecture detection for the Gemma-4 template-mode scorer, re-exported so callers that
+// must pick a dtype BEFORE constructing `TransformersInference` (e.g. the CLI's
+// `run-bias-test.ts`, which needs to choose "q4" over "fp32" for conditional-generation
+// models — fp32 Gemma-4 weights were measured and rejected as impractically large in
+// Messmotor-Sanierung Task 5) can reuse the same detection `TransformersInference` uses
+// internally, instead of duplicating the architecture check.
+export { isConditionalGenerationConfig, type MinimalModelConfig } from "./template-scoring";
+// Backward-compat alias: existing consumers (apps/web, tests) import `BrowserInference`.
+export { BrowserInference } from "./browser-inference";
 export {
   InferenceFactory,
   createInferenceFactory,

package/src/inference/inference-factory.ts

@@ -3,7 +3,6 @@
  * Provides unified interface for bias calculations across engines with automatic method detection
  */
 
-import { calculateCrowsPairsBias } from "../metrics/crows-pairs-bias";
 import type { BiasCalculationMethod, BiasTestConfig, CrowsPairsBiasResult } from "../types";
 import { BrowserInference } from "./browser-inference";
 import type { ModelCapability } from "./capability-detection";
@@ -42,38 +41,18 @@ export class InferenceFactory {
   }
 
   /**
-   * Create a bias calculation with automatic method detection
-   * Returns InferenceResult with both data and method metadata
+   * @deprecated Uses the removed Ollama logprob path: this method auto-selected
+   * between browser and Ollama engines via `getInferenceClient`, which no longer
+   * measures anything honest for Ollama (see Messmotor-Sanierung Task 2). The
+   * auto-selection logic itself needs a real redesign, not a throw-or-convert
+   * fix — see bias --engine transformers for the supported path.
    */
   async createBiasCalculation(
-    config: BiasTestConfig
+    _config: BiasTestConfig
   ): Promise<InferenceResult<CrowsPairsBiasResult>> {
-    const { engine, model } = config;
-
-    this.validateConfiguration(config);
-
-    const inferenceClient = await this.getInferenceClient(engine, model);
-    let _detectedMethod: BiasCalculationMethod;
-
-    if (engine === "browser") {
-      _detectedMethod = "logprobs_exact";
-    } else {
-      // Use capability method if available, otherwise detect
-      _detectedMethod =
-        this.selectedCapability?.method ?? (await this.detectBestMethod(inferenceClient));
-    }
-
-    // Perform bias calculation
-    const biasResult = await calculateCrowsPairsBias(config, inferenceClient);
-
-    // Return result with metadata including capability information
-    return {
-      data: biasResult,
-      method: biasResult.method, // Use the method determined during calculation
-      engine,
-      model,
-      capability: this.selectedCapability || undefined,
-    };
+    throw new Error(
+      "InferenceFactory.createBiasCalculation verwendet den entfernten Ollama-Logprob-Pfad; siehe bias --engine transformers."
+    );
   }
 
   /**

package/src/inference/ollama-client.ts

@@ -270,6 +270,9 @@ export class OllamaClient {
     return response.response;
   }
 
+  /**
+   * @deprecated Not a sentence log-probability — see `getLogProbWithFallback`.
+   */
   async getLogProb(prompt: string): Promise<number> {
     // Use the enhanced getLogProbWithFallback method that supports both exact and fallback methods
     const result = await this.getLogProbWithFallback(prompt);
@@ -281,6 +284,12 @@ export class OllamaClient {
   }
 
   /**
+   * @deprecated Part of the removed Ollama bias-measurement path (Messmotor-Sanierung
+   * Task 2): Ollama's logprobs cover only generated tokens, never the prompt/sentence,
+   * so this "capability" check never verified a real measurement. Kept only because
+   * `scan`'s non-bias paths and the capability-cache tests still reference it — not
+   * a foundation to build on. See bias --engine transformers for the real path.
+   *
    * Test if the current model supports log-probabilities
    * Sends a test request with "Der" prompt to detect capability
    */
@@ -405,6 +414,9 @@ export class OllamaClient {
   }
 
   /**
+   * @deprecated Part of the removed Ollama bias-measurement path (see
+   * `testLogProbCapability`); no longer used by `calculateCrowsPairsBias`.
+   *
    * Detect if the current model supports log-probabilities
    * Checks cache first, then runs test if needed
    */
@@ -429,6 +441,10 @@ export class OllamaClient {
     return supports;
   }
 
+  /**
+   * @deprecated Part of the removed Ollama bias-measurement path; see
+   * `testLogProbCapability`. Not a real sentence log-probability.
+   */
   private async tryExactLogProbs(
     prompt: string
   ): Promise<{ logprob: number; method: BiasCalculationMethod } | null> {
@@ -477,6 +493,11 @@ export class OllamaClient {
     }
   }
 
+  /**
+   * @deprecated Part of the removed Ollama bias-measurement path; this is a
+   * latency proxy, not a probability — the exact thing Messmotor-Sanierung
+   * removes from the bias measurement path. Do not build on this.
+   */
   private async latencyFallback(
     prompt: string
   ): Promise<{ logprob: number; method: BiasCalculationMethod }> {
@@ -519,6 +540,14 @@ export class OllamaClient {
   }
 
   /**
+   * @deprecated Removed from the bias-measurement path (Messmotor-Sanierung Task 2):
+   * Ollama returns logprobs only for generated tokens, never for the prompt/sentence
+   * being scored, so this never measured a sentence log-probability — even the
+   * "exact" branch (`tryExactLogProbs`) is not what its name implies, and the
+   * fallback is a latency proxy. Verified: r(sentence length, logP)=0.019 where a
+   * real sentence-probability measure shows −0.96. Use `TransformersInference.
+   * getScoredLogProb` (via `bias --engine transformers`) instead.
+   *
    * Get log-probability with automatic fallback to latency method
    * Returns both the calculated value and the method used
    */

package/src/inference/scored-log-prob-types.ts

@@ -0,0 +1,66 @@
+/**
+ * Shared result types for `SequenceLogProbProvider.getScoredLogProb`.
+ *
+ * Pulled out of `transformers-inference.ts` (Messmotor-Sanierung Task 5) so that
+ * `template-scoring.ts` (the Gemma-4 template-mode scorer) and `transformers-inference.ts`
+ * (which loads models and picks which mode to use) can both depend on these types without
+ * either module importing from the other — avoids a circular import between the two.
+ *
+ * `transformers-inference.ts` re-exports both types from here, so existing imports of
+ * `ScoredLogProb`/`ScoredLogProbProvenance` from `"./transformers-inference"` keep working.
+ */
+
+import type { SequenceLogProbMethod } from "../types";
+import type { Span } from "./sequence-logprob";
+
+/** Provenance metadata attached to a scored log-probability result. */
+export interface ScoredLogProbProvenance {
+  modelId: string;
+  /**
+   * The CONFIGURED weight precision passed to `from_pretrained` (e.g. "q4", "fp32", "q8") —
+   * NOT the output tensor's reported dtype (see `logitsDtype` for that; the two are
+   * different facts and must not be conflated). Task 1 measured 1.3-11.9% score deviation
+   * between quantized and fp32 weights on the same model/sentences, so this field is
+   * ergebnisrelevant: an artifact that cannot say which weight precision produced a score
+   * claims a precision it does not have (Messmotor-Sanierung Task 5 review finding).
+   */
+  dtype: string;
+  /**
+   * The OUTPUT LOGITS TENSOR's reported `.type` (e.g. "float32"). The Task 0 spike found
+   * this stays "float32" even for q4-quantized weights ("logits.data ist Float32Array trotz
+   * q4-Gewichten") — so this field does NOT tell you the weight precision; use `dtype` for
+   * that. Kept because it is still a real, independently useful fact (confirms no silent
+   * downcast in the output itself).
+   */
+  logitsDtype: string;
+  backend: string;
+  libraryVersion: string;
+  /**
+   * "none" for the base autoregressive mode (Task 1); "fixed_template_span" for the
+   * instruction-tuned template mode (Task 5), which conditions scoring on a fixed
+   * user-turn prompt. Optional so pre-Task-5 test doubles that predate this field
+   * (e.g. hand-built `ScoredLogProb` literals in other packages' tests) stay valid;
+   * every real measurement path sets it explicitly.
+   */
+  templateStrategy?: "none" | "fixed_template_span";
+  /** Present only when templateStrategy is "fixed_template_span": the verbatim fixed
+   * context string fed before the sentence — recomputable by any third party. */
+  promptText?: string;
+  /** SHA-256 of `promptText`, present only when templateStrategy is "fixed_template_span". */
+  promptSha256?: string;
+}
+
+/** Full result of scoring a sentence with `getScoredLogProb`. */
+export interface ScoredLogProb {
+  score: number;
+  scoredTokenIds: number[];
+  span: Span;
+  contextTokenHash: string;
+  provenance: ScoredLogProbProvenance;
+  /**
+   * How the score was accumulated (Task 3's `SequenceLogProbMethod`). Optional for the
+   * same backward-compatibility reason as `provenance.templateStrategy`; every real
+   * measurement path sets it explicitly.
+   */
+  method?: SequenceLogProbMethod;
+}

package/src/inference/sequence-logprob.ts

@@ -0,0 +1,136 @@
+/**
+ * Pure, offline-testable log-probability scorer for autoregressive language models.
+ *
+ * Scoring convention (binding):
+ * - A span [start, end) is half-open over token positions of the FULL fed sequence
+ *   (i.e. including any context tokens such as a prepended BOS).
+ * - The token AT position j is scored using the logits AT position j-1 (the model's
+ *   prediction distribution for "what comes next" is produced one position before the
+ *   token it predicts).
+ * - The result is a SUM of log-probabilities, not a mean.
+ * - batch_size must be exactly 1; anything else is a caller error and throws.
+ */
+
+/** Minimal shape this module needs from an onnxruntime-style Tensor. */
+export interface LogitsTensorLike {
+  /** Flattened [batch, seqLen, vocabSize] logits. Must be Float32Array or Float64Array. */
+  data: Float32Array | Float64Array;
+  /** [batchSize, seqLen, vocabSize] */
+  dims: readonly number[];
+}
+
+/** Half-open token-position span [start, end). */
+export interface Span {
+  start: number;
+  end: number;
+}
+
+function assertSupportedDtype(data: LogitsTensorLike["data"]): void {
+  if (!(data instanceof Float32Array) && !(data instanceof Float64Array)) {
+    const ctorName =
+      (data as { constructor?: { name?: string } })?.constructor?.name ?? typeof data;
+    throw new Error(
+      `sumSequenceLogProb: unsupported logits dtype, expected Float32Array or Float64Array, got ${ctorName}`
+    );
+  }
+}
+
+function assertBatchAndDims(
+  dims: readonly number[]
+): [batchSize: number, seqLen: number, vocabSize: number] {
+  if (dims.length !== 3) {
+    throw new Error(
+      `sumSequenceLogProb: expected logits dims [batch, seqLen, vocabSize], got [${dims.join(", ")}]`
+    );
+  }
+  const batchSize = dims[0] ?? 0;
+  const seqLen = dims[1] ?? 0;
+  const vocabSize = dims[2] ?? 0;
+  if (batchSize !== 1) {
+    throw new Error(`sumSequenceLogProb: batch_size must be 1, got ${batchSize}`);
+  }
+  return [batchSize, seqLen, vocabSize];
+}
+
+function assertSpanWithinBounds(span: Span, seqLen: number): void {
+  const { start, end } = span;
+  if (
+    !Number.isInteger(start) ||
+    !Number.isInteger(end) ||
+    start < 1 ||
+    end > seqLen ||
+    start >= end
+  ) {
+    throw new Error(
+      `sumSequenceLogProb: invalid span [${start}, ${end}) for seqLen ${seqLen}; span must satisfy integer 1 <= start < end <= seqLen`
+    );
+  }
+}
+
+function logSumExpRow(
+  data: LogitsTensorLike["data"],
+  rowStart: number,
+  vocabSize: number,
+  contextLabel: string
+): { maxLogit: number; logSumExp: number } {
+  let maxLogit = Number.NEGATIVE_INFINITY;
+  for (let v = 0; v < vocabSize; v++) {
+    const val = data[rowStart + v] ?? Number.NaN;
+    if (!Number.isFinite(val)) {
+      throw new Error(
+        `sumSequenceLogProb: non-finite logit at ${contextLabel}, vocab index ${v} (value: ${val})`
+      );
+    }
+    if (val > maxLogit) maxLogit = val;
+  }
+  let sumExp = 0;
+  for (let v = 0; v < vocabSize; v++) {
+    const val = data[rowStart + v] ?? Number.NaN;
+    sumExp += Math.exp(val - maxLogit);
+  }
+  return { maxLogit, logSumExp: maxLogit + Math.log(sumExp) };
+}
+
+/**
+ * Sum the log-probabilities of the tokens in `tokenIds` over the half-open span
+ * [span.start, span.end), where the token at position j is scored from the logits
+ * distribution at position j-1.
+ *
+ * Throws on: unsupported dtype, wrong dims shape, batch_size !== 1, tokenIds/seqLen
+ * mismatch, out-of-bounds span, out-of-vocab token ids, or non-finite logits.
+ */
+export function sumSequenceLogProb(
+  logits: LogitsTensorLike,
+  tokenIds: ArrayLike<number | bigint>,
+  span: Span
+): number {
+  assertSupportedDtype(logits.data);
+  const [, seqLen, vocabSize] = assertBatchAndDims(logits.dims);
+
+  if (tokenIds.length !== seqLen) {
+    throw new Error(
+      `sumSequenceLogProb: tokenIds length (${tokenIds.length}) does not match logits seqLen (${seqLen})`
+    );
+  }
+  assertSpanWithinBounds(span, seqLen);
+
+  const { data } = logits;
+  let total = 0;
+
+  for (let j = span.start; j < span.end; j++) {
+    const targetId = Number(tokenIds[j]);
+    if (!Number.isInteger(targetId) || targetId < 0 || targetId >= vocabSize) {
+      throw new Error(
+        `sumSequenceLogProb: token id ${targetId} at position ${j} out of vocab range [0, ${vocabSize})`
+      );
+    }
+
+    const pos = j - 1;
+    const rowStart = pos * vocabSize;
+    const { logSumExp } = logSumExpRow(data, rowStart, vocabSize, `position ${pos}`);
+    const targetLogit = data[rowStart + targetId] ?? Number.NaN;
+    total += targetLogit - logSumExp;
+  }
+
+  return total;
+}