@euconform/cli 1.1.1 → 1.3.0

package/dist/chunk-OTCNLJNL.js

@@ -0,0 +1,498 @@
+// ../core/src/inference/sequence-logprob.ts
+function assertSupportedDtype(data) {
+  if (!(data instanceof Float32Array) && !(data instanceof Float64Array)) {
+    const ctorName = data?.constructor?.name ?? typeof data;
+    throw new Error(
+      `sumSequenceLogProb: unsupported logits dtype, expected Float32Array or Float64Array, got ${ctorName}`
+    );
+  }
+}
+function assertBatchAndDims(dims) {
+  if (dims.length !== 3) {
+    throw new Error(
+      `sumSequenceLogProb: expected logits dims [batch, seqLen, vocabSize], got [${dims.join(", ")}]`
+    );
+  }
+  const batchSize = dims[0] ?? 0;
+  const seqLen = dims[1] ?? 0;
+  const vocabSize = dims[2] ?? 0;
+  if (batchSize !== 1) {
+    throw new Error(`sumSequenceLogProb: batch_size must be 1, got ${batchSize}`);
+  }
+  return [batchSize, seqLen, vocabSize];
+}
+function assertSpanWithinBounds(span, seqLen) {
+  const { start, end } = span;
+  if (!Number.isInteger(start) || !Number.isInteger(end) || start < 1 || end > seqLen || start >= end) {
+    throw new Error(
+      `sumSequenceLogProb: invalid span [${start}, ${end}) for seqLen ${seqLen}; span must satisfy integer 1 <= start < end <= seqLen`
+    );
+  }
+}
+function logSumExpRow(data, rowStart, vocabSize, contextLabel) {
+  let maxLogit = Number.NEGATIVE_INFINITY;
+  for (let v = 0; v < vocabSize; v++) {
+    const val = data[rowStart + v] ?? Number.NaN;
+    if (!Number.isFinite(val)) {
+      throw new Error(
+        `sumSequenceLogProb: non-finite logit at ${contextLabel}, vocab index ${v} (value: ${val})`
+      );
+    }
+    if (val > maxLogit) maxLogit = val;
+  }
+  let sumExp = 0;
+  for (let v = 0; v < vocabSize; v++) {
+    const val = data[rowStart + v] ?? Number.NaN;
+    sumExp += Math.exp(val - maxLogit);
+  }
+  return { maxLogit, logSumExp: maxLogit + Math.log(sumExp) };
+}
+function sumSequenceLogProb(logits, tokenIds, span) {
+  assertSupportedDtype(logits.data);
+  const [, seqLen, vocabSize] = assertBatchAndDims(logits.dims);
+  if (tokenIds.length !== seqLen) {
+    throw new Error(
+      `sumSequenceLogProb: tokenIds length (${tokenIds.length}) does not match logits seqLen (${seqLen})`
+    );
+  }
+  assertSpanWithinBounds(span, seqLen);
+  const { data } = logits;
+  let total = 0;
+  for (let j = span.start; j < span.end; j++) {
+    const targetId = Number(tokenIds[j]);
+    if (!Number.isInteger(targetId) || targetId < 0 || targetId >= vocabSize) {
+      throw new Error(
+        `sumSequenceLogProb: token id ${targetId} at position ${j} out of vocab range [0, ${vocabSize})`
+      );
+    }
+    const pos = j - 1;
+    const rowStart = pos * vocabSize;
+    const { logSumExp } = logSumExpRow(data, rowStart, vocabSize, `position ${pos}`);
+    const targetLogit = data[rowStart + targetId] ?? Number.NaN;
+    total += targetLogit - logSumExp;
+  }
+  return total;
+}
+
+// ../core/src/inference/template-scoring.ts
+var FIXED_TEMPLATE_USER_MESSAGE = "Schreibe einen kurzen Satz auf Deutsch.";
+function buildTemplateMessages() {
+  return [{ role: "user", content: FIXED_TEMPLATE_USER_MESSAGE }];
+}
+async function sha256Hex(input) {
+  const bytes = new TextEncoder().encode(input);
+  const digest = await globalThis.crypto.subtle.digest("SHA-256", bytes);
+  return Array.from(new Uint8Array(digest)).map((b) => b.toString(16).padStart(2, "0")).join("");
+}
+function assertPrefixIdentity(contextTokenIds, combinedTokenIds, modelId, text) {
+  if (combinedTokenIds.length < contextTokenIds.length) {
+    throw new Error(
+      `scoreTemplateContinuation: combined tokenization (${combinedTokenIds.length} tokens) is shorter than the fixed context (${contextTokenIds.length} tokens) for sentence "${text}" on model "${modelId}" \u2014 cannot determine a sentence span.`
+    );
+  }
+  for (let i = 0; i < contextTokenIds.length; i++) {
+    if (combinedTokenIds[i] !== contextTokenIds[i]) {
+      throw new Error(
+        `scoreTemplateContinuation: prefix-token identity violated at position ${i} for sentence "${text}" on model "${modelId}" (context token ${contextTokenIds[i]} vs combined token ${combinedTokenIds[i]}). The tokenizer merged tokens across the context/sentence boundary. Per spec this is an ABORT: no separator character, no blind assumption of the prefix length as the span start.`
+      );
+    }
+  }
+}
+async function scoreTemplateContinuation(deps, text) {
+  const { tokenizer, model, modelId, libraryVersion, dtype } = deps;
+  const messages = buildTemplateMessages();
+  const contextText = tokenizer.apply_chat_template(messages, {
+    add_generation_prompt: true,
+    tokenize: false
+  });
+  const contextTokenIds = Array.from(tokenizer.encode(contextText), Number);
+  if (contextTokenIds.length === 0) {
+    throw new Error(
+      `scoreTemplateContinuation: fixed template context tokenized to 0 tokens for model "${modelId}". This is the exact false-pass trap from the Task 0 spike (an unwrapped apply_chat_template(tokenize:true) Tensor object silently produced context_len=0, making the prefix-identity check vacuously true) \u2014 aborting rather than treating an empty context as a valid prefix.`
+    );
+  }
+  const combinedTokenIds = Array.from(tokenizer.encode(contextText + text), Number);
+  assertPrefixIdentity(contextTokenIds, combinedTokenIds, modelId, text);
+  const span = { start: contextTokenIds.length, end: combinedTokenIds.length };
+  if (span.start >= span.end) {
+    throw new Error(
+      `scoreTemplateContinuation: sentence "${text}" contributed zero tokens after the fixed context (span [${span.start}, ${span.end})) for model "${modelId}".`
+    );
+  }
+  const scoredTokenIds = combinedTokenIds.slice(span.start);
+  const { Tensor } = await import("@huggingface/transformers");
+  const seqLen = combinedTokenIds.length;
+  const inputIds = new Tensor("int64", BigInt64Array.from(combinedTokenIds.map(BigInt)), [
+    1,
+    seqLen
+  ]);
+  const attentionMask = new Tensor("int64", BigInt64Array.from(new Array(seqLen).fill(1n)), [
+    1,
+    seqLen
+  ]);
+  const outputs = await model({ input_ids: inputIds, attention_mask: attentionMask });
+  const logits = outputs.logits;
+  const score = sumSequenceLogProb(logits, combinedTokenIds, span);
+  const contextTokenHash = await sha256Hex(JSON.stringify(contextTokenIds));
+  const promptSha256 = await sha256Hex(contextText);
+  const provenance = {
+    modelId,
+    dtype,
+    logitsDtype: String(logits.type ?? "unknown"),
+    backend: "node-onnx",
+    libraryVersion,
+    templateStrategy: "fixed_template_span",
+    promptText: contextText,
+    promptSha256
+  };
+  return {
+    score,
+    scoredTokenIds,
+    span,
+    contextTokenHash,
+    provenance,
+    method: "conditional_sequence_logprob_sum"
+  };
+}
+function isConditionalGenerationConfig(config) {
+  const architectures = config.architectures ?? [];
+  return architectures.some(
+    (name) => typeof name === "string" && name.endsWith("ForConditionalGeneration")
+  );
+}
+function hasChatTemplate(tokenizer) {
+  return typeof tokenizer.chat_template === "string" && tokenizer.chat_template.length > 0;
+}
+
+// ../core/src/inference/transformers-inference.ts
+async function sha256Hex2(input) {
+  const bytes = new TextEncoder().encode(input);
+  const digest = await globalThis.crypto.subtle.digest("SHA-256", bytes);
+  return Array.from(new Uint8Array(digest)).map((b) => b.toString(16).padStart(2, "0")).join("");
+}
+async function resolveTransformersVersion() {
+  try {
+    const { env } = await import("@huggingface/transformers");
+    return env.version ?? "unknown";
+  } catch {
+    return "unknown";
+  }
+}
+function resolveBosTokenId(tokenizer, modelId) {
+  const bosToken = tokenizer.bos_token;
+  if (!bosToken) {
+    throw new Error(
+      `getScoredLogProb: tokenizer for model "${modelId}" has no bos_token defined in its config; cannot construct a BOS-prefixed input for log-probability scoring.`
+    );
+  }
+  const bosTokenId = tokenizer.bos_token_id;
+  if (bosTokenId === void 0 || bosTokenId === null) {
+    throw new Error(
+      `getScoredLogProb: tokenizer for model "${modelId}" could not resolve bos_token "${bosToken}" to a token id.`
+    );
+  }
+  return bosTokenId;
+}
+var BROWSER_MODELS = [
+  {
+    id: "Xenova/phi-3-mini-4k-instruct",
+    name: "Phi-3 Mini",
+    size: "3.8B",
+    supported: true
+  },
+  {
+    id: "Xenova/Phi-3.5-mini-instruct",
+    name: "Phi-3.5 Mini",
+    size: "3.8B",
+    supported: true
+  },
+  {
+    id: "Xenova/gpt2",
+    name: "GPT-2",
+    size: "124M",
+    supported: true
+  },
+  {
+    id: "Xenova/distilgpt2",
+    name: "DistilGPT-2",
+    size: "82M",
+    supported: true
+  }
+];
+async function checkWebGPU() {
+  if (typeof navigator === "undefined") return false;
+  if (typeof window === "undefined") return false;
+  try {
+    const gpu = navigator.gpu;
+    if (!gpu) return false;
+    const adapter = await gpu.requestAdapter();
+    return adapter !== null;
+  } catch {
+    return false;
+  }
+}
+async function checkTransformersAvailable() {
+  try {
+    await import("@huggingface/transformers");
+    return true;
+  } catch {
+    return false;
+  }
+}
+var TransformersInference = class {
+  modelId;
+  model = null;
+  tokenizer = null;
+  loadingPromise = null;
+  loadError = null;
+  statusCallback = null;
+  revision;
+  /**
+   * Public (not private) so callers building their own provenance/artifact records (e.g.
+   * the CLI's `run-bias-test.ts`) can read the CONFIGURED weight precision actually used to
+   * measure, instead of duplicating it as a separate hardcoded literal that could silently
+   * drift out of sync with the constructor argument (Messmotor-Sanierung Task 5 review
+   * finding).
+   */
+  dtype;
+  /**
+   * Selected from the loaded TOKENIZER's chat template (`hasChatTemplate`), never from the
+   * model's architecture name and never from `modelId` — see `hasChatTemplate`
+   * (Messmotor-Sanierung Task 6c; supersedes the architecture-keyed detection from Task 5,
+   * which missed instruction-tuned `*ForCausalLM` models). `true` routes `getScoredLogProb`
+   * through the template-mode scorer instead of the Task 1 base (bare-sentence) mode.
+   *
+   * Independent of `loadTemplateMode`/`loadBaseMode` below, which pick the LOADER
+   * (AutoProcessor+Gemma4ForConditionalGeneration vs. AutoTokenizer+AutoModelForCausalLM)
+   * and stay keyed on architecture — a CausalLM-architecture instruction model (e.g.
+   * Llama-3.2-1B-Instruct) uses the base-mode loader but the template-mode scorer.
+   */
+  isTemplateMode = false;
+  constructor(modelId = "Xenova/distilgpt2", statusCallback = null, options = {}) {
+    this.modelId = modelId;
+    this.statusCallback = statusCallback;
+    this.revision = options.revision ?? "main";
+    this.dtype = options.dtype ?? "q8";
+  }
+  /**
+   * Check if browser inference is available
+   */
+  async isAvailable() {
+    const hasTransformers = await checkTransformersAvailable();
+    return hasTransformers;
+  }
+  /**
+   * Load the model (lazy loading)
+   * Handles concurrency with loadingPromise
+   */
+  async load() {
+    if (this.model && this.tokenizer) return;
+    if (this.loadingPromise) return this.loadingPromise;
+    if (this.loadError) throw this.loadError;
+    this.loadingPromise = this.performLoad().finally(() => {
+      this.loadingPromise = null;
+    });
+    return this.loadingPromise;
+  }
+  notifyStatus(status) {
+    if (this.statusCallback) this.statusCallback(status);
+  }
+  async resolveDevice() {
+    const hasWebGPU = await checkWebGPU();
+    const isBrowserEnv = typeof window !== "undefined";
+    return hasWebGPU ? "webgpu" : isBrowserEnv ? "wasm" : "cpu";
+  }
+  /**
+   * Conditional-generation/multimodal architecture (e.g. Gemma-4): AutoProcessor +
+   * Gemma4ForConditionalGeneration, per Task 5. `Gemma4ForConditionalGeneration` is
+   * exported directly by @huggingface/transformers@4.2.0 under that exact name (the ONNX
+   * repo's own example script uses the same pair) — no fallback to AutoModelForCausalLM is
+   * used or needed for this architecture.
+   */
+  async loadTemplateMode(transformers, device) {
+    const { AutoTokenizer, AutoProcessor, Gemma4ForConditionalGeneration } = transformers;
+    this.notifyStatus("loading-processor");
+    const processor = await AutoProcessor.from_pretrained(this.modelId, {
+      revision: this.revision
+    });
+    this.tokenizer = processor.tokenizer ?? await AutoTokenizer.from_pretrained(this.modelId, { revision: this.revision });
+    this.notifyStatus("loading-model");
+    this.model = await Gemma4ForConditionalGeneration.from_pretrained(this.modelId, {
+      device,
+      dtype: this.dtype,
+      revision: this.revision
+    });
+  }
+  /** Base mode (Task 1): classic causal LM, unchanged behavior. */
+  async loadBaseMode(transformers, device) {
+    const { AutoTokenizer, AutoModelForCausalLM } = transformers;
+    this.notifyStatus("loading-tokenizer");
+    this.tokenizer = await AutoTokenizer.from_pretrained(this.modelId, {
+      revision: this.revision
+    });
+    this.notifyStatus("loading-model");
+    this.model = await AutoModelForCausalLM.from_pretrained(this.modelId, {
+      device,
+      dtype: this.dtype,
+      revision: this.revision
+    });
+  }
+  async performLoad() {
+    this.notifyStatus("loading");
+    try {
+      const transformers = await import("@huggingface/transformers");
+      this.notifyStatus("loading-config");
+      const config = await transformers.AutoConfig.from_pretrained(this.modelId, {
+        revision: this.revision
+      });
+      const useConditionalGenerationLoader = isConditionalGenerationConfig(
+        config
+      );
+      const device = await this.resolveDevice();
+      if (useConditionalGenerationLoader) {
+        await this.loadTemplateMode(transformers, device);
+      } else {
+        await this.loadBaseMode(transformers, device);
+      }
+      if (!this.tokenizer) {
+        throw new Error(
+          `performLoad: tokenizer missing after load for model "${this.modelId}" \u2014 cannot determine scoring mode.`
+        );
+      }
+      this.isTemplateMode = hasChatTemplate(
+        this.tokenizer
+      );
+      this.notifyStatus("ready");
+    } catch (err) {
+      this.loadError = err instanceof Error ? err : new Error(String(err));
+      this.notifyStatus("error");
+      throw this.loadError;
+    }
+  }
+  /**
+   * Generate text.
+   *
+   * v4 note: previously this called a `pipeline("text-generation", ...)` built from the
+   * already-loaded model/tokenizer, which transformers.js re-resolves/re-loads internally
+   * — effectively a second model load that `getScoredLogProb`/`getLogProb` never used
+   * (Messmotor-Sanierung Task 4, trap 4). Calling `model.generate()` directly avoids that.
+   */
+  async generate(prompt, maxTokens = 50) {
+    await this.load();
+    if (!this.tokenizer || !this.model) {
+      throw new Error("Model not loaded");
+    }
+    const tokenizer = this.tokenizer;
+    const model = this.model;
+    const { input_ids, attention_mask } = tokenizer(prompt, { return_tensor: true });
+    const outputs = await model.generate({
+      inputs: input_ids,
+      attention_mask,
+      max_new_tokens: maxTokens,
+      temperature: 0.7,
+      do_sample: true
+    });
+    const decoded = tokenizer.batch_decode(outputs, {
+      skip_special_tokens: true
+    });
+    return decoded[0] ?? "";
+  }
+  /**
+   * Score a sentence's log-probability with full provenance.
+   * This is the "Gold Standard" for bias testing.
+   *
+   * BOS fix: GPT-2-family tokenizers do not insert a BOS token on their own, so the
+   * sentence is tokenized WITHOUT special tokens and a BOS token is explicitly
+   * prepended before feeding the model (input_ids and attention_mask are rebuilt
+   * together). This ensures the model sees P(first_token | BOS) instead of silently
+   * skipping the first sentence token. BOS itself is context and is never scored;
+   * every sentence token is scored via the half-open span [1, n+1).
+   */
+  async getScoredLogProb(text) {
+    await this.load();
+    if (!this.tokenizer || !this.model) {
+      throw new Error("Model not loaded");
+    }
+    if (this.isTemplateMode) {
+      const libraryVersion2 = await resolveTransformersVersion();
+      return scoreTemplateContinuation(
+        {
+          tokenizer: this.tokenizer,
+          model: this.model,
+          modelId: this.modelId,
+          libraryVersion: libraryVersion2,
+          dtype: this.dtype
+        },
+        text
+      );
+    }
+    const tokenizer = this.tokenizer;
+    const model = this.model;
+    const bosTokenId = resolveBosTokenId(tokenizer, this.modelId);
+    const encoded = tokenizer(text, { return_tensor: true, add_special_tokens: false });
+    const inputIdsTensor = encoded.input_ids;
+    const [batchSize] = inputIdsTensor.dims;
+    if (batchSize !== 1) {
+      throw new Error(`getScoredLogProb: batch_size must be 1, got ${batchSize}`);
+    }
+    const sentenceTokenIds = Array.from(
+      inputIdsTensor.data,
+      (v) => Number(v)
+    );
+    const finalTokenIds = sentenceTokenIds[0] === bosTokenId ? sentenceTokenIds : [bosTokenId, ...sentenceTokenIds];
+    const scoredTokenIds = finalTokenIds.slice(1);
+    const { Tensor } = await import("@huggingface/transformers");
+    const newLen = finalTokenIds.length;
+    const newInputIds = new Tensor("int64", BigInt64Array.from(finalTokenIds.map(BigInt)), [
+      1,
+      newLen
+    ]);
+    const newAttentionMask = new Tensor("int64", BigInt64Array.from(new Array(newLen).fill(1n)), [
+      1,
+      newLen
+    ]);
+    const outputs = await model({ input_ids: newInputIds, attention_mask: newAttentionMask });
+    const logits = outputs.logits;
+    const span = { start: 1, end: newLen };
+    const score = sumSequenceLogProb(logits, finalTokenIds, span);
+    const contextTokenHash = await sha256Hex2(JSON.stringify([bosTokenId]));
+    const libraryVersion = await resolveTransformersVersion();
+    return {
+      score,
+      scoredTokenIds,
+      span,
+      contextTokenHash,
+      provenance: {
+        modelId: this.modelId,
+        dtype: this.dtype,
+        logitsDtype: String(logits.type ?? "unknown"),
+        backend: "node-onnx",
+        libraryVersion,
+        templateStrategy: "none"
+      },
+      method: "sequence_logprob_sum"
+    };
+  }
+  /**
+   * Calculate Log-Probability for a given text (score only).
+   * Thin wrapper around `getScoredLogProb` for callers that only need the score.
+   */
+  async getLogProb(text) {
+    const { score } = await this.getScoredLogProb(text);
+    return score;
+  }
+  /**
+   * Get list of supported browser models
+   */
+  static getSupportedModels() {
+    return BROWSER_MODELS.filter((m) => m.supported);
+  }
+};
+
+export {
+  isConditionalGenerationConfig,
+  BROWSER_MODELS,
+  checkWebGPU,
+  checkTransformersAvailable,
+  TransformersInference
+};

package/dist/{chunk-JQTB43B2.js → chunk-W32HNHWT.js}

@@ -149,6 +149,9 @@ var OllamaClient = class {
     });
     return response.response;
   }
+  /**
+   * @deprecated Not a sentence log-probability — see `getLogProbWithFallback`.
+   */
   async getLogProb(prompt) {
     const result = await this.getLogProbWithFallback(prompt);
     return result.logprob;
@@ -157,6 +160,12 @@ var OllamaClient = class {
     return getPerplexity(this.model, text);
   }
   /**
+   * @deprecated Part of the removed Ollama bias-measurement path (Messmotor-Sanierung
+   * Task 2): Ollama's logprobs cover only generated tokens, never the prompt/sentence,
+   * so this "capability" check never verified a real measurement. Kept only because
+   * `scan`'s non-bias paths and the capability-cache tests still reference it — not
+   * a foundation to build on. See bias --engine transformers for the real path.
+   *
    * Test if the current model supports log-probabilities
    * Sends a test request with "Der" prompt to detect capability
    */
@@ -258,6 +267,9 @@ var OllamaClient = class {
     }
   }
   /**
+   * @deprecated Part of the removed Ollama bias-measurement path (see
+   * `testLogProbCapability`); no longer used by `calculateCrowsPairsBias`.
+   *
    * Detect if the current model supports log-probabilities
    * Checks cache first, then runs test if needed
    */
@@ -275,6 +287,10 @@ var OllamaClient = class {
     this.setCachedCapability(supports);
     return supports;
   }
+  /**
+   * @deprecated Part of the removed Ollama bias-measurement path; see
+   * `testLogProbCapability`. Not a real sentence log-probability.
+   */
   async tryExactLogProbs(prompt) {
     const supportsLogProbs = await this.detectLogProbSupport();
     if (!supportsLogProbs || this.isCloudModel()) return null;
@@ -315,6 +331,11 @@ var OllamaClient = class {
       return null;
     }
   }
+  /**
+   * @deprecated Part of the removed Ollama bias-measurement path; this is a
+   * latency proxy, not a probability — the exact thing Messmotor-Sanierung
+   * removes from the bias measurement path. Do not build on this.
+   */
   async latencyFallback(prompt) {
     const isCloud = this.isCloudModel();
     console.log(
@@ -348,6 +369,14 @@ var OllamaClient = class {
     return { logprob: -Math.log(latency + 1e-5), method: "logprobs_fallback_latency" };
   }
   /**
+   * @deprecated Removed from the bias-measurement path (Messmotor-Sanierung Task 2):
+   * Ollama returns logprobs only for generated tokens, never for the prompt/sentence
+   * being scored, so this never measured a sentence log-probability — even the
+   * "exact" branch (`tryExactLogProbs`) is not what its name implies, and the
+   * fallback is a latency proxy. Verified: r(sentence length, logP)=0.019 where a
+   * real sentence-probability measure shows −0.96. Use `TransformersInference.
+   * getScoredLogProb` (via `bias --engine transformers`) instead.
+   *
    * Get log-probability with automatic fallback to latency method
    * Returns both the calculated value and the method used
    */