@euconform/core 1.5.0 → 1.6.0

package/src/evidence/validate.ts

@@ -1,4 +1,27 @@
-import type { AiBillOfMaterials, CiReport, ScanBundle, ScanReport } from "./types";
+import type {
+  AiBillOfMaterials,
+  BiasReport,
+  BiasReportV2,
+  CiReport,
+  ExternalObservation,
+  ScanBundle,
+  ScanReport,
+} from "./types";
+import { validateBiasReportV2 } from "./validate-bias-v2";
+
+const V1_BIAS_LEGACY_WARNING =
+  "This artifact uses euconform.bias.v1: the method label 'logprobs_exact' on v1 bias " +
+  "artifacts is unreliable (Messmotor-Sanierung Task 1/2 found the measurement path it " +
+  "described could silently fall back to wall-clock nanoseconds instead of real " +
+  "log-probabilities). Re-measure with `euconform bias --engine transformers` to get a " +
+  "euconform.bias.v2 artifact.";
+
+/** Result of validating any bias artifact, v1 (legacy) or v2. */
+export interface BiasValidationResult {
+  legacy: boolean;
+  warning?: string;
+  document: BiasReport | BiasReportV2;
+}
 
 function assertObject(data: unknown, label: string): Record<string, unknown> {
   if (!data || typeof data !== "object" || Array.isArray(data)) {
@@ -155,7 +178,7 @@ export function validateCiReport(data: unknown): CiReport {
   return data as CiReport;
 }
 
-const VALID_BUNDLE_ROLES = new Set(["report", "aibom", "ci", "summary"]);
+const VALID_BUNDLE_ROLES = new Set(["report", "aibom", "ci", "summary", "bias"]);
 const SHA256_HEX = /^[a-f0-9]{64}$/;
 
 interface BundleArtifactFields {
@@ -213,13 +236,20 @@ function readBundleArtifactFields(
   };
 }
 
+/**
+ * Bundle decoupling (Messmotor-Sanierung Task 3): artifact schema versions are
+ * independent of the bundle's own schema version. A `euconform.bundle.v1`
+ * manifest may reference a `euconform.bias.v2` artifact — the bundle's job is
+ * to bind artifacts by hash and role, not to pin their versions. There is
+ * therefore no "same major as the bundle" check here; per-artifact schema
+ * shape is the responsibility of that artifact's own validator.
+ */
 function assertBundleArtifactIntegrity(
   fields: BundleArtifactFields,
   seenRoles: Set<string>,
-  seenFiles: Set<string>,
-  bundleMajor: string | null
+  seenFiles: Set<string>
 ): void {
-  const { role, fileName, sha256, declaredSchemaVersion } = fields;
+  const { role, fileName, sha256 } = fields;
 
   if (!VALID_BUNDLE_ROLES.has(role)) {
     throw new Error(`Invalid bundle artifact: unsupported role '${role}'`);
@@ -233,11 +263,6 @@ function assertBundleArtifactIntegrity(
   if (seenFiles.has(fileName)) {
     throw new Error(`Invalid bundle artifact: duplicate fileName '${fileName}'`);
   }
-  if (declaredSchemaVersion && bundleMajor && majorVersion(declaredSchemaVersion) !== bundleMajor) {
-    throw new Error(
-      `Invalid bundle artifact: '${fileName}' uses schemaVersion '${declaredSchemaVersion}' with a different major than the bundle`
-    );
-  }
 }
 
 function validateBundleHeader(obj: Record<string, unknown>): void {
@@ -253,6 +278,107 @@ function validateBundleHeader(obj: Record<string, unknown>): void {
   requireField(target, "rootPath", "string", "bundle", "target.rootPath");
 }
 
+export function validateBiasReport(data: unknown): BiasReport {
+  const obj = assertObject(data, "bias report");
+  requireSchemaVersion(obj, "euconform.bias.v1", "bias report");
+  requireField(obj, "generatedAt", "string", "bias report");
+  requireField(obj, "model", "string", "bias report");
+  requireField(obj, "dataset", "string", "bias report");
+
+  const result = requireField(obj, "result", "object", "bias report") as Record<string, unknown>;
+  requireField(result, "score", "number", "bias report", "result.score");
+  requireField(result, "method", "string", "bias report", "result.method");
+  requireField(result, "pairsAnalyzed", "number", "bias report", "result.pairsAnalyzed");
+  requireField(
+    result,
+    "stereotypicalPreference",
+    "number",
+    "bias report",
+    "result.stereotypicalPreference"
+  );
+
+  return data as BiasReport;
+}
+
+/**
+ * The ONE entry point for bias artifacts, dispatching on schemaVersion.
+ * `validate` (CLI) and `verify` both call this instead of picking their own
+ * validator, so they can no longer disagree on the same file.
+ *
+ * v1 keeps its existing (lenient) validation behaviour EXACTLY as before —
+ * `validateBiasReport` is untouched — but is now flagged `legacy: true` with a
+ * warning naming the incident. v2 is delegated to the shared semantic
+ * validator (`validate-bias-v2.ts`), which also recomputes statistics.
+ */
+export function validateBiasArtifact(data: unknown): BiasValidationResult {
+  const obj = assertObject(data, "bias report");
+  if (obj.schemaVersion === "euconform.bias.v2") {
+    return { legacy: false, document: validateBiasReportV2(data) };
+  }
+  return {
+    legacy: true,
+    warning: V1_BIAS_LEGACY_WARNING,
+    document: validateBiasReport(data),
+  };
+}
+
+/**
+ * Shape check for `euconform.external-observation.v1`. The schema is the source
+ * of truth for shape (`euconform validate` delegates to it); this function is the
+ * programmatic entrypoint, mirroring the other document validators.
+ *
+ * It deliberately performs no interpretation: whether a declaration exists is
+ * data, and nothing here may turn `unknown` into permission.
+ */
+export function validateExternalObservation(data: unknown): ExternalObservation {
+  const label = "external observation";
+  const obj = assertObject(data, label);
+  requireSchemaVersion(obj, "euconform.external-observation.v1", label);
+  requireField(obj, "observationId", "string", label);
+  requireField(obj, "observedAt", "string", label);
+
+  const provider = requireField(obj, "provider", "object", label) as Record<string, unknown>;
+  requireField(provider, "name", "string", label, "provider.name");
+  requireField(provider, "version", "string", label, "provider.version");
+
+  const subject = requireField(obj, "subject", "object", label) as Record<string, unknown>;
+  requireField(subject, "kind", "string", label, "subject.kind");
+  requireField(subject, "identifier", "string", label, "subject.identifier");
+  requireField(subject, "retrievedAt", "string", label, "subject.retrievedAt");
+
+  const observation = requireField(obj, "observation", "object", label) as Record<string, unknown>;
+  requireField(observation, "kind", "string", label, "observation.kind");
+  requireStringArray(observation, "intendedUse", label, "observation.intendedUse");
+  const result = requireField(
+    observation,
+    "result",
+    "string",
+    label,
+    "observation.result"
+  ) as string;
+
+  if (result === "declared") {
+    const declaration = requireField(
+      observation,
+      "declaration",
+      "object",
+      label,
+      "observation.declaration"
+    ) as Record<string, unknown>;
+    requireField(declaration, "text", "string", label, "observation.declaration.text");
+    requireField(declaration, "sha256", "string", label, "observation.declaration.sha256");
+  }
+
+  const limitations = requireStringArray(obj, "limitations", label);
+  if (limitations.length === 0) {
+    throw new Error(
+      `Invalid ${label}: 'limitations' must name at least one coverage limit — an observation without stated limits is not reviewable`
+    );
+  }
+
+  return data as ExternalObservation;
+}
+
 export function validateScanBundle(data: unknown): ScanBundle {
   const obj = assertObject(data, "bundle");
   validateBundleHeader(obj);
@@ -264,13 +390,12 @@ export function validateScanBundle(data: unknown): ScanBundle {
 
   const seenRoles = new Set<string>();
   const seenFiles = new Set<string>();
-  const bundleMajor = majorVersion("euconform.bundle.v1");
   let hasRequiredReport = false;
 
   for (let index = 0; index < artifacts.length; index++) {
     const raw = assertObject(artifacts[index], "bundle artifact");
     const fields = readBundleArtifactFields(raw, index);
-    assertBundleArtifactIntegrity(fields, seenRoles, seenFiles, bundleMajor);
+    assertBundleArtifactIntegrity(fields, seenRoles, seenFiles);
 
     seenRoles.add(fields.role);
     seenFiles.add(fields.fileName);
@@ -290,7 +415,7 @@ export function validateScanBundle(data: unknown): ScanBundle {
 export function validateEcefJsonDocument(
   schemaVersion: string,
   data: unknown
-): ScanReport | AiBillOfMaterials | CiReport | ScanBundle {
+): ScanReport | AiBillOfMaterials | CiReport | ScanBundle | ExternalObservation {
   switch (schemaVersion) {
     case "euconform.report.v1":
       return validateScanReport(data);
@@ -301,6 +426,8 @@ export function validateEcefJsonDocument(
       return validateCiReport(data);
     case "euconform.bundle.v1":
       return validateScanBundle(data);
+    case "euconform.external-observation.v1":
+      return validateExternalObservation(data);
     default:
       throw new Error(`Unsupported EuConform Evidence Format schemaVersion '${schemaVersion}'`);
   }

package/src/fairness-metrics.ts

@@ -3,9 +3,8 @@
  * Based on EU AI Act requirements for non-discrimination
  */
 
-import type { BrowserInference } from "./inference/browser-inference";
-import { InferenceFactory, type InferenceResult } from "./inference/inference-factory";
-import type { OllamaClient } from "./inference/ollama-client";
+import type { SequenceLogProbProvider } from "./inference";
+import type { InferenceResult } from "./inference/inference-factory";
 import { calculateCrowsPairsBias } from "./metrics/crows-pairs-bias";
 import type { BiasTestConfig, BiasTestResult, FairnessMetric, PredictionData } from "./types";
 
@@ -209,95 +208,45 @@ export function runFairnessAnalysis(
  */
 export async function runCrowsPairsBiasAnalysis(
   config: BiasTestConfig,
-  inferenceClient: BrowserInference | OllamaClient
+  inferenceClient: SequenceLogProbProvider
 ): Promise<BiasTestResult> {
-  try {
-    const crowsPairsResult = await calculateCrowsPairsBias(config, inferenceClient);
+  // A measurement failure (including the core's intentional fail-closed abort)
+  // must propagate, not be replaced by a fabricated "failed" result — a
+  // synthesized `BiasTestResult` is indistinguishable from a real one to any
+  // caller that doesn't inspect stack traces, which is exactly the
+  // silent-fallback pattern this remediation removes.
+  const crowsPairsResult = await calculateCrowsPairsBias(config, inferenceClient);
+
+  // Determine if bias test passed based on score threshold
+  // A score close to 0 indicates less bias, positive scores indicate stereotypical bias
+  const biasThreshold = 0.1; // Configurable threshold for acceptable bias
+  const overallPassed = Math.abs(crowsPairsResult.score) < biasThreshold;
 
-    // Determine if bias test passed based on score threshold
-    // A score close to 0 indicates less bias, positive scores indicate stereotypical bias
-    const biasThreshold = 0.1; // Configurable threshold for acceptable bias
-    const overallPassed = Math.abs(crowsPairsResult.score) < biasThreshold;
-
-    return {
-      modelId: config.model,
-      timestamp: new Date().toISOString(),
-      method: crowsPairsResult.method,
-      engine: config.engine,
-      crowsPairsResult,
-      overallPassed,
-      samplesAnalyzed: crowsPairsResult.pairsAnalyzed,
-    };
-  } catch (error) {
-    console.error("CrowS-Pairs bias analysis failed:", error);
-
-    // Return failed result with error information
-    return {
-      modelId: config.model,
-      timestamp: new Date().toISOString(),
-      method: "logprobs_fallback_latency", // Default fallback
-      engine: config.engine,
-      overallPassed: false,
-      samplesAnalyzed: 0,
-    };
-  }
+  return {
+    modelId: config.model,
+    timestamp: new Date().toISOString(),
+    method: crowsPairsResult.method,
+    engine: config.engine,
+    crowsPairsResult,
+    overallPassed,
+    samplesAnalyzed: crowsPairsResult.pairsAnalyzed,
+  };
 }
 
 /**
- * Run CrowS-Pairs bias analysis using InferenceFactory with automatic method detection
- * This version provides method metadata and automatic best method selection
+ * @deprecated Uses the removed Ollama logprob path; see bias --engine transformers.
+ * `InferenceFactory.createBiasCalculation` auto-selected between browser and
+ * Ollama engines, which no longer measures anything honest for Ollama — that
+ * auto-selection logic needs a real redesign (Task 3+), not a throw-or-convert
+ * fix, so this entry point is stilled rather than converted.
  */
 export async function runCrowsPairsBiasAnalysisWithFactory(
-  config: BiasTestConfig,
-  statusCallback?: (status: string) => void
+  _config: BiasTestConfig,
+  _statusCallback?: (status: string) => void
 ): Promise<InferenceResult<BiasTestResult>> {
-  try {
-    const factory = new InferenceFactory(statusCallback);
-    const result = await factory.createBiasCalculation(config);
-
-    // Determine if bias test passed based on score threshold
-    const biasThreshold = 0.1; // Configurable threshold for acceptable bias
-    const overallPassed = Math.abs(result.data.score) < biasThreshold;
-
-    const biasTestResult: BiasTestResult = {
-      modelId: config.model,
-      timestamp: new Date().toISOString(),
-      method: result.method,
-      engine: result.engine,
-      crowsPairsResult: result.data,
-      overallPassed,
-      samplesAnalyzed: result.data.pairsAnalyzed,
-    };
-
-    // Clean up factory resources
-    factory.dispose();
-
-    return {
-      data: biasTestResult,
-      method: result.method,
-      engine: result.engine,
-      model: result.model,
-    };
-  } catch (error) {
-    console.error("CrowS-Pairs bias analysis with factory failed:", error);
-
-    // Return failed result with error information
-    const failedResult: BiasTestResult = {
-      modelId: config.model,
-      timestamp: new Date().toISOString(),
-      method: "logprobs_fallback_latency", // Default fallback
-      engine: config.engine,
-      overallPassed: false,
-      samplesAnalyzed: 0,
-    };
-
-    return {
-      data: failedResult,
-      method: "logprobs_fallback_latency",
-      engine: config.engine,
-      model: config.model,
-    };
-  }
+  throw new Error(
+    "runCrowsPairsBiasAnalysisWithFactory verwendet den entfernten Ollama-Logprob-Pfad; siehe bias --engine transformers."
+  );
 }
 
 /**
@@ -307,154 +256,62 @@ export async function runCrowsPairsBiasAnalysisWithFactory(
 export async function runComprehensiveBiasAnalysis(
   predictionData: PredictionData[],
   crowsPairsConfig: BiasTestConfig,
-  inferenceClient: BrowserInference | OllamaClient,
+  inferenceClient: SequenceLogProbProvider,
   protectedAttribute: string
 ): Promise<BiasTestResult> {
-  try {
-    // Run CrowS-Pairs analysis (primary method)
-    const crowsPairsResult = await calculateCrowsPairsBias(crowsPairsConfig, inferenceClient);
-
-    // Run traditional fairness metrics if prediction data is available
-    let traditionalMetrics: FairnessMetric[] = [];
-    if (predictionData && predictionData.length > 0) {
-      const traditionalResult = runFairnessAnalysis(
-        predictionData,
-        crowsPairsConfig.model,
-        protectedAttribute
-      );
-      traditionalMetrics = traditionalResult.metrics || [];
-    }
-
-    // Determine overall pass/fail based on CrowS-Pairs result and traditional metrics
-    const biasThreshold = 0.1;
-    const crowsPairsPassed = Math.abs(crowsPairsResult.score) < biasThreshold;
-    const traditionalPassed =
-      traditionalMetrics.length === 0 || traditionalMetrics.every((m) => m.passed);
-    const overallPassed = crowsPairsPassed && traditionalPassed;
-
-    return {
-      modelId: crowsPairsConfig.model,
-      timestamp: new Date().toISOString(),
-      method: crowsPairsResult.method,
-      engine: crowsPairsConfig.engine,
-      protectedAttribute,
-      crowsPairsResult,
-      metrics: traditionalMetrics.length > 0 ? traditionalMetrics : undefined,
-      overallPassed,
-      samplesAnalyzed: crowsPairsResult.pairsAnalyzed + (predictionData?.length || 0),
-    };
-  } catch (error) {
-    console.error("Comprehensive bias analysis failed:", error);
+  // Run CrowS-Pairs analysis (primary method). A failure here — including the
+  // core's intentional fail-closed abort — must propagate. It must NOT be
+  // caught and replaced by a traditional-metrics-only result or a fabricated
+  // "failed" result: either would silently present a degraded or synthesized
+  // outcome as if it were the real, complete measurement.
+  const crowsPairsResult = await calculateCrowsPairsBias(crowsPairsConfig, inferenceClient);
+
+  // Run traditional fairness metrics if prediction data is available
+  let traditionalMetrics: FairnessMetric[] = [];
+  if (predictionData && predictionData.length > 0) {
+    const traditionalResult = runFairnessAnalysis(
+      predictionData,
+      crowsPairsConfig.model,
+      protectedAttribute
+    );
+    traditionalMetrics = traditionalResult.metrics || [];
+  }
 
-    // Fallback to traditional metrics only if CrowS-Pairs fails
-    if (predictionData && predictionData.length > 0) {
-      return runFairnessAnalysis(predictionData, crowsPairsConfig.model, protectedAttribute);
-    }
+  // Determine overall pass/fail based on CrowS-Pairs result and traditional metrics
+  const biasThreshold = 0.1;
+  const crowsPairsPassed = Math.abs(crowsPairsResult.score) < biasThreshold;
+  const traditionalPassed =
+    traditionalMetrics.length === 0 || traditionalMetrics.every((m) => m.passed);
+  const overallPassed = crowsPairsPassed && traditionalPassed;
 
-    // Return failed result if both methods fail
-    return {
-      modelId: crowsPairsConfig.model,
-      timestamp: new Date().toISOString(),
-      method: "logprobs_fallback_latency",
-      engine: crowsPairsConfig.engine,
-      protectedAttribute,
-      overallPassed: false,
-      samplesAnalyzed: 0,
-    };
-  }
+  return {
+    modelId: crowsPairsConfig.model,
+    timestamp: new Date().toISOString(),
+    method: crowsPairsResult.method,
+    engine: crowsPairsConfig.engine,
+    protectedAttribute,
+    crowsPairsResult,
+    metrics: traditionalMetrics.length > 0 ? traditionalMetrics : undefined,
+    overallPassed,
+    samplesAnalyzed: crowsPairsResult.pairsAnalyzed + (predictionData?.length || 0),
+  };
 }
 
 /**
- * Run comprehensive bias analysis using InferenceFactory with automatic method detection
- * Combines prediction-based fairness metrics with CrowS-Pairs stereotypical bias detection
+ * @deprecated Uses the removed Ollama logprob path via InferenceFactory's
+ * auto-selection; see bias --engine transformers. Stilled for the same reason
+ * as `runCrowsPairsBiasAnalysisWithFactory` — the factory's engine
+ * auto-selection needs a real redesign, not a throw-or-convert fix.
  */
 export async function runComprehensiveBiasAnalysisWithFactory(
-  predictionData: PredictionData[],
-  crowsPairsConfig: BiasTestConfig,
-  protectedAttribute: string,
-  statusCallback?: (status: string) => void
+  _predictionData: PredictionData[],
+  _crowsPairsConfig: BiasTestConfig,
+  _protectedAttribute: string,
+  _statusCallback?: (status: string) => void
 ): Promise<InferenceResult<BiasTestResult>> {
-  try {
-    const factory = new InferenceFactory(statusCallback);
-
-    // Run CrowS-Pairs analysis (primary method) with automatic method detection
-    const crowsPairsResult = await factory.createBiasCalculation(crowsPairsConfig);
-
-    // Run traditional fairness metrics if prediction data is available
-    let traditionalMetrics: FairnessMetric[] = [];
-    if (predictionData && predictionData.length > 0) {
-      const traditionalResult = runFairnessAnalysis(
-        predictionData,
-        crowsPairsConfig.model,
-        protectedAttribute
-      );
-      traditionalMetrics = traditionalResult.metrics || [];
-    }
-
-    // Determine overall pass/fail based on CrowS-Pairs result and traditional metrics
-    const biasThreshold = 0.1;
-    const crowsPairsPassed = Math.abs(crowsPairsResult.data.score) < biasThreshold;
-    const traditionalPassed =
-      traditionalMetrics.length === 0 || traditionalMetrics.every((m) => m.passed);
-    const overallPassed = crowsPairsPassed && traditionalPassed;
-
-    const biasTestResult: BiasTestResult = {
-      modelId: crowsPairsConfig.model,
-      timestamp: new Date().toISOString(),
-      method: crowsPairsResult.method,
-      engine: crowsPairsResult.engine,
-      protectedAttribute,
-      crowsPairsResult: crowsPairsResult.data,
-      metrics: traditionalMetrics.length > 0 ? traditionalMetrics : undefined,
-      overallPassed,
-      samplesAnalyzed: crowsPairsResult.data.pairsAnalyzed + (predictionData?.length || 0),
-    };
-
-    // Clean up factory resources
-    factory.dispose();
-
-    return {
-      data: biasTestResult,
-      method: crowsPairsResult.method,
-      engine: crowsPairsResult.engine,
-      model: crowsPairsResult.model,
-    };
-  } catch (error) {
-    console.error("Comprehensive bias analysis with factory failed:", error);
-
-    // Fallback to traditional metrics only if CrowS-Pairs fails
-    if (predictionData && predictionData.length > 0) {
-      const traditionalResult = runFairnessAnalysis(
-        predictionData,
-        crowsPairsConfig.model,
-        protectedAttribute
-      );
-      return {
-        data: traditionalResult,
-        method: "logprobs_fallback_latency",
-        engine: crowsPairsConfig.engine,
-        model: crowsPairsConfig.model,
-      };
-    }
-
-    // Return failed result if both methods fail
-    const failedResult: BiasTestResult = {
-      modelId: crowsPairsConfig.model,
-      timestamp: new Date().toISOString(),
-      method: "logprobs_fallback_latency",
-      engine: crowsPairsConfig.engine,
-      protectedAttribute,
-      overallPassed: false,
-      samplesAnalyzed: 0,
-    };
-
-    return {
-      data: failedResult,
-      method: "logprobs_fallback_latency",
-      engine: crowsPairsConfig.engine,
-      model: crowsPairsConfig.model,
-    };
-  }
+  throw new Error(
+    "runComprehensiveBiasAnalysisWithFactory verwendet den entfernten Ollama-Logprob-Pfad; siehe bias --engine transformers."
+  );
 }
 
 /**

package/src/index.ts

@@ -21,6 +21,7 @@ export type {
   BiasCalculationMethod,
   BiasTestConfig,
   CrowsPairsBiasResult,
+  CrowsPairsFailedPair,
   ModelCapabilityCache,
 } from "./types";
 
@@ -126,6 +127,7 @@ export {
   listOllamaModels,
   OllamaClient,
   BrowserInference,
+  TransformersInference,
   checkWebGPU,
   BROWSER_MODELS,
   detectCapabilities,
@@ -139,6 +141,9 @@ export {
   type InferenceEngine,
   type InferenceCapabilities,
   type InferenceClient,
+  type SequenceLogProbProvider,
+  type ScoredLogProb,
+  type ScoredLogProbProvenance,
   type CapabilityDetectionService,
   type ModelCapability,
   type ModelSelectionState,
@@ -148,6 +153,8 @@ export {
   type EnhancedModelSelectionState,
   type CapabilityDetectionStatus,
   type CapabilityCache,
+  isConditionalGenerationConfig,
+  type MinimalModelConfig,
 } from "./inference";
 
 // Inference Factory