@euconform/core 1.5.0 → 1.6.0

package/src/types.ts

@@ -65,7 +65,7 @@ export interface FairnessMetric {
 export interface BiasTestResult {
   modelId: string;
   timestamp: string;
-  method: BiasCalculationMethod;
+  method: BiasCalculationMethod | SequenceLogProbMethod;
   engine: "browser" | "ollama";
   protectedAttribute?: string;
   crowsPairsResult?: CrowsPairsBiasResult;
@@ -92,6 +92,17 @@ export interface CrowsPairsEntry {
 /** Bias calculation method types */
 export type BiasCalculationMethod = "logprobs_exact" | "logprobs_fallback_latency";
 
+/**
+ * How a sentence's log-probability was accumulated for the autoregressive
+ * CrowS-Pairs adaptation (Messmotor-Sanierung Task 3). "sequence_logprob_sum" is
+ * the base mode (sum of scored-token log-probs, BOS excluded). Public naming:
+ * "autoregressive CrowS-Pairs adaptation", never "original methodology" — the
+ * original CrowS-Pairs paper used masked-model pseudo-log-likelihood, which this
+ * is not. "conditional_sequence_logprob_sum" is reserved for a future template
+ * mode that conditions scoring on a fixed prompt/context span; not yet emitted.
+ */
+export type SequenceLogProbMethod = "sequence_logprob_sum" | "conditional_sequence_logprob_sum";
+
 /** Model capability cache for log-probability support */
 export interface ModelCapabilityCache {
   model: string;
@@ -100,17 +111,106 @@ export interface ModelCapabilityCache {
   ollamaVersion?: string;
 }
 
+/** Result for a single CrowS-Pairs sentence pair. */
+export interface CrowsPairsPairResult {
+  pairId: number | null;
+  stereotype: string;
+  antiStereotype: string;
+  biasType: string | null;
+  stereotypeLogProb: number;
+  antiStereotypeLogProb: number;
+  biasScore: number;
+  biasDirection: "stereotype" | "anti-stereotype" | "neutral";
+  method: SequenceLogProbMethod;
+  /** Number of scored tokens (BOS excluded) in the stereotype sentence. */
+  stereotypeTokenCount: number;
+  /** Number of scored tokens (BOS excluded) in the anti-stereotype sentence. */
+  antiStereotypeTokenCount: number;
+  /** stereotypeTokenCount - antiStereotypeTokenCount; 0 means the pair is part of the equal-token-length confounder-control subset. */
+  tokenCountDelta: number;
+  /** SHA-256 over the compact decimal-token-id JSON array of the stereotype sentence's scored tokens. */
+  stereotypeTokenHash: string;
+  /** SHA-256 over the compact decimal-token-id JSON array of the anti-stereotype sentence's scored tokens. */
+  antiStereotypeTokenHash: string;
+}
+
+/** Aggregate over all pairs sharing one bias type. */
+export interface CrowsPairsBiasTypeSummary {
+  biasType: string;
+  pairsCount: number;
+  stereotypePreferenceCount: number;
+  stereotypePreferencePercent: number;
+  averageBiasScore: number;
+}
+
+/** A pair that was selected for measurement but whose scoring failed (allowPartial runs only). */
+export interface CrowsPairsFailedPair {
+  pairId: number | null;
+  reason: string;
+}
+
+/**
+ * Snapshot of the measurement provenance the provider actually reported for a run,
+ * echoed by `calculateCrowsPairsBias` so CLI callers (see `run-bias-test.ts`) can build
+ * the artifact's top-level `provenance` object from what was really measured instead of
+ * a hardcoded literal (Messmotor-Sanierung Task 5 review finding: the CLI once wrote a
+ * hardcoded dtype; Task 6 found the same mistake for `templateStrategy`). Deliberately a
+ * standalone interface rather than importing `ScoredLogProbProvenance` from
+ * `./inference/scored-log-prob-types` — that module imports types FROM this file, and
+ * mirroring the handful of fields needed here avoids relying on TypeScript's type-only
+ * import erasure to keep the cycle inert.
+ */
+export interface CrowsPairsScoringProvenance {
+  dtype: string;
+  logitsDtype: string;
+  backend: string;
+  libraryVersion: string;
+  templateStrategy?: "none" | "fixed_template_span";
+  /** Present only when templateStrategy is "fixed_template_span". */
+  promptText?: string;
+  promptSha256?: string;
+  /** From the scored result's own `contextTokenHash` — constant across a run's pairs
+   * (same fixed context every call), so one representative sample is the whole run's value. */
+  contextTokenHash?: string;
+}
+
 /** CrowS-Pairs bias calculation result */
 export interface CrowsPairsBiasResult {
   score: number;
-  method: BiasCalculationMethod;
+  method: SequenceLogProbMethod;
   pairsAnalyzed: number;
   stereotypicalPreference: number; // percentage
+  /** One entry per analysed pair. Empty only if nothing could be analysed. */
+  pairResults: CrowsPairsPairResult[];
+  /** Per-category aggregates. The whole point: the overall score hides these. */
+  byBiasType: CrowsPairsBiasTypeSummary[];
+  /**
+   * "complete": every selected pair was analysed. "partial": at least one pair
+   * failed and `allowPartial` was set — this run is a diagnostic, not a full
+   * measurement. Fail-closed by default: without `allowPartial`, any failure
+   * aborts the whole run rather than producing a "partial" result.
+   */
+  runStatus: "complete" | "partial";
+  /** Pairs selected for measurement after dataset-extraction filtering. */
+  selectedCount: number;
+  /** Pairs successfully measured; the statistical denominator for score/preference. */
+  analyzedCount: number;
+  /** Pairs selected but that failed measurement (only possible with allowPartial). */
+  failedCount: number;
+  /** One entry per failed pair. Invariant: selectedCount === analyzedCount + failedCount. */
+  failedPairs: CrowsPairsFailedPair[];
   metadata: {
     engine: "browser" | "ollama";
     model: string;
     timestamp: string;
   };
+  /**
+   * The provider's actual scoring provenance (from the first successfully-measured
+   * pair — the fixed template context, if any, does not vary across pairs in one run).
+   * Undefined only if analyzedCount is 0, which `calculateCrowsPairsBias` already
+   * refuses to return (see "No valid bias scores calculated" throw).
+   */
+  scoringProvenance?: CrowsPairsScoringProvenance;
 }
 
 /** Configuration for bias test execution */
@@ -118,6 +218,12 @@ export interface BiasTestConfig {
   dataset: CrowsPairsEntry[];
   model: string;
   engine: "browser" | "ollama";
+  /**
+   * Diagnostic mode: a pair that fails measurement is recorded in `failedPairs`
+   * instead of aborting the whole run. NOT suitable as a complete measurement —
+   * see `runStatus: "partial"`. Default: false (fail-closed).
+   */
+  allowPartial?: boolean;
 }
 
 /** Annex IV Technical Documentation Report */

package/src/validation/schema-validator.ts

@@ -12,7 +12,15 @@ export interface ValidationError {
   received?: unknown;
 }
 
-export type SchemaType = "report.v1" | "aibom.v1" | "aibom.v1.1" | "ci.v1" | "bundle.v1";
+export type SchemaType =
+  | "report.v1"
+  | "aibom.v1"
+  | "aibom.v1.1"
+  | "ci.v1"
+  | "bundle.v1"
+  | "bias.v1"
+  | "bias.v2"
+  | "external-observation.v1";
 
 export interface ValidationResult {
   valid: boolean;
@@ -45,6 +53,12 @@ const SCHEMA_MAP: Record<string, { schemaFile: string; type: SchemaType }> = {
   "euconform.aibom.v1.1": { schemaFile: "aibom-v1.1.schema.json", type: "aibom.v1.1" },
   "euconform.ci.v1": { schemaFile: "ci-v1.schema.json", type: "ci.v1" },
   "euconform.bundle.v1": { schemaFile: "bundle-v1.schema.json", type: "bundle.v1" },
+  "euconform.bias.v1": { schemaFile: "bias-v1.schema.json", type: "bias.v1" },
+  "euconform.bias.v2": { schemaFile: "bias-v2.schema.json", type: "bias.v2" },
+  "euconform.external-observation.v1": {
+    schemaFile: "external-observation-v1.schema.json",
+    type: "external-observation.v1",
+  },
 };
 
 let ajvInstance: Ajv2020 | null = null;

package/tests/annex-iv-report.test.ts

@@ -78,7 +78,7 @@ describe("Annex IV Report Generation", () => {
           engine: "browser",
           crowsPairsResult: {
             score: 0.15,
-            method: "logprobs_exact",
+            method: "sequence_logprob_sum",
             pairsAnalyzed: 100,
             stereotypicalPreference: 65.5,
             metadata: {

package/tests/capability-detection.test.ts

@@ -0,0 +1,57 @@
+import { describe, expect, it } from "vitest";
+import {
+  DefaultCapabilityDetectionService,
+  EnhancedCapabilityDetectionService,
+} from "../src/inference/capability-detection";
+
+/**
+ * Browser/transformers.js capability entries are declared from a static model
+ * list — no model is actually loaded or tested to produce them. Reporting
+ * `status: "available"` with a fresh `lastTested` for that would claim a test
+ * ran when it didn't (see Messmotor-Sanierung Task 2). This pins the honest
+ * runtime values: `status: "unverified"` and no `lastTested`.
+ */
+describe("capability detection: browser/transformers entries are not overclaimed", () => {
+  it("DefaultCapabilityDetectionService.detectModelCapability reports 'unverified', not 'available', for browser", async () => {
+    const service = new DefaultCapabilityDetectionService();
+    const capability = await service.detectModelCapability("Xenova/distilgpt2", "browser");
+
+    expect(capability.status).toBe("unverified");
+    expect(capability.lastTested).toBeUndefined();
+    // The declared method may still be reported (renaming is Task 3's job) —
+    // it just must not be dressed up as measured.
+    expect(capability.method).toBe("logprobs_exact");
+  });
+
+  it("EnhancedCapabilityDetectionService.detectModelCapability reports 'unverified', not 'available', for browser", async () => {
+    const service = new EnhancedCapabilityDetectionService();
+    const capability = await service.detectModelCapability("Xenova/distilgpt2", "browser");
+
+    expect(capability.status).toBe("unverified");
+    expect(capability.lastTested).toBeUndefined();
+  });
+
+  it("'unverified' is not scored as 0 by the recommendation ranking (regression guard)", async () => {
+    // Exercise the same code path detectAllCapabilities uses, without touching
+    // the network (no live Ollama in CI): detect a single browser capability
+    // and confirm it sorts ahead of an explicitly zero-scored "error" entry.
+    const service = new DefaultCapabilityDetectionService();
+    const browserCapability = await service.detectModelCapability("Xenova/distilgpt2", "browser");
+    const errorCapability = {
+      modelId: "broken-model",
+      engine: "ollama" as const,
+      method: "logprobs_fallback_latency" as const,
+      status: "error" as const,
+      lastTested: new Date().toISOString(),
+    };
+
+    // sortByRecommendation is protected; reach it through the public
+    // detectAllCapabilities-equivalent ordering contract by calling the same
+    // scoring indirectly via Array#sort with the class's own comparator is not
+    // exposed, so assert the documented invariant directly instead: an
+    // "unverified" capability must score above a definitionally-zero one.
+    // biome-ignore lint/suspicious/noExplicitAny: reaching a protected method for a focused unit check
+    const score = (service as any).getRecommendationScore.bind(service);
+    expect(score(browserCapability)).toBeGreaterThan(score(errorCapability));
+  });
+});