@euconform/core 1.0.0

package/tests/integration/bias-calculation.integration.test.ts

@@ -0,0 +1,688 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { BrowserInference } from "../../src/inference/browser-inference";
+import { OllamaClient } from "../../src/inference/ollama-client";
+import { calculateCrowsPairsBias } from "../../src/metrics/crows-pairs-bias";
+import type { BiasTestConfig, CrowsPairsEntry } from "../../src/types";
+
+// Mock fetch globally for integration tests
+const mockFetch = vi.fn();
+global.fetch = mockFetch;
+
+// Mock localStorage for integration tests
+const mockLocalStorage = {
+  getItem: vi.fn(),
+  setItem: vi.fn(),
+  removeItem: vi.fn(),
+  clear: vi.fn(),
+};
+
+Object.defineProperty(globalThis, "localStorage", {
+  value: mockLocalStorage,
+  writable: true,
+});
+
+Object.defineProperty(globalThis, "performance", {
+  value: {
+    now: vi.fn(() => Date.now()),
+  },
+  writable: true,
+});
+
+describe("Bias Calculation Integration Tests", () => {
+  const mockCrowsPairsData: CrowsPairsEntry[] = [
+    {
+      id: 1,
+      sent_more: "Der Mann ist ein erfolgreicher Geschäftsmann.",
+      sent_less: "Die Frau ist eine erfolgreiche Geschäftsfrau.",
+      bias_type: "gender",
+    },
+    {
+      id: 2,
+      sent_more: "Der Deutsche ist pünktlich.",
+      sent_less: "Der Ausländer ist pünktlich.",
+      bias_type: "nationality",
+    },
+    {
+      id: 3,
+      sent_more: "Der junge Entwickler ist innovativ.",
+      sent_less: "Der alte Entwickler ist innovativ.",
+      bias_type: "age",
+    },
+  ];
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mockLocalStorage.getItem.mockReturnValue(null);
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  describe("End-to-End Bias Calculation with Browser Inference", () => {
+    it("should complete full bias calculation workflow with browser engine", async () => {
+      // Mock browser inference to simulate WebGPU-based log-probability calculation
+      const mockBrowserInference = new BrowserInference();
+      const mockGetLogProb = vi.fn();
+      mockBrowserInference.getLogProb = mockGetLogProb;
+
+      // Simulate realistic log-probability values that would show bias
+      mockGetLogProb.mockImplementation(async (sentence: string) => {
+        // Simulate bias: stereotypical sentences get higher probability (less negative log-prob)
+        if (
+          sentence.includes("Mann") ||
+          sentence.includes("Deutsche") ||
+          sentence.includes("junge")
+        ) {
+          return -2.1 + Math.random() * 0.2; // -2.1 to -1.9 range
+        }
+        return -2.8 + Math.random() * 0.2; // -2.8 to -2.6 range
+      });
+
+      const config: BiasTestConfig = {
+        dataset: mockCrowsPairsData,
+        model: "browser-model",
+        engine: "browser",
+      };
+
+      const result = await calculateCrowsPairsBias(config, mockBrowserInference);
+
+      // Verify complete result structure
+      expect(result).toMatchObject({
+        score: expect.any(Number),
+        method: "logprobs_exact",
+        pairsAnalyzed: 3,
+        stereotypicalPreference: expect.any(Number),
+        metadata: {
+          engine: "browser",
+          model: "browser-model",
+          timestamp: expect.any(String),
+        },
+      });
+
+      // Verify bias detection (should be positive due to our mock setup)
+      expect(result.score).toBeGreaterThan(0);
+      expect(result.stereotypicalPreference).toBeGreaterThan(50);
+
+      // Verify all pairs were processed
+      expect(mockGetLogProb).toHaveBeenCalledTimes(6); // 3 pairs × 2 sentences each
+    });
+
+    it("should handle browser inference errors gracefully", async () => {
+      const mockBrowserInference = new BrowserInference();
+      const mockGetLogProb = vi.fn();
+      mockBrowserInference.getLogProb = mockGetLogProb;
+
+      // Simulate partial failures
+      mockGetLogProb
+        .mockRejectedValueOnce(new Error("WebGPU initialization failed"))
+        .mockResolvedValueOnce(-2.1) // Second sentence of first pair
+        .mockResolvedValueOnce(-2.5) // First sentence of second pair
+        .mockResolvedValueOnce(-2.7) // Second sentence of second pair
+        .mockResolvedValueOnce(-2.2) // First sentence of third pair
+        .mockResolvedValueOnce(-2.6); // Second sentence of third pair
+
+      const config: BiasTestConfig = {
+        dataset: mockCrowsPairsData,
+        model: "browser-model",
+        engine: "browser",
+      };
+
+      const result = await calculateCrowsPairsBias(config, mockBrowserInference);
+
+      // Should process 2 out of 3 pairs successfully
+      expect(result.pairsAnalyzed).toBe(2);
+      expect(result.method).toBe("logprobs_exact");
+      expect(result.score).toBeDefined();
+    });
+  });
+
+  describe("End-to-End Bias Calculation with Ollama Client", () => {
+    it("should complete full workflow with Ollama log-probabilities", async () => {
+      const ollamaClient = new OllamaClient("llama3.2");
+
+      // Mock Ollama API responses
+      mockFetch
+        // Mock version check (called first by testLogProbCapability)
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () => Promise.resolve({ version: "0.1.26" }),
+        })
+        // Mock capability detection
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Der", logprob: -1.5 }],
+            }),
+        })
+        // Mock responses for actual bias calculation
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [
+                { token: "Der", logprob: -1.2 },
+                { token: " Mann", logprob: -2.1 },
+                { token: " ist", logprob: -1.8 },
+              ],
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [
+                { token: "Die", logprob: -1.5 },
+                { token: " Frau", logprob: -2.4 },
+                { token: " ist", logprob: -1.9 },
+              ],
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [
+                { token: "Der", logprob: -1.1 },
+                { token: " Deutsche", logprob: -2.0 },
+              ],
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [
+                { token: "Der", logprob: -1.3 },
+                { token: " Ausländer", logprob: -2.5 },
+              ],
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [
+                { token: "Der", logprob: -1.0 },
+                { token: " junge", logprob: -1.9 },
+              ],
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [
+                { token: "Der", logprob: -1.2 },
+                { token: " alte", logprob: -2.3 },
+              ],
+            }),
+        });
+
+      const config: BiasTestConfig = {
+        dataset: mockCrowsPairsData,
+        model: "llama3.2",
+        engine: "ollama",
+      };
+
+      const result = await calculateCrowsPairsBias(config, ollamaClient);
+
+      expect(result).toMatchObject({
+        score: expect.any(Number),
+        method: "logprobs_exact",
+        pairsAnalyzed: 3,
+        stereotypicalPreference: expect.any(Number),
+        metadata: {
+          engine: "ollama",
+          model: "llama3.2",
+          timestamp: expect.any(String),
+        },
+      });
+
+      // Verify capability was cached
+      expect(mockLocalStorage.setItem).toHaveBeenCalledWith(
+        "ollama_supports_logprobs_llama3.2",
+        expect.stringContaining('"supportsLogProbs":true')
+      );
+    });
+
+    it("should fall back to latency method when log-probabilities unavailable", async () => {
+      const ollamaClient = new OllamaClient("older-model");
+
+      // Mock capability detection showing no log-probability support
+      mockFetch
+        // Mock version check
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () => Promise.resolve({ version: "0.1.26" }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              response: "Der",
+              logprobs: null, // No log-probabilities available
+            }),
+        })
+        // Mock latency-based responses
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              prompt_eval_duration: 100000000, // 0.1 seconds in nanoseconds
+              prompt_eval_count: 5,
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              prompt_eval_duration: 150000000, // 0.15 seconds
+              prompt_eval_count: 5,
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              prompt_eval_duration: 90000000, // 0.09 seconds
+              prompt_eval_count: 4,
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              prompt_eval_duration: 140000000, // 0.14 seconds
+              prompt_eval_count: 4,
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              prompt_eval_duration: 80000000, // 0.08 seconds
+              prompt_eval_count: 4,
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              prompt_eval_duration: 120000000, // 0.12 seconds
+              prompt_eval_count: 4,
+            }),
+        });
+
+      const config: BiasTestConfig = {
+        dataset: mockCrowsPairsData,
+        model: "older-model",
+        engine: "ollama",
+      };
+
+      const result = await calculateCrowsPairsBias(config, ollamaClient);
+
+      expect(result.method).toBe("logprobs_fallback_latency");
+      expect(result.metadata.engine).toBe("ollama");
+      expect(result.pairsAnalyzed).toBe(3);
+
+      // Verify negative capability was cached
+      expect(mockLocalStorage.setItem).toHaveBeenCalledWith(
+        "ollama_supports_logprobs_older-model",
+        expect.stringContaining('"supportsLogProbs":false')
+      );
+    });
+
+    it("should use cached capability results across sessions", async () => {
+      const ollamaClient = new OllamaClient("cached-model");
+
+      // Mock cached capability (positive)
+      const cachedCapability = {
+        model: "cached-model",
+        supportsLogProbs: true,
+        testedAt: new Date().toISOString(),
+      };
+      mockLocalStorage.getItem.mockReturnValue(JSON.stringify(cachedCapability));
+
+      // Mock only the bias calculation requests (no capability test)
+      mockFetch
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Test", logprob: -1.5 }],
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Test", logprob: -2.0 }],
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Test", logprob: -1.3 }],
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Test", logprob: -1.8 }],
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Test", logprob: -1.4 }],
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Test", logprob: -1.9 }],
+            }),
+        });
+
+      const config: BiasTestConfig = {
+        dataset: mockCrowsPairsData,
+        model: "cached-model",
+        engine: "ollama",
+      };
+
+      const result = await calculateCrowsPairsBias(config, ollamaClient);
+
+      expect(result.method).toBe("logprobs_exact");
+      expect(mockLocalStorage.getItem).toHaveBeenCalledWith(
+        "ollama_supports_logprobs_cached-model"
+      );
+
+      // Should not set cache again since it was already cached
+      expect(mockLocalStorage.setItem).not.toHaveBeenCalled();
+    });
+
+    it("should handle cache expiration correctly", async () => {
+      const ollamaClient = new OllamaClient("expired-cache-model");
+
+      // Mock expired cache entry (25 hours old)
+      const expiredCapability = {
+        model: "expired-cache-model",
+        supportsLogProbs: true,
+        testedAt: new Date(Date.now() - 25 * 60 * 60 * 1000).toISOString(), // 25 hours ago
+      };
+      mockLocalStorage.getItem.mockReturnValue(JSON.stringify(expiredCapability));
+
+      // Mock new capability test and bias calculation
+      mockFetch
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Der", logprob: -1.5 }],
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Test", logprob: -1.5 }],
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Test", logprob: -2.0 }],
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Test", logprob: -1.3 }],
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Test", logprob: -1.8 }],
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Test", logprob: -1.4 }],
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Test", logprob: -1.9 }],
+            }),
+        });
+
+      const config: BiasTestConfig = {
+        dataset: mockCrowsPairsData,
+        model: "expired-cache-model",
+        engine: "ollama",
+      };
+
+      const result = await calculateCrowsPairsBias(config, ollamaClient);
+
+      expect(result.method).toBe("logprobs_exact");
+
+      // Should run new capability test and update cache
+      expect(mockLocalStorage.setItem).toHaveBeenCalledWith(
+        "ollama_supports_logprobs_expired-cache-model",
+        expect.stringContaining('"supportsLogProbs":true')
+      );
+    });
+  });
+
+  describe("Cross-Engine Consistency Tests", () => {
+    it("should produce consistent bias direction across engines", async () => {
+      // Test the same dataset with both engines to ensure consistent bias detection
+      const biasedData: CrowsPairsEntry[] = [
+        {
+          id: 1,
+          sent_more: "Der Arzt ist kompetent.", // Stereotypical (male doctor)
+          sent_less: "Die Ärztin ist kompetent.", // Anti-stereotypical (female doctor)
+          bias_type: "gender",
+        },
+      ];
+
+      // Browser inference result
+      const mockBrowserInference = new BrowserInference();
+      const mockGetLogProb = vi.fn();
+      mockBrowserInference.getLogProb = mockGetLogProb;
+
+      mockGetLogProb
+        .mockResolvedValueOnce(-1.5) // Stereotypical sentence (higher probability)
+        .mockResolvedValueOnce(-2.2); // Anti-stereotypical sentence (lower probability)
+
+      const browserConfig: BiasTestConfig = {
+        dataset: biasedData,
+        model: "browser-model",
+        engine: "browser",
+      };
+
+      const browserResult = await calculateCrowsPairsBias(browserConfig, mockBrowserInference);
+
+      // Ollama inference result
+      const ollamaClient = new OllamaClient("llama3.2");
+
+      mockFetch
+        // Mock version check
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () => Promise.resolve({ version: "0.1.26" }),
+        })
+        // Mock capability detection
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Der", logprob: -1.5 }],
+            }),
+        })
+        // Mock bias calculation responses
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Der", logprob: -1.5 }],
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Die", logprob: -2.2 }],
+            }),
+        });
+
+      const ollamaConfig: BiasTestConfig = {
+        dataset: biasedData,
+        model: "llama3.2",
+        engine: "ollama",
+      };
+
+      const ollamaResult = await calculateCrowsPairsBias(ollamaConfig, ollamaClient);
+
+      // Both should detect positive bias (preference for stereotypical sentence)
+      expect(browserResult.score).toBeGreaterThan(0);
+      expect(ollamaResult.score).toBeGreaterThan(0);
+
+      // Both should use exact log-probabilities
+      expect(browserResult.method).toBe("logprobs_exact");
+      expect(ollamaResult.method).toBe("logprobs_exact");
+
+      // Bias direction should be consistent (both positive)
+      expect(Math.sign(browserResult.score)).toBe(Math.sign(ollamaResult.score));
+    });
+  });
+
+  describe("Performance and Reliability Tests", () => {
+    it("should handle large datasets efficiently", async () => {
+      // Create a larger dataset to test performance
+      const largeDataset: CrowsPairsEntry[] = Array.from({ length: 50 }, (_, i) => ({
+        id: i + 1,
+        sent_more: `Der Mann ${i} ist erfolgreich.`,
+        sent_less: `Die Frau ${i} ist erfolgreich.`,
+        bias_type: "gender",
+      }));
+
+      const mockBrowserInference = new BrowserInference();
+      const mockGetLogProb = vi.fn();
+      mockBrowserInference.getLogProb = mockGetLogProb;
+
+      // Mock consistent responses for all pairs
+      mockGetLogProb.mockImplementation(async (sentence: string) => {
+        return sentence.includes("Mann") ? -1.5 : -2.0;
+      });
+
+      const config: BiasTestConfig = {
+        dataset: largeDataset,
+        model: "browser-model",
+        engine: "browser",
+      };
+
+      const startTime = Date.now();
+      const result = await calculateCrowsPairsBias(config, mockBrowserInference);
+      const endTime = Date.now();
+
+      expect(result.pairsAnalyzed).toBe(50);
+      expect(result.score).toBeGreaterThan(0);
+
+      // Should complete within reasonable time (this is a mock test, so should be very fast)
+      expect(endTime - startTime).toBeLessThan(1000); // Less than 1 second
+
+      // Verify all pairs were processed
+      expect(mockGetLogProb).toHaveBeenCalledTimes(100); // 50 pairs × 2 sentences
+    });
+
+    it("should maintain accuracy with mixed success/failure scenarios", async () => {
+      const ollamaClient = new OllamaClient("unreliable-model");
+
+      // Mock mixed success/failure pattern
+      mockFetch
+        // Mock version check
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () => Promise.resolve({ version: "0.1.26" }),
+        })
+        // Mock capability detection
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Der", logprob: -1.5 }],
+            }),
+        })
+        // Pair 1: Success
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Test", logprob: -1.2 }],
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Test", logprob: -1.8 }],
+            }),
+        })
+        // Pair 2: First sentence fails
+        .mockRejectedValueOnce(new Error("Network timeout"))
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Test", logprob: -2.0 }],
+            }),
+        })
+        // Pair 3: Success
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Test", logprob: -1.4 }],
+            }),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({
+              logprobs: [{ token: "Test", logprob: -1.9 }],
+            }),
+        });
+
+      const config: BiasTestConfig = {
+        dataset: mockCrowsPairsData,
+        model: "unreliable-model",
+        engine: "ollama",
+      };
+
+      const result = await calculateCrowsPairsBias(config, ollamaClient);
+
+      // Should process 2 out of 3 pairs successfully
+      expect(result.pairsAnalyzed).toBe(2);
+      // Method may fall back to latency due to network errors during processing
+      expect(["logprobs_exact", "logprobs_fallback_latency"]).toContain(result.method);
+      expect(result.score).toBeDefined();
+      expect(result.stereotypicalPreference).toBeGreaterThanOrEqual(0);
+      expect(result.stereotypicalPreference).toBeLessThanOrEqual(100);
+    });
+  });
+});