@artemiskit/core 0.1.5 → 0.2.0

package/src/evaluators/similarity.test.ts

@@ -0,0 +1,333 @@
+/**
+ * Tests for semantic similarity evaluator
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { SimilarityEvaluator } from './similarity';
+import type { EvaluatorContext } from './types';
+
+describe('SimilarityEvaluator', () => {
+  const evaluator = new SimilarityEvaluator();
+
+  test('has correct type', () => {
+    expect(evaluator.type).toBe('similarity');
+  });
+
+  test('throws on invalid expected type', async () => {
+    await expect(
+      evaluator.evaluate('response', { type: 'exact', value: 'test' } as any)
+    ).rejects.toThrow('Invalid expected type');
+  });
+
+  test('fails when no client is provided', async () => {
+    const result = await evaluator.evaluate('The capital of France is Paris', {
+      type: 'similarity',
+      value: 'Paris is the capital of France',
+      threshold: 0.75,
+    });
+
+    expect(result.passed).toBe(false);
+    expect(result.score).toBe(0);
+    expect(result.reason).toContain('requires a ModelClient');
+    expect(result.details?.method).toBe('unavailable');
+  });
+
+  test('uses embedding-based similarity when embed is available', async () => {
+    // Mock client with embedding support
+    const mockContext: EvaluatorContext = {
+      client: {
+        provider: 'mock',
+        embed: async (text: string) => {
+          // Simulate semantic embeddings where similar texts have similar vectors
+          if (text.toLowerCase().includes('paris') && text.toLowerCase().includes('france')) {
+            return [0.9, 0.3, 0.1, 0.4];
+          }
+          if (text.toLowerCase().includes('paris') || text.toLowerCase().includes('france')) {
+            return [0.85, 0.35, 0.15, 0.38];
+          }
+          return [0.1, 0.8, 0.5, 0.2];
+        },
+        generate: async () => ({
+          id: '',
+          model: '',
+          text: '',
+          tokens: { prompt: 0, completion: 0, total: 0 },
+          latencyMs: 0,
+        }),
+        capabilities: async () => ({
+          streaming: false,
+          functionCalling: false,
+          toolUse: false,
+          maxContext: 4096,
+        }),
+      },
+    };
+
+    const result = await evaluator.evaluate(
+      'The capital of France is Paris',
+      {
+        type: 'similarity',
+        value: 'Paris is the capital city of France',
+        threshold: 0.75,
+      },
+      mockContext
+    );
+
+    expect(result.passed).toBe(true);
+    expect(result.score).toBeGreaterThan(0.75);
+    expect(result.details?.method).toBe('embedding');
+    expect(result.reason).toContain('embedding');
+  });
+
+  test('uses high threshold correctly', async () => {
+    const mockContext: EvaluatorContext = {
+      client: {
+        provider: 'mock',
+        embed: async (text: string) => {
+          // Return dissimilar vectors for different texts
+          if (text.includes('capital')) {
+            return [1.0, 0.0, 0.0, 0.0];
+          }
+          return [0.0, 1.0, 0.0, 0.0]; // Orthogonal vector = 0 similarity
+        },
+        generate: async () => ({
+          id: '',
+          model: '',
+          text: '',
+          tokens: { prompt: 0, completion: 0, total: 0 },
+          latencyMs: 0,
+        }),
+        capabilities: async () => ({
+          streaming: false,
+          functionCalling: false,
+          toolUse: false,
+          maxContext: 4096,
+        }),
+      },
+    };
+
+    const result = await evaluator.evaluate(
+      'Some text about weather',
+      {
+        type: 'similarity',
+        value: 'Related text about capitals',
+        threshold: 0.5, // Even moderate threshold should fail
+      },
+      mockContext
+    );
+
+    expect(result.passed).toBe(false);
+    expect(result.score).toBeLessThan(0.5);
+  });
+
+  test('falls back to LLM when embedding fails', async () => {
+    const mockContext: EvaluatorContext = {
+      client: {
+        provider: 'mock',
+        embed: async () => {
+          throw new Error('Embedding model not available');
+        },
+        generate: async () => ({
+          id: 'test',
+          model: 'mock',
+          text: '{"score": 0.85, "reason": "Both texts describe Paris as the capital of France"}',
+          tokens: { prompt: 100, completion: 20, total: 120 },
+          latencyMs: 100,
+        }),
+        capabilities: async () => ({
+          streaming: false,
+          functionCalling: false,
+          toolUse: false,
+          maxContext: 4096,
+        }),
+      },
+    };
+
+    const result = await evaluator.evaluate(
+      'The capital of France is Paris',
+      {
+        type: 'similarity',
+        value: 'Paris is the capital city of France',
+        threshold: 0.75,
+      },
+      mockContext
+    );
+
+    expect(result.passed).toBe(true);
+    expect(result.score).toBe(0.85);
+    expect(result.details?.method).toBe('llm');
+    expect(result.reason).toContain('LLM');
+  });
+
+  test('uses LLM when embed is not available', async () => {
+    const mockContext: EvaluatorContext = {
+      client: {
+        provider: 'mock',
+        // No embed method
+        generate: async () => ({
+          id: 'test',
+          model: 'mock',
+          text: '{"score": 0.92, "reason": "Semantically equivalent statements"}',
+          tokens: { prompt: 100, completion: 20, total: 120 },
+          latencyMs: 100,
+        }),
+        capabilities: async () => ({
+          streaming: false,
+          functionCalling: false,
+          toolUse: false,
+          maxContext: 4096,
+        }),
+      },
+    };
+
+    const result = await evaluator.evaluate(
+      'The weather is nice today',
+      {
+        type: 'similarity',
+        value: "It's a beautiful day outside",
+        threshold: 0.8,
+      },
+      mockContext
+    );
+
+    expect(result.passed).toBe(true);
+    expect(result.score).toBe(0.92);
+    expect(result.details?.method).toBe('llm');
+  });
+
+  test('handles invalid LLM response gracefully', async () => {
+    const mockContext: EvaluatorContext = {
+      client: {
+        provider: 'mock',
+        generate: async () => ({
+          id: 'test',
+          model: 'mock',
+          text: 'This is not valid JSON',
+          tokens: { prompt: 100, completion: 20, total: 120 },
+          latencyMs: 100,
+        }),
+        capabilities: async () => ({
+          streaming: false,
+          functionCalling: false,
+          toolUse: false,
+          maxContext: 4096,
+        }),
+      },
+    };
+
+    const result = await evaluator.evaluate(
+      'Some text',
+      {
+        type: 'similarity',
+        value: 'Some other text',
+        threshold: 0.75,
+      },
+      mockContext
+    );
+
+    expect(result.passed).toBe(false);
+    expect(result.score).toBe(0);
+    expect(result.details?.method).toBe('failed');
+  });
+
+  test('clamps score to 0-1 range', async () => {
+    const mockContext: EvaluatorContext = {
+      client: {
+        provider: 'mock',
+        generate: async () => ({
+          id: 'test',
+          model: 'mock',
+          text: '{"score": 1.5, "reason": "Invalid score"}', // Score > 1
+          tokens: { prompt: 100, completion: 20, total: 120 },
+          latencyMs: 100,
+        }),
+        capabilities: async () => ({
+          streaming: false,
+          functionCalling: false,
+          toolUse: false,
+          maxContext: 4096,
+        }),
+      },
+    };
+
+    const result = await evaluator.evaluate(
+      'Text',
+      {
+        type: 'similarity',
+        value: 'Text',
+        threshold: 0.5,
+      },
+      mockContext
+    );
+
+    expect(result.score).toBe(1); // Clamped to 1
+  });
+
+  test('uses default threshold of 0.75', async () => {
+    const mockContext: EvaluatorContext = {
+      client: {
+        provider: 'mock',
+        generate: async () => ({
+          id: 'test',
+          model: 'mock',
+          text: '{"score": 0.74, "reason": "Just below threshold"}',
+          tokens: { prompt: 100, completion: 20, total: 120 },
+          latencyMs: 100,
+        }),
+        capabilities: async () => ({
+          streaming: false,
+          functionCalling: false,
+          toolUse: false,
+          maxContext: 4096,
+        }),
+      },
+    };
+
+    const result = await evaluator.evaluate(
+      'Text A',
+      {
+        type: 'similarity',
+        value: 'Text B',
+        // No threshold specified, should use default 0.75
+      } as any,
+      mockContext
+    );
+
+    expect(result.passed).toBe(false);
+    expect(result.score).toBe(0.74);
+  });
+
+  test('extracts JSON from LLM response with surrounding text', async () => {
+    const mockContext: EvaluatorContext = {
+      client: {
+        provider: 'mock',
+        generate: async () => ({
+          id: 'test',
+          model: 'mock',
+          text: 'Here is my analysis:\n\n{"score": 0.88, "reason": "Very similar meaning"}\n\nHope this helps!',
+          tokens: { prompt: 100, completion: 50, total: 150 },
+          latencyMs: 100,
+        }),
+        capabilities: async () => ({
+          streaming: false,
+          functionCalling: false,
+          toolUse: false,
+          maxContext: 4096,
+        }),
+      },
+    };
+
+    const result = await evaluator.evaluate(
+      'Hello world',
+      {
+        type: 'similarity',
+        value: 'Hello, world!',
+        threshold: 0.8,
+      },
+      mockContext
+    );
+
+    expect(result.passed).toBe(true);
+    expect(result.score).toBe(0.88);
+  });
+});

package/src/evaluators/similarity.ts

@@ -0,0 +1,258 @@
+/**
+ * Semantic similarity evaluator
+ * Uses vector embeddings for semantic similarity matching when available,
+ * falls back to LLM-based semantic comparison otherwise.
+ */
+
+import type { Expected } from '../scenario/schema';
+import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types';
+
+/**
+ * Cosine similarity between two vectors
+ */
+function cosineSimilarity(vecA: number[], vecB: number[]): number {
+  if (vecA.length !== vecB.length) {
+    throw new Error('Vector dimensions must match');
+  }
+
+  let dotProduct = 0;
+  let magnitudeA = 0;
+  let magnitudeB = 0;
+
+  for (let i = 0; i < vecA.length; i++) {
+    dotProduct += vecA[i] * vecB[i];
+    magnitudeA += vecA[i] * vecA[i];
+    magnitudeB += vecB[i] * vecB[i];
+  }
+
+  magnitudeA = Math.sqrt(magnitudeA);
+  magnitudeB = Math.sqrt(magnitudeB);
+
+  if (magnitudeA === 0 || magnitudeB === 0) {
+    return 0;
+  }
+
+  return dotProduct / (magnitudeA * magnitudeB);
+}
+
+/**
+ * Normalize similarity score to 0-1 range
+ * Cosine similarity can be -1 to 1, we map it to 0 to 1
+ */
+function normalizeSimilarity(similarity: number): number {
+  return (similarity + 1) / 2;
+}
+
+const LLM_SIMILARITY_PROMPT = `You are a semantic similarity evaluator. Compare the semantic meaning of two texts and rate their similarity.
+
+Text A (Reference):
+"""
+{{expected}}
+"""
+
+Text B (Response):
+"""
+{{response}}
+"""
+
+Rate the semantic similarity between these texts on a scale from 0.0 to 1.0:
+- 1.0: Identical meaning, same information conveyed
+- 0.8-0.9: Very similar meaning, minor differences in phrasing
+- 0.6-0.7: Similar topic and general meaning, some differences in detail
+- 0.4-0.5: Related topics but different focus or conclusions
+- 0.2-0.3: Loosely related, different meanings
+- 0.0-0.1: Completely unrelated or contradictory
+
+Respond with ONLY a JSON object in this exact format:
+{"score": <number between 0 and 1>, "reason": "<brief 1-sentence explanation>"}`;
+
+export class SimilarityEvaluator implements Evaluator {
+  readonly type = 'similarity';
+
+  async evaluate(
+    response: string,
+    expected: Expected,
+    context?: EvaluatorContext
+  ): Promise<EvaluatorResult> {
+    if (expected.type !== 'similarity') {
+      throw new Error('Invalid expected type for SimilarityEvaluator');
+    }
+
+    const threshold = expected.threshold ?? 0.75;
+    const expectedValue = expected.value;
+    const mode = expected.mode; // 'embedding' | 'llm' | undefined (auto)
+
+    // If mode is explicitly 'llm', skip embedding and go straight to LLM
+    if (mode === 'llm') {
+      return this.evaluateWithLLM(response, expectedValue, expected.model, threshold, context);
+    }
+
+    // If mode is 'embedding' or auto (undefined), try embedding first
+    if (mode === 'embedding' || mode === undefined) {
+      // Check if embedding is available
+      if (context?.client?.embed) {
+        try {
+          const embeddingModel = expected.embeddingModel;
+          const [responseEmbedding, expectedEmbedding] = await Promise.all([
+            context.client.embed(response, embeddingModel),
+            context.client.embed(expectedValue, embeddingModel),
+          ]);
+
+          const rawSimilarity = cosineSimilarity(responseEmbedding, expectedEmbedding);
+          // For semantic embeddings, cosine similarity is typically 0-1 for similar texts
+          // We use raw similarity directly if positive, otherwise normalize
+          const similarity =
+            rawSimilarity >= 0 ? rawSimilarity : normalizeSimilarity(rawSimilarity);
+          const passed = similarity >= threshold;
+
+          return {
+            passed,
+            score: similarity,
+            reason: `Semantic similarity (embedding${embeddingModel ? `: ${embeddingModel}` : ''}): ${(similarity * 100).toFixed(1)}% (threshold: ${(threshold * 100).toFixed(1)}%)`,
+            details: {
+              method: 'embedding',
+              embeddingModel: embeddingModel || 'default',
+              similarity,
+              threshold,
+              expected: expectedValue.slice(0, 200),
+              actual: response.slice(0, 200),
+            },
+          };
+        } catch (error) {
+          // If mode is explicitly 'embedding', fail instead of falling back
+          if (mode === 'embedding') {
+            return {
+              passed: false,
+              score: 0,
+              reason: `Embedding evaluation failed: ${(error as Error).message}`,
+              details: {
+                error: (error as Error).message,
+                method: 'embedding',
+                embeddingModel: expected.embeddingModel || 'default',
+              },
+            };
+          }
+          // Auto mode: fall through to LLM-based evaluation
+          console.warn(`Embedding failed, falling back to LLM: ${(error as Error).message}`);
+        }
+      } else if (mode === 'embedding') {
+        // Explicitly requested embedding mode but no embed function available
+        return {
+          passed: false,
+          score: 0,
+          reason:
+            'Embedding mode requested but no embedding function available. Ensure the provider supports embeddings.',
+          details: {
+            error: 'No embed function available on client',
+            method: 'embedding',
+            embeddingModel: expected.embeddingModel || 'not-configured',
+          },
+        };
+      }
+    }
+
+    // Fall back to LLM-based semantic comparison (auto mode only reaches here if embedding failed/unavailable)
+    return this.evaluateWithLLM(response, expectedValue, expected.model, threshold, context);
+  }
+
+  /**
+   * Evaluate similarity using LLM-based comparison
+   */
+  private async evaluateWithLLM(
+    response: string,
+    expectedValue: string,
+    model: string | undefined,
+    threshold: number,
+    context?: EvaluatorContext
+  ): Promise<EvaluatorResult> {
+    if (!context?.client) {
+      return {
+        passed: false,
+        score: 0,
+        reason: 'Similarity evaluation requires a ModelClient (for embeddings or LLM comparison)',
+        details: {
+          error: 'No ModelClient provided in context',
+          method: 'unavailable',
+        },
+      };
+    }
+
+    try {
+      const prompt = LLM_SIMILARITY_PROMPT.replace('{{expected}}', expectedValue).replace(
+        '{{response}}',
+        response
+      );
+
+      // Note: Some models (like o1, o3, reasoning models) only support temperature=1
+      // We omit temperature to let the API use its default
+      const result = await context.client.generate({
+        prompt,
+        model,
+        maxTokens: 150,
+      });
+
+      // Parse JSON response - handle various formats including reasoning model outputs
+      // Try to find the JSON object containing score and reason
+      const jsonMatch = result.text.match(
+        /\{\s*"score"\s*:\s*[\d.]+\s*,\s*"reason"\s*:\s*"[^"]*"\s*\}/
+      );
+      const fallbackMatch = result.text.match(/\{[\s\S]*?"score"[\s\S]*?"reason"[\s\S]*?\}/);
+      const matchText = jsonMatch?.[0] || fallbackMatch?.[0];
+
+      if (!matchText) {
+        // Try to extract score from plain text as last resort
+        const scoreMatch = result.text.match(/(?:score|similarity)[:\s]*(\d*\.?\d+)/i);
+        if (scoreMatch) {
+          const extractedScore = Number.parseFloat(scoreMatch[1]);
+          const normalizedScore = extractedScore > 1 ? extractedScore / 100 : extractedScore;
+          const passed = normalizedScore >= threshold;
+          return {
+            passed,
+            score: normalizedScore,
+            reason: `Semantic similarity (LLM${model ? `: ${model}` : ''}): ${(normalizedScore * 100).toFixed(1)}% (threshold: ${(threshold * 100).toFixed(1)}%)`,
+            details: {
+              method: 'llm',
+              model: model || 'default',
+              similarity: normalizedScore,
+              threshold,
+              expected: expectedValue.slice(0, 200),
+              actual: response.slice(0, 200),
+              llmReason: 'Extracted from plain text response',
+              rawResponse: result.text.slice(0, 500),
+            },
+          };
+        }
+        throw new Error('Invalid LLM response format - could not extract score');
+      }
+
+      const parsed = JSON.parse(matchText) as { score: number; reason: string };
+      const similarity = Math.max(0, Math.min(1, parsed.score));
+      const passed = similarity >= threshold;
+
+      return {
+        passed,
+        score: similarity,
+        reason: `Semantic similarity (LLM${model ? `: ${model}` : ''}): ${(similarity * 100).toFixed(1)}% - ${parsed.reason}`,
+        details: {
+          method: 'llm',
+          model: model || 'default',
+          similarity,
+          threshold,
+          expected: expectedValue.slice(0, 200),
+          actual: response.slice(0, 200),
+          llmReason: parsed.reason,
+        },
+      };
+    } catch (error) {
+      return {
+        passed: false,
+        score: 0,
+        reason: `Similarity evaluation failed: ${(error as Error).message}`,
+        details: {
+          error: (error as Error).message,
+          method: 'failed',
+        },
+      };
+    }
+  }
+}

package/src/index.ts

@@ -29,3 +29,6 @@ export * from './utils';
 
 // Redaction
 export * from './redaction';
+
+// Cost estimation
+export * from './cost';