@artemiskit/core 0.1.6 → 0.2.0

package/dist/scenario/schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/scenario/schema.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB;;GAEG;AACH,eAAO,MAAM,cAAc,mIAWzB,CAAC;AAEH;;;;GAIG;AACH,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoBpB,CAAC;AAEd;;GAEG;AACH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IA2CzB,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,iBAAiB;;;;;;;;;EAG5B,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,eAAe,gFAAuE,CAAC;AAEpG;;;GAGG;AACH,QAAA,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;GAAuC,CAAC;AAEnE;;GAEG;AACH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAazB,6DAA6D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAE7D,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAYzB,6CAA6C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAnB7C,6DAA6D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAiC7D,CAAC;AAEH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAChE,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC"}
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/scenario/schema.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB;;GAEG;AACH,eAAO,MAAM,cAAc,mIAWzB,CAAC;AAEH;;;;GAIG;AACH,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqBpB,CAAC;AAoFd;;;GAGG;AACH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IA7BvB,0GAA0G;;IAE1G,8EAA8E;;IAE9E,uFAAuF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAJvF,0GAA0G;;QAE1G,8EAA8E;;QAE9E,uFAAuF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAyBR,CAAC;AAEpF;;GAEG;AACH,eAAO,MAAM,iBAAiB;;;;;;;;;EAG5B,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,eAAe,gFAAuE,CAAC;AAEpG;;;GAGG;AACH,QAAA,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;GAAuC,CAAC;AAEnE;;GAEG;AACH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QArDvB,0GAA0G;;QAE1G,8EAA8E;;QAE9E,uFAAuF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YAJvF,0GAA0G;;YAE1G,8EAA8E;;YAE9E,uFAAuF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IA8DzF,6DAA6D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAE7D,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAYzB,6CAA6C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YArF3C,0GAA0G;;YAE1G,8EAA8E;;YAE9E,uFAAuF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gBAJvF,0GAA0G;;gBAE1G,8EAA8E;;gBAE9E,uFAAuF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QA8DzF,6DAA6D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAiC7D,CAAC;AAEH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAChE,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@artemiskit/core",
-  "version": "0.1.6",
+  "version": "0.2.0",
   "description": "Core runner, evaluators, and storage for ArtemisKit LLM evaluation toolkit",
   "type": "module",
   "license": "Apache-2.0",

package/src/adapters/types.ts

@@ -105,7 +105,7 @@ export interface ModelClient {
 
   stream?(options: GenerateOptions, onChunk: (chunk: string) => void): AsyncIterable<string>;
 
-  embed?(text: string): Promise<number[]>;
+  embed?(text: string, model?: string): Promise<number[]>;
 
   capabilities(): Promise<ModelCapabilities>;
 
@@ -155,6 +155,8 @@ export interface AzureOpenAIAdapterConfig extends BaseAdapterConfig {
   resourceName: string;
   deploymentName: string;
   apiVersion: string;
+  /** Optional separate deployment name for embedding models */
+  embeddingDeploymentName?: string;
 }
 
 /**

package/src/artifacts/types.ts

@@ -322,6 +322,15 @@ export interface StressRequestResult {
   error?: string;
   /** Timestamp of the request */
   timestamp: number;
+  /** Token usage for this request */
+  tokens?: {
+    /** Prompt/input tokens */
+    prompt: number;
+    /** Completion/output tokens */
+    completion: number;
+    /** Total tokens */
+    total: number;
+  };
 }
 
 /**
@@ -352,6 +361,36 @@ export interface StressMetrics {
   p95_latency_ms: number;
   /** 99th percentile latency */
   p99_latency_ms: number;
+  /** Token usage metrics (optional - only if provider returns token counts) */
+  tokens?: {
+    /** Total prompt/input tokens across all requests */
+    total_prompt_tokens: number;
+    /** Total completion/output tokens across all requests */
+    total_completion_tokens: number;
+    /** Total tokens (prompt + completion) */
+    total_tokens: number;
+    /** Average tokens per request */
+    avg_tokens_per_request: number;
+  };
+  /** Estimated cost metrics (optional - only if cost estimation is available) */
+  cost?: {
+    /** Estimated total cost in USD */
+    estimated_total_usd: number;
+    /** Cost breakdown by token type */
+    breakdown: {
+      /** Cost for prompt/input tokens */
+      prompt_cost_usd: number;
+      /** Cost for completion/output tokens */
+      completion_cost_usd: number;
+    };
+    /** Model used for cost calculation */
+    model: string;
+    /** Pricing used (per 1K tokens) */
+    pricing: {
+      prompt_per_1k: number;
+      completion_per_1k: number;
+    };
+  };
 }
 
 /**

package/src/cost/index.ts

@@ -0,0 +1,14 @@
+/**
+ * Cost estimation module
+ */
+
+export {
+  MODEL_PRICING,
+  DEFAULT_PRICING,
+  getModelPricing,
+  estimateCost,
+  formatCost,
+  listKnownModels,
+  type ModelPricing,
+  type CostEstimate,
+} from './pricing.js';

package/src/cost/pricing.ts

@@ -0,0 +1,273 @@
+/**
+ * LLM Pricing Data and Cost Estimation
+ *
+ * Pricing is per 1,000 tokens (1K tokens) in USD
+ * Data is updated periodically - always verify with provider's official pricing
+ */
+
+export interface ModelPricing {
+  /** Price per 1K prompt/input tokens in USD */
+  promptPer1K: number;
+  /** Price per 1K completion/output tokens in USD */
+  completionPer1K: number;
+  /** Last updated date */
+  lastUpdated: string;
+  /** Notes about the pricing */
+  notes?: string;
+}
+
+export interface CostEstimate {
+  /** Estimated total cost in USD */
+  totalUsd: number;
+  /** Cost for prompt tokens */
+  promptCostUsd: number;
+  /** Cost for completion tokens */
+  completionCostUsd: number;
+  /** The model used for pricing */
+  model: string;
+  /** Pricing used */
+  pricing: ModelPricing;
+}
+
+/**
+ * Known model pricing data
+ * Prices are in USD per 1,000 tokens
+ */
+export const MODEL_PRICING: Record<string, ModelPricing> = {
+  // OpenAI GPT-4 family
+  'gpt-4': {
+    promptPer1K: 0.03,
+    completionPer1K: 0.06,
+    lastUpdated: '2024-01',
+  },
+  'gpt-4-32k': {
+    promptPer1K: 0.06,
+    completionPer1K: 0.12,
+    lastUpdated: '2024-01',
+  },
+  'gpt-4-turbo': {
+    promptPer1K: 0.01,
+    completionPer1K: 0.03,
+    lastUpdated: '2024-01',
+  },
+  'gpt-4-turbo-preview': {
+    promptPer1K: 0.01,
+    completionPer1K: 0.03,
+    lastUpdated: '2024-01',
+  },
+  'gpt-4o': {
+    promptPer1K: 0.005,
+    completionPer1K: 0.015,
+    lastUpdated: '2024-05',
+  },
+  'gpt-4o-mini': {
+    promptPer1K: 0.00015,
+    completionPer1K: 0.0006,
+    lastUpdated: '2024-07',
+  },
+
+  // OpenAI GPT-3.5 family
+  'gpt-3.5-turbo': {
+    promptPer1K: 0.0005,
+    completionPer1K: 0.0015,
+    lastUpdated: '2024-01',
+  },
+  'gpt-3.5-turbo-16k': {
+    promptPer1K: 0.003,
+    completionPer1K: 0.004,
+    lastUpdated: '2024-01',
+  },
+
+  // Anthropic Claude family
+  'claude-3-opus-20240229': {
+    promptPer1K: 0.015,
+    completionPer1K: 0.075,
+    lastUpdated: '2024-03',
+  },
+  'claude-3-sonnet-20240229': {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: '2024-03',
+  },
+  'claude-3-haiku-20240307': {
+    promptPer1K: 0.00025,
+    completionPer1K: 0.00125,
+    lastUpdated: '2024-03',
+  },
+  'claude-3-5-sonnet-20240620': {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: '2024-06',
+  },
+  'claude-3-5-sonnet-20241022': {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: '2024-10',
+  },
+  'claude-3-5-haiku-20241022': {
+    promptPer1K: 0.0008,
+    completionPer1K: 0.004,
+    lastUpdated: '2024-10',
+  },
+  // Aliases
+  'claude-3-opus': {
+    promptPer1K: 0.015,
+    completionPer1K: 0.075,
+    lastUpdated: '2024-03',
+  },
+  'claude-3-sonnet': {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: '2024-03',
+  },
+  'claude-3-haiku': {
+    promptPer1K: 0.00025,
+    completionPer1K: 0.00125,
+    lastUpdated: '2024-03',
+  },
+  'claude-3.5-sonnet': {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: '2024-10',
+  },
+  'claude-3.5-haiku': {
+    promptPer1K: 0.0008,
+    completionPer1K: 0.004,
+    lastUpdated: '2024-10',
+  },
+
+  // Legacy Claude
+  'claude-2': {
+    promptPer1K: 0.008,
+    completionPer1K: 0.024,
+    lastUpdated: '2024-01',
+  },
+  'claude-instant-1': {
+    promptPer1K: 0.0008,
+    completionPer1K: 0.0024,
+    lastUpdated: '2024-01',
+  },
+
+  // Azure OpenAI (same pricing as OpenAI typically)
+  // Add 'azure-' prefix versions if needed
+};
+
+/**
+ * Default pricing for unknown models
+ * Uses conservative estimates
+ */
+export const DEFAULT_PRICING: ModelPricing = {
+  promptPer1K: 0.01,
+  completionPer1K: 0.03,
+  lastUpdated: '2024-01',
+  notes: 'Default pricing - verify with provider',
+};
+
+/**
+ * Get pricing for a model
+ * @param model Model identifier
+ * @returns Pricing data or default if unknown
+ */
+export function getModelPricing(model: string): ModelPricing {
+  // Try exact match first
+  if (MODEL_PRICING[model]) {
+    return MODEL_PRICING[model];
+  }
+
+  // Try case-insensitive match
+  const lowerModel = model.toLowerCase();
+  for (const [key, pricing] of Object.entries(MODEL_PRICING)) {
+    if (key.toLowerCase() === lowerModel) {
+      return pricing;
+    }
+  }
+
+  // Try partial match for common patterns
+  if (lowerModel.includes('gpt-4o-mini')) {
+    return MODEL_PRICING['gpt-4o-mini'];
+  }
+  if (lowerModel.includes('gpt-4o')) {
+    return MODEL_PRICING['gpt-4o'];
+  }
+  if (lowerModel.includes('gpt-4-turbo')) {
+    return MODEL_PRICING['gpt-4-turbo'];
+  }
+  if (lowerModel.includes('gpt-4')) {
+    return MODEL_PRICING['gpt-4'];
+  }
+  if (lowerModel.includes('gpt-3.5')) {
+    return MODEL_PRICING['gpt-3.5-turbo'];
+  }
+  if (lowerModel.includes('claude-3-5-sonnet') || lowerModel.includes('claude-3.5-sonnet')) {
+    return MODEL_PRICING['claude-3.5-sonnet'];
+  }
+  if (lowerModel.includes('claude-3-5-haiku') || lowerModel.includes('claude-3.5-haiku')) {
+    return MODEL_PRICING['claude-3.5-haiku'];
+  }
+  if (lowerModel.includes('claude-3-opus')) {
+    return MODEL_PRICING['claude-3-opus'];
+  }
+  if (lowerModel.includes('claude-3-sonnet')) {
+    return MODEL_PRICING['claude-3-sonnet'];
+  }
+  if (lowerModel.includes('claude-3-haiku')) {
+    return MODEL_PRICING['claude-3-haiku'];
+  }
+  if (lowerModel.includes('claude')) {
+    return MODEL_PRICING['claude-2'];
+  }
+
+  return DEFAULT_PRICING;
+}
+
+/**
+ * Estimate cost for token usage
+ * @param promptTokens Number of prompt/input tokens
+ * @param completionTokens Number of completion/output tokens
+ * @param model Model identifier
+ * @returns Cost estimate
+ */
+export function estimateCost(
+  promptTokens: number,
+  completionTokens: number,
+  model: string
+): CostEstimate {
+  const pricing = getModelPricing(model);
+
+  const promptCostUsd = (promptTokens / 1000) * pricing.promptPer1K;
+  const completionCostUsd = (completionTokens / 1000) * pricing.completionPer1K;
+  const totalUsd = promptCostUsd + completionCostUsd;
+
+  return {
+    totalUsd,
+    promptCostUsd,
+    completionCostUsd,
+    model,
+    pricing,
+  };
+}
+
+/**
+ * Format cost for display
+ * @param costUsd Cost in USD
+ * @returns Formatted string
+ */
+export function formatCost(costUsd: number): string {
+  if (costUsd < 0.01) {
+    return `$${(costUsd * 100).toFixed(4)} cents`;
+  }
+  if (costUsd < 1) {
+    return `$${costUsd.toFixed(4)}`;
+  }
+  return `$${costUsd.toFixed(2)}`;
+}
+
+/**
+ * List all known models with pricing
+ */
+export function listKnownModels(): Array<{ model: string; pricing: ModelPricing }> {
+  return Object.entries(MODEL_PRICING).map(([model, pricing]) => ({
+    model,
+    pricing,
+  }));
+}

package/src/evaluators/combined.test.ts

@@ -0,0 +1,172 @@
+/**
+ * Tests for CombinedEvaluator
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { CombinedEvaluator } from './combined';
+
+describe('CombinedEvaluator', () => {
+  const evaluator = new CombinedEvaluator();
+
+  describe('AND operator', () => {
+    test('passes when all expectations pass', async () => {
+      const result = await evaluator.evaluate('The answer is 42 and it is correct.', {
+        type: 'combined',
+        operator: 'and',
+        expectations: [
+          { type: 'contains', values: ['42'], mode: 'all' },
+          { type: 'contains', values: ['correct'], mode: 'all' },
+        ],
+      });
+      expect(result.passed).toBe(true);
+      expect(result.score).toBe(1);
+      expect(result.reason).toContain('All 2 expectations passed');
+    });
+
+    test('fails when one expectation fails', async () => {
+      const result = await evaluator.evaluate('The answer is 42.', {
+        type: 'combined',
+        operator: 'and',
+        expectations: [
+          { type: 'contains', values: ['42'], mode: 'all' },
+          { type: 'contains', values: ['correct'], mode: 'all' },
+        ],
+      });
+      expect(result.passed).toBe(false);
+      expect(result.score).toBe(0.5);
+      expect(result.reason).toContain('1/2 expectations passed');
+    });
+
+    test('fails when all expectations fail', async () => {
+      const result = await evaluator.evaluate('Nothing here.', {
+        type: 'combined',
+        operator: 'and',
+        expectations: [
+          { type: 'contains', values: ['42'], mode: 'all' },
+          { type: 'contains', values: ['correct'], mode: 'all' },
+        ],
+      });
+      expect(result.passed).toBe(false);
+      expect(result.score).toBe(0);
+    });
+  });
+
+  describe('OR operator', () => {
+    test('passes when all expectations pass', async () => {
+      const result = await evaluator.evaluate('The answer is 42 and it is correct.', {
+        type: 'combined',
+        operator: 'or',
+        expectations: [
+          { type: 'contains', values: ['42'], mode: 'all' },
+          { type: 'contains', values: ['correct'], mode: 'all' },
+        ],
+      });
+      expect(result.passed).toBe(true);
+      expect(result.score).toBe(1);
+    });
+
+    test('passes when one expectation passes', async () => {
+      const result = await evaluator.evaluate('The answer is 42.', {
+        type: 'combined',
+        operator: 'or',
+        expectations: [
+          { type: 'contains', values: ['42'], mode: 'all' },
+          { type: 'contains', values: ['correct'], mode: 'all' },
+        ],
+      });
+      expect(result.passed).toBe(true);
+      expect(result.score).toBe(1); // Max score
+      expect(result.reason).toContain('1/2 expectations passed');
+    });
+
+    test('fails when all expectations fail', async () => {
+      const result = await evaluator.evaluate('Nothing here.', {
+        type: 'combined',
+        operator: 'or',
+        expectations: [
+          { type: 'contains', values: ['42'], mode: 'all' },
+          { type: 'contains', values: ['correct'], mode: 'all' },
+        ],
+      });
+      expect(result.passed).toBe(false);
+      expect(result.score).toBe(0);
+      expect(result.reason).toContain('No expectations passed');
+    });
+  });
+
+  describe('mixed expectation types', () => {
+    test('combines contains and regex expectations', async () => {
+      const result = await evaluator.evaluate('The result is 123-456-7890.', {
+        type: 'combined',
+        operator: 'and',
+        expectations: [
+          { type: 'contains', values: ['result'], mode: 'all' },
+          { type: 'regex', pattern: '\\d{3}-\\d{3}-\\d{4}' },
+        ],
+      });
+      expect(result.passed).toBe(true);
+    });
+
+    test('combines exact and contains expectations', async () => {
+      const result = await evaluator.evaluate('Hello World', {
+        type: 'combined',
+        operator: 'or',
+        expectations: [
+          { type: 'exact', value: 'Goodbye World', caseSensitive: true },
+          { type: 'contains', values: ['Hello'], mode: 'all' },
+        ],
+      });
+      expect(result.passed).toBe(true);
+    });
+
+    test('combines not_contains with contains', async () => {
+      const result = await evaluator.evaluate('The answer is correct.', {
+        type: 'combined',
+        operator: 'and',
+        expectations: [
+          { type: 'contains', values: ['correct'], mode: 'all' },
+          { type: 'not_contains', values: ['error', 'wrong'], mode: 'all' },
+        ],
+      });
+      expect(result.passed).toBe(true);
+    });
+  });
+
+  describe('edge cases', () => {
+    test('handles empty expectations array', async () => {
+      const result = await evaluator.evaluate('Any text', {
+        type: 'combined',
+        operator: 'and',
+        expectations: [],
+      });
+      expect(result.passed).toBe(true);
+      expect(result.score).toBe(1);
+    });
+
+    test('handles single expectation', async () => {
+      const result = await evaluator.evaluate('Hello World', {
+        type: 'combined',
+        operator: 'and',
+        expectations: [{ type: 'contains', values: ['Hello'], mode: 'all' }],
+      });
+      expect(result.passed).toBe(true);
+    });
+
+    test('provides detailed results', async () => {
+      const result = await evaluator.evaluate('The answer is 42.', {
+        type: 'combined',
+        operator: 'and',
+        expectations: [
+          { type: 'contains', values: ['42'], mode: 'all' },
+          { type: 'contains', values: ['correct'], mode: 'all' },
+        ],
+      });
+
+      expect(result.details).toBeDefined();
+      expect(result.details?.operator).toBe('and');
+      expect(result.details?.results).toHaveLength(2);
+      expect(result.details?.passedCount).toBe(1);
+      expect(result.details?.totalCount).toBe(2);
+    });
+  });
+});

package/src/evaluators/combined.ts

@@ -0,0 +1,95 @@
+/**
+ * Combined evaluator - evaluates multiple expectations with and/or logic
+ */
+
+import type { Expected } from '../scenario/schema';
+import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types';
+
+/**
+ * Get an evaluator by type (imported dynamically to avoid circular deps)
+ */
+async function getEvaluatorForType(type: string): Promise<Evaluator> {
+  // Dynamic import to avoid circular dependency with index.ts
+  const { getEvaluator } = await import('./index.js');
+  return getEvaluator(type);
+}
+
+export class CombinedEvaluator implements Evaluator {
+  readonly type = 'combined';
+
+  async evaluate(
+    response: string,
+    expected: Expected,
+    context?: EvaluatorContext
+  ): Promise<EvaluatorResult> {
+    if (expected.type !== 'combined') {
+      throw new Error('Invalid expected type for CombinedEvaluator');
+    }
+
+    const { operator, expectations } = expected;
+
+    if (!expectations || expectations.length === 0) {
+      return {
+        passed: true,
+        score: 1,
+        reason: 'No expectations to evaluate',
+        details: { operator, results: [] },
+      };
+    }
+
+    // Evaluate all sub-expectations
+    const results: Array<{
+      type: string;
+      passed: boolean;
+      score: number;
+      reason?: string;
+    }> = [];
+
+    for (const subExpected of expectations) {
+      const evaluator = await getEvaluatorForType(subExpected.type);
+      const result = await evaluator.evaluate(response, subExpected, context);
+      results.push({
+        type: subExpected.type,
+        passed: result.passed,
+        score: result.score,
+        reason: result.reason,
+      });
+    }
+
+    // Calculate combined result based on operator
+    let passed: boolean;
+    let score: number;
+    let reason: string;
+
+    if (operator === 'and') {
+      // AND: all must pass
+      passed = results.every((r) => r.passed);
+      score = results.reduce((sum, r) => sum + r.score, 0) / results.length;
+      const passedCount = results.filter((r) => r.passed).length;
+      reason = passed
+        ? `All ${results.length} expectations passed`
+        : `${passedCount}/${results.length} expectations passed (all required)`;
+    } else {
+      // OR: at least one must pass
+      passed = results.some((r) => r.passed);
+      // For OR, take the max score
+      score = Math.max(...results.map((r) => r.score));
+      const passedCount = results.filter((r) => r.passed).length;
+      reason = passed
+        ? `${passedCount}/${results.length} expectations passed (at least one required)`
+        : 'No expectations passed (at least one required)';
+    }
+
+    return {
+      passed,
+      score,
+      reason,
+      details: {
+        operator,
+        results,
+        passedCount: results.filter((r) => r.passed).length,
+        totalCount: results.length,
+      },
+    };
+  }
+}

package/src/evaluators/index.ts

@@ -2,12 +2,16 @@
  * Evaluators module - exports all evaluator types and utilities
  */
 
+import { CombinedEvaluator } from './combined';
 import { ContainsEvaluator } from './contains';
 import { ExactEvaluator } from './exact';
 import { FuzzyEvaluator } from './fuzzy';
+import { InlineEvaluator } from './inline';
 import { JsonSchemaEvaluator } from './json-schema';
 import { LLMGraderEvaluator } from './llm-grader';
+import { NotContainsEvaluator } from './not-contains';
 import { RegexEvaluator } from './regex';
+import { SimilarityEvaluator } from './similarity';
 import type { Evaluator } from './types';
 
 const evaluators = new Map<string, Evaluator>();
@@ -15,8 +19,12 @@ evaluators.set('exact', new ExactEvaluator());
 evaluators.set('regex', new RegexEvaluator());
 evaluators.set('fuzzy', new FuzzyEvaluator());
 evaluators.set('contains', new ContainsEvaluator());
+evaluators.set('not_contains', new NotContainsEvaluator());
+evaluators.set('combined', new CombinedEvaluator());
 evaluators.set('json_schema', new JsonSchemaEvaluator());
 evaluators.set('llm_grader', new LLMGraderEvaluator());
+evaluators.set('similarity', new SimilarityEvaluator());
+evaluators.set('inline', new InlineEvaluator());
 
 /**
  * Get an evaluator by type
@@ -49,5 +57,9 @@ export { ExactEvaluator } from './exact';
 export { RegexEvaluator } from './regex';
 export { FuzzyEvaluator } from './fuzzy';
 export { ContainsEvaluator } from './contains';
+export { NotContainsEvaluator } from './not-contains';
+export { CombinedEvaluator } from './combined';
 export { JsonSchemaEvaluator } from './json-schema';
 export { LLMGraderEvaluator } from './llm-grader';
+export { SimilarityEvaluator } from './similarity';
+export { InlineEvaluator, SUPPORTED_EXPRESSIONS } from './inline';