@artemiskit/core 0.2.0 → 0.2.3

package/src/cost/pricing.ts

@@ -3,6 +3,7 @@
  *
  * Pricing is per 1,000 tokens (1K tokens) in USD
  * Data is updated periodically - always verify with provider's official pricing
+ * Last comprehensive update: January 2026
  */
 
 export interface ModelPricing {
@@ -34,132 +35,230 @@ export interface CostEstimate {
  * Prices are in USD per 1,000 tokens
  */
 export const MODEL_PRICING: Record<string, ModelPricing> = {
-  // OpenAI GPT-4 family
-  'gpt-4': {
-    promptPer1K: 0.03,
-    completionPer1K: 0.06,
-    lastUpdated: '2024-01',
+  // ============================================
+  // OpenAI GPT-5 family (Latest - 2025)
+  // ============================================
+  'gpt-5': {
+    promptPer1K: 0.00125,
+    completionPer1K: 0.01,
+    lastUpdated: '2026-01',
+    notes: '400K context window',
   },
-  'gpt-4-32k': {
-    promptPer1K: 0.06,
-    completionPer1K: 0.12,
-    lastUpdated: '2024-01',
+  'gpt-5.1': {
+    promptPer1K: 0.00125,
+    completionPer1K: 0.01,
+    lastUpdated: '2026-01',
   },
-  'gpt-4-turbo': {
-    promptPer1K: 0.01,
-    completionPer1K: 0.03,
-    lastUpdated: '2024-01',
+  'gpt-5.2': {
+    promptPer1K: 0.00175,
+    completionPer1K: 0.014,
+    lastUpdated: '2026-01',
   },
-  'gpt-4-turbo-preview': {
-    promptPer1K: 0.01,
-    completionPer1K: 0.03,
-    lastUpdated: '2024-01',
+  'gpt-5-mini': {
+    promptPer1K: 0.00025,
+    completionPer1K: 0.002,
+    lastUpdated: '2026-01',
+  },
+  'gpt-5-nano': {
+    promptPer1K: 0.00005,
+    completionPer1K: 0.0004,
+    lastUpdated: '2026-01',
+  },
+
+  // ============================================
+  // OpenAI GPT-4.1 family (2025)
+  // ============================================
+  'gpt-4.1': {
+    promptPer1K: 0.002,
+    completionPer1K: 0.008,
+    lastUpdated: '2026-01',
+    notes: '1M context window',
+  },
+  'gpt-4.1-mini': {
+    promptPer1K: 0.0004,
+    completionPer1K: 0.0016,
+    lastUpdated: '2026-01',
+  },
+  'gpt-4.1-nano': {
+    promptPer1K: 0.0001,
+    completionPer1K: 0.0004,
+    lastUpdated: '2026-01',
   },
+
+  // ============================================
+  // OpenAI GPT-4o family (2024-2025)
+  // ============================================
   'gpt-4o': {
-    promptPer1K: 0.005,
-    completionPer1K: 0.015,
-    lastUpdated: '2024-05',
+    promptPer1K: 0.0025,
+    completionPer1K: 0.01,
+    lastUpdated: '2026-01',
+    notes: '128K context window',
   },
   'gpt-4o-mini': {
     promptPer1K: 0.00015,
     completionPer1K: 0.0006,
-    lastUpdated: '2024-07',
+    lastUpdated: '2026-01',
+    notes: '128K context window',
   },
 
-  // OpenAI GPT-3.5 family
+  // ============================================
+  // OpenAI O-series (Reasoning models)
+  // ============================================
+  o1: {
+    promptPer1K: 0.015,
+    completionPer1K: 0.06,
+    lastUpdated: '2026-01',
+    notes: 'Reasoning model - internal thinking tokens billed as output',
+  },
+  o3: {
+    promptPer1K: 0.002,
+    completionPer1K: 0.008,
+    lastUpdated: '2026-01',
+  },
+  'o3-mini': {
+    promptPer1K: 0.0011,
+    completionPer1K: 0.0044,
+    lastUpdated: '2026-01',
+  },
+  'o4-mini': {
+    promptPer1K: 0.0011,
+    completionPer1K: 0.0044,
+    lastUpdated: '2026-01',
+  },
+
+  // ============================================
+  // OpenAI Legacy GPT-4 family
+  // ============================================
+  'gpt-4-turbo': {
+    promptPer1K: 0.01,
+    completionPer1K: 0.03,
+    lastUpdated: '2026-01',
+  },
+  'gpt-4': {
+    promptPer1K: 0.03,
+    completionPer1K: 0.06,
+    lastUpdated: '2026-01',
+  },
   'gpt-3.5-turbo': {
     promptPer1K: 0.0005,
     completionPer1K: 0.0015,
-    lastUpdated: '2024-01',
+    lastUpdated: '2026-01',
   },
-  'gpt-3.5-turbo-16k': {
+
+  // ============================================
+  // Anthropic Claude 4.5 family (Latest - 2025)
+  // ============================================
+  'claude-opus-4.5': {
+    promptPer1K: 0.005,
+    completionPer1K: 0.025,
+    lastUpdated: '2026-01',
+    notes: 'Most capable Claude model',
+  },
+  'claude-sonnet-4.5': {
     promptPer1K: 0.003,
-    completionPer1K: 0.004,
-    lastUpdated: '2024-01',
+    completionPer1K: 0.015,
+    lastUpdated: '2026-01',
+    notes: 'Balanced performance and cost',
+  },
+  'claude-haiku-4.5': {
+    promptPer1K: 0.001,
+    completionPer1K: 0.005,
+    lastUpdated: '2026-01',
+    notes: 'Fastest Claude model',
   },
 
-  // Anthropic Claude family
-  'claude-3-opus-20240229': {
+  // ============================================
+  // Anthropic Claude 4 family (2025)
+  // ============================================
+  'claude-opus-4': {
     promptPer1K: 0.015,
     completionPer1K: 0.075,
-    lastUpdated: '2024-03',
+    lastUpdated: '2026-01',
   },
-  'claude-3-sonnet-20240229': {
+  'claude-opus-4.1': {
+    promptPer1K: 0.015,
+    completionPer1K: 0.075,
+    lastUpdated: '2026-01',
+  },
+  'claude-sonnet-4': {
     promptPer1K: 0.003,
     completionPer1K: 0.015,
-    lastUpdated: '2024-03',
+    lastUpdated: '2026-01',
   },
-  'claude-3-haiku-20240307': {
-    promptPer1K: 0.00025,
-    completionPer1K: 0.00125,
-    lastUpdated: '2024-03',
+
+  // ============================================
+  // Anthropic Claude 3.7 family
+  // ============================================
+  'claude-sonnet-3.7': {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: '2026-01',
   },
-  'claude-3-5-sonnet-20240620': {
+  'claude-3-7-sonnet': {
     promptPer1K: 0.003,
     completionPer1K: 0.015,
-    lastUpdated: '2024-06',
+    lastUpdated: '2026-01',
   },
+
+  // ============================================
+  // Anthropic Claude 3.5 family (Legacy)
+  // ============================================
   'claude-3-5-sonnet-20241022': {
     promptPer1K: 0.003,
     completionPer1K: 0.015,
-    lastUpdated: '2024-10',
+    lastUpdated: '2026-01',
   },
   'claude-3-5-haiku-20241022': {
     promptPer1K: 0.0008,
     completionPer1K: 0.004,
-    lastUpdated: '2024-10',
+    lastUpdated: '2026-01',
   },
-  // Aliases
+  'claude-haiku-3.5': {
+    promptPer1K: 0.0008,
+    completionPer1K: 0.004,
+    lastUpdated: '2026-01',
+  },
+
+  // ============================================
+  // Anthropic Claude 3 family (Legacy)
+  // ============================================
   'claude-3-opus': {
     promptPer1K: 0.015,
     completionPer1K: 0.075,
-    lastUpdated: '2024-03',
+    lastUpdated: '2026-01',
   },
   'claude-3-sonnet': {
     promptPer1K: 0.003,
     completionPer1K: 0.015,
-    lastUpdated: '2024-03',
+    lastUpdated: '2026-01',
   },
   'claude-3-haiku': {
     promptPer1K: 0.00025,
     completionPer1K: 0.00125,
-    lastUpdated: '2024-03',
+    lastUpdated: '2026-01',
   },
+
+  // Aliases for common naming patterns
   'claude-3.5-sonnet': {
     promptPer1K: 0.003,
     completionPer1K: 0.015,
-    lastUpdated: '2024-10',
+    lastUpdated: '2026-01',
   },
   'claude-3.5-haiku': {
     promptPer1K: 0.0008,
     completionPer1K: 0.004,
-    lastUpdated: '2024-10',
-  },
-
-  // Legacy Claude
-  'claude-2': {
-    promptPer1K: 0.008,
-    completionPer1K: 0.024,
-    lastUpdated: '2024-01',
+    lastUpdated: '2026-01',
   },
-  'claude-instant-1': {
-    promptPer1K: 0.0008,
-    completionPer1K: 0.0024,
-    lastUpdated: '2024-01',
-  },
-
-  // Azure OpenAI (same pricing as OpenAI typically)
-  // Add 'azure-' prefix versions if needed
 };
 
 /**
  * Default pricing for unknown models
- * Uses conservative estimates
+ * Uses conservative estimates based on mid-tier model pricing
  */
 export const DEFAULT_PRICING: ModelPricing = {
-  promptPer1K: 0.01,
-  completionPer1K: 0.03,
-  lastUpdated: '2024-01',
+  promptPer1K: 0.003,
+  completionPer1K: 0.015,
+  lastUpdated: '2026-01',
   notes: 'Default pricing - verify with provider',
 };
 
@@ -183,12 +282,57 @@ export function getModelPricing(model: string): ModelPricing {
   }
 
   // Try partial match for common patterns
+  // GPT-5 family
+  if (lowerModel.includes('gpt-5.2')) {
+    return MODEL_PRICING['gpt-5.2'];
+  }
+  if (lowerModel.includes('gpt-5.1')) {
+    return MODEL_PRICING['gpt-5.1'];
+  }
+  if (lowerModel.includes('gpt-5-mini')) {
+    return MODEL_PRICING['gpt-5-mini'];
+  }
+  if (lowerModel.includes('gpt-5-nano')) {
+    return MODEL_PRICING['gpt-5-nano'];
+  }
+  if (lowerModel.includes('gpt-5')) {
+    return MODEL_PRICING['gpt-5'];
+  }
+
+  // GPT-4.1 family
+  if (lowerModel.includes('gpt-4.1-mini')) {
+    return MODEL_PRICING['gpt-4.1-mini'];
+  }
+  if (lowerModel.includes('gpt-4.1-nano')) {
+    return MODEL_PRICING['gpt-4.1-nano'];
+  }
+  if (lowerModel.includes('gpt-4.1')) {
+    return MODEL_PRICING['gpt-4.1'];
+  }
+
+  // GPT-4o family
   if (lowerModel.includes('gpt-4o-mini')) {
     return MODEL_PRICING['gpt-4o-mini'];
   }
   if (lowerModel.includes('gpt-4o')) {
     return MODEL_PRICING['gpt-4o'];
   }
+
+  // O-series
+  if (lowerModel.includes('o4-mini')) {
+    return MODEL_PRICING['o4-mini'];
+  }
+  if (lowerModel.includes('o3-mini')) {
+    return MODEL_PRICING['o3-mini'];
+  }
+  if (lowerModel.includes('o3')) {
+    return MODEL_PRICING.o3;
+  }
+  if (lowerModel.includes('o1')) {
+    return MODEL_PRICING.o1;
+  }
+
+  // Legacy GPT
   if (lowerModel.includes('gpt-4-turbo')) {
     return MODEL_PRICING['gpt-4-turbo'];
   }
@@ -198,12 +342,43 @@ export function getModelPricing(model: string): ModelPricing {
   if (lowerModel.includes('gpt-3.5')) {
     return MODEL_PRICING['gpt-3.5-turbo'];
   }
+
+  // Claude 4.5 family
+  if (lowerModel.includes('opus-4.5') || lowerModel.includes('opus-4-5')) {
+    return MODEL_PRICING['claude-opus-4.5'];
+  }
+  if (lowerModel.includes('sonnet-4.5') || lowerModel.includes('sonnet-4-5')) {
+    return MODEL_PRICING['claude-sonnet-4.5'];
+  }
+  if (lowerModel.includes('haiku-4.5') || lowerModel.includes('haiku-4-5')) {
+    return MODEL_PRICING['claude-haiku-4.5'];
+  }
+
+  // Claude 4 family
+  if (lowerModel.includes('opus-4.1') || lowerModel.includes('opus-4-1')) {
+    return MODEL_PRICING['claude-opus-4.1'];
+  }
+  if (lowerModel.includes('opus-4')) {
+    return MODEL_PRICING['claude-opus-4'];
+  }
+  if (lowerModel.includes('sonnet-4')) {
+    return MODEL_PRICING['claude-sonnet-4'];
+  }
+
+  // Claude 3.7 family
+  if (lowerModel.includes('sonnet-3.7') || lowerModel.includes('sonnet-3-7')) {
+    return MODEL_PRICING['claude-sonnet-3.7'];
+  }
+
+  // Claude 3.5 family
   if (lowerModel.includes('claude-3-5-sonnet') || lowerModel.includes('claude-3.5-sonnet')) {
     return MODEL_PRICING['claude-3.5-sonnet'];
   }
   if (lowerModel.includes('claude-3-5-haiku') || lowerModel.includes('claude-3.5-haiku')) {
     return MODEL_PRICING['claude-3.5-haiku'];
   }
+
+  // Claude 3 family
   if (lowerModel.includes('claude-3-opus')) {
     return MODEL_PRICING['claude-3-opus'];
   }
@@ -213,8 +388,10 @@ export function getModelPricing(model: string): ModelPricing {
   if (lowerModel.includes('claude-3-haiku')) {
     return MODEL_PRICING['claude-3-haiku'];
   }
+
+  // Generic Claude fallback
   if (lowerModel.includes('claude')) {
-    return MODEL_PRICING['claude-2'];
+    return MODEL_PRICING['claude-sonnet-4.5'];
   }
 
   return DEFAULT_PRICING;

package/src/evaluators/llm-grader.ts

@@ -5,22 +5,27 @@
 import type { Expected } from '../scenario/schema';
 import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types';
 
-const GRADER_PROMPT = `You are an evaluator grading an AI response based on a rubric.
+const GRADER_PROMPT = `You are a strict JSON-only evaluator. You grade AI responses based on rubrics.
 
-## RUBRIC
+RUBRIC:
 {{rubric}}
 
-## RESPONSE TO EVALUATE
+RESPONSE TO EVALUATE:
 {{response}}
 
-## INSTRUCTIONS
-Score the response from 0.0 to 1.0 based on the rubric.
-Be objective and consistent in your scoring.
+TASK: Score the response from 0.0 to 1.0 based on the rubric above.
 
-Respond with ONLY a JSON object in this exact format:
-{"score": <number between 0 and 1>, "reason": "<brief explanation of score>"}
+OUTPUT FORMAT: You MUST respond with ONLY this exact JSON structure, nothing else:
+{"score":0.0,"reason":"explanation"}
 
-Do not include any other text, markdown, or formatting.`;
+RULES:
+- Output ONLY valid JSON, no markdown, no code blocks, no extra text
+- "score" must be a number between 0.0 and 1.0
+- "reason" must be a brief string explaining the score
+- Do NOT wrap in \`\`\`json or any formatting
+- Your entire response must be parseable by JSON.parse()
+
+JSON OUTPUT:`;
 
 export class LLMGraderEvaluator implements Evaluator {
   readonly type = 'llm_grader';
@@ -44,11 +49,13 @@ export class LLMGraderEvaluator implements Evaluator {
     );
 
     try {
+      // Note: Some models (like o1, o3, gpt-5-mini, reasoning models) only support temperature=1
+      // We omit temperature to let the API use its default for maximum compatibility
+      // Use higher maxTokens for reasoning models which use tokens for internal "thinking"
       const result = await context.client.generate({
         prompt,
         model: expected.model,
-        temperature: 0,
-        maxTokens: 200,
+        maxTokens: 1000,
       });
 
       const parsed = this.parseGraderResponse(result.text);
@@ -76,9 +83,25 @@ export class LLMGraderEvaluator implements Evaluator {
   }
 
   private parseGraderResponse(text: string): { score: number; reason?: string } {
-    const jsonMatch = text.match(/\{[\s\S]*?\}/);
+    // Clean up the response - remove markdown code blocks if present
+    const cleanedText = text
+      .replace(/```json\s*/gi, '')
+      .replace(/```\s*/g, '')
+      .trim();
+
+    // Try to find JSON object in the response
+    const jsonMatch = cleanedText.match(/\{[\s\S]*?\}/);
+
     if (!jsonMatch) {
-      throw new Error('No JSON found in grader response');
+      // Fallback: try to extract score from plain text patterns like "Score: 0.8" or "0.85"
+      const scoreMatch = cleanedText.match(/(?:score[:\s]*)?(\d+\.?\d*)/i);
+      if (scoreMatch) {
+        const score = Number(scoreMatch[1]);
+        if (!Number.isNaN(score) && score >= 0 && score <= 1) {
+          return { score, reason: cleanedText };
+        }
+      }
+      throw new Error(`No JSON found in grader response: ${text.substring(0, 100)}...`);
     }
 
     try {
@@ -94,6 +117,15 @@ export class LLMGraderEvaluator implements Evaluator {
         reason: parsed.reason,
       };
     } catch (error) {
+      // If JSON parsing fails, try extracting score directly
+      const scoreMatch = jsonMatch[0].match(/"score"[:\s]*(\d+\.?\d*)/i);
+      if (scoreMatch) {
+        const score = Number(scoreMatch[1]);
+        if (!Number.isNaN(score) && score >= 0 && score <= 1) {
+          const reasonMatch = jsonMatch[0].match(/"reason"[:\s]*"([^"]+)"/i);
+          return { score, reason: reasonMatch?.[1] };
+        }
+      }
       throw new Error(`Failed to parse grader response: ${(error as Error).message}`);
     }
   }

package/src/evaluators/similarity.test.ts

@@ -15,7 +15,8 @@ describe('SimilarityEvaluator', () => {
 
   test('throws on invalid expected type', async () => {
     await expect(
-      evaluator.evaluate('response', { type: 'exact', value: 'test' } as any)
+      // @ts-expect-error Testing invalid type handling
+      evaluator.evaluate('response', { type: 'exact', value: 'test' })
     ).rejects.toThrow('Invalid expected type');
   });
 
@@ -288,8 +289,8 @@ describe('SimilarityEvaluator', () => {
       {
         type: 'similarity',
         value: 'Text B',
-        // No threshold specified, should use default 0.75
-      } as any,
+        threshold: undefined, // Testing default threshold (0.75)
+      },
       mockContext
     );
 

package/src/scenario/schema.ts

@@ -44,6 +44,10 @@ export const ProviderConfigSchema = z
     apiVersion: z.string().optional(),
     embeddingDeploymentName: z.string().optional(),
 
+    // Model family for parameter detection (e.g., 'gpt-5-mini' when deployment is '5-mini')
+    // Used by OpenAI/Azure to determine which API parameters to use (max_tokens vs max_completion_tokens)
+    modelFamily: z.string().optional(),
+
     // Vercel AI specific
     underlyingProvider: z.enum(['openai', 'azure', 'anthropic', 'google', 'mistral']).optional(),
   })