@artemiskit/core 0.1.6 → 0.2.2

package/src/evaluators/inline.ts

@@ -0,0 +1,393 @@
+/**
+ * Inline custom matcher evaluator
+ * Allows users to define simple matching expressions directly in YAML
+ *
+ * Supports a safe subset of JavaScript-like expressions:
+ * - String methods: includes, startsWith, endsWith, length, toLowerCase, toUpperCase, trim
+ * - Comparisons: ==, !=, >, <, >=, <=
+ * - Logical operators: &&, ||, !
+ * - Regex matching: response.match(/pattern/)
+ * - JSON parsing: JSON.parse(response)
+ */
+
+import type { Expected } from '../scenario/schema';
+import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types';
+
+/**
+ * Split a string on a delimiter, but only when the delimiter is outside of quotes
+ */
+function splitOutsideQuotes(str: string, delimiter: string): string[] {
+  const parts: string[] = [];
+  let current = '';
+  let inSingleQuote = false;
+  let inDoubleQuote = false;
+  let i = 0;
+
+  while (i < str.length) {
+    const char = str[i];
+
+    // Check for delimiter outside of quotes
+    if (!inSingleQuote && !inDoubleQuote && str.slice(i, i + delimiter.length) === delimiter) {
+      parts.push(current);
+      current = '';
+      i += delimiter.length;
+      continue;
+    }
+
+    // Track quote state (handle escaped quotes)
+    if (char === "'" && !inDoubleQuote && (i === 0 || str[i - 1] !== '\\')) {
+      inSingleQuote = !inSingleQuote;
+    } else if (char === '"' && !inSingleQuote && (i === 0 || str[i - 1] !== '\\')) {
+      inDoubleQuote = !inDoubleQuote;
+    }
+
+    current += char;
+    i++;
+  }
+
+  parts.push(current);
+  return parts;
+}
+
+/**
+ * Safe expression context with allowed variables and functions
+ */
+interface ExpressionContext {
+  response: string;
+  expected: string | undefined;
+  length: number;
+  words: string[];
+  lines: string[];
+  json: unknown | null;
+}
+
+/**
+ * Evaluate a safe expression
+ */
+function evaluateExpression(
+  expression: string,
+  context: ExpressionContext
+): { result: boolean; score: number } {
+  const { response, expected, length, words, lines, json } = context;
+
+  // Normalize the expression
+  const expr = expression.trim();
+
+  // Handle common patterns with safe evaluation
+  try {
+    // Handle combined patterns with && and || FIRST (before individual patterns)
+    // Split on && or || that appear outside of quoted strings
+    const andParts = splitOutsideQuotes(expr, '&&');
+    if (andParts.length > 1) {
+      const results = andParts.map((p) => evaluateExpression(p.trim(), context));
+      const allPassed = results.every((r) => r.result);
+      const avgScore = results.reduce((sum, r) => sum + r.score, 0) / results.length;
+      return { result: allPassed, score: avgScore };
+    }
+
+    const orParts = splitOutsideQuotes(expr, '||');
+    if (orParts.length > 1) {
+      const results = orParts.map((p) => evaluateExpression(p.trim(), context));
+      const anyPassed = results.some((r) => r.result);
+      const maxScore = Math.max(...results.map((r) => r.score));
+      return { result: anyPassed, score: maxScore };
+    }
+
+    // Pattern: response.includes("text")
+    const includesMatch = expr.match(/^response\.includes\s*\(\s*["'](.+?)["']\s*\)$/);
+    if (includesMatch) {
+      const result = response.includes(includesMatch[1]);
+      return { result, score: result ? 1 : 0 };
+    }
+
+    // Pattern: !response.includes("text")
+    const notIncludesMatch = expr.match(/^!\s*response\.includes\s*\(\s*["'](.+?)["']\s*\)$/);
+    if (notIncludesMatch) {
+      const result = !response.includes(notIncludesMatch[1]);
+      return { result, score: result ? 1 : 0 };
+    }
+
+    // Pattern: response.startsWith("text")
+    const startsWithMatch = expr.match(/^response\.startsWith\s*\(\s*["'](.+?)["']\s*\)$/);
+    if (startsWithMatch) {
+      const result = response.startsWith(startsWithMatch[1]);
+      return { result, score: result ? 1 : 0 };
+    }
+
+    // Pattern: response.endsWith("text")
+    const endsWithMatch = expr.match(/^response\.endsWith\s*\(\s*["'](.+?)["']\s*\)$/);
+    if (endsWithMatch) {
+      const result = response.endsWith(endsWithMatch[1]);
+      return { result, score: result ? 1 : 0 };
+    }
+
+    // Pattern: response.toLowerCase().includes("text")
+    const lowerIncludesMatch = expr.match(
+      /^response\.toLowerCase\s*\(\s*\)\.includes\s*\(\s*["'](.+?)["']\s*\)$/
+    );
+    if (lowerIncludesMatch) {
+      const result = response.toLowerCase().includes(lowerIncludesMatch[1].toLowerCase());
+      return { result, score: result ? 1 : 0 };
+    }
+
+    // Pattern: response.match(/regex/)
+    const regexMatch = expr.match(/^response\.match\s*\(\s*\/(.+?)\/([gimsuy]*)\s*\)$/);
+    if (regexMatch) {
+      const regex = new RegExp(regexMatch[1], regexMatch[2]);
+      const result = regex.test(response);
+      return { result, score: result ? 1 : 0 };
+    }
+
+    // Pattern: !response.match(/regex/)
+    const notRegexMatch = expr.match(/^!\s*response\.match\s*\(\s*\/(.+?)\/([gimsuy]*)\s*\)$/);
+    if (notRegexMatch) {
+      const regex = new RegExp(notRegexMatch[1], notRegexMatch[2]);
+      const result = !regex.test(response);
+      return { result, score: result ? 1 : 0 };
+    }
+
+    // Pattern: length > N, length < N, length >= N, length <= N, length == N
+    const lengthMatch = expr.match(/^length\s*(>=|<=|>|<|==|!=)\s*(\d+)$/);
+    if (lengthMatch) {
+      const op = lengthMatch[1];
+      const num = Number.parseInt(lengthMatch[2], 10);
+      let result = false;
+      switch (op) {
+        case '>':
+          result = length > num;
+          break;
+        case '<':
+          result = length < num;
+          break;
+        case '>=':
+          result = length >= num;
+          break;
+        case '<=':
+          result = length <= num;
+          break;
+        case '==':
+          result = length === num;
+          break;
+        case '!=':
+          result = length !== num;
+          break;
+      }
+      return { result, score: result ? 1 : 0 };
+    }
+
+    // Pattern: words.length > N
+    const wordsLengthMatch = expr.match(/^words\.length\s*(>=|<=|>|<|==|!=)\s*(\d+)$/);
+    if (wordsLengthMatch) {
+      const op = wordsLengthMatch[1];
+      const num = Number.parseInt(wordsLengthMatch[2], 10);
+      let result = false;
+      switch (op) {
+        case '>':
+          result = words.length > num;
+          break;
+        case '<':
+          result = words.length < num;
+          break;
+        case '>=':
+          result = words.length >= num;
+          break;
+        case '<=':
+          result = words.length <= num;
+          break;
+        case '==':
+          result = words.length === num;
+          break;
+        case '!=':
+          result = words.length !== num;
+          break;
+      }
+      return { result, score: result ? 1 : 0 };
+    }
+
+    // Pattern: lines.length > N
+    const linesLengthMatch = expr.match(/^lines\.length\s*(>=|<=|>|<|==|!=)\s*(\d+)$/);
+    if (linesLengthMatch) {
+      const op = linesLengthMatch[1];
+      const num = Number.parseInt(linesLengthMatch[2], 10);
+      let result = false;
+      switch (op) {
+        case '>':
+          result = lines.length > num;
+          break;
+        case '<':
+          result = lines.length < num;
+          break;
+        case '>=':
+          result = lines.length >= num;
+          break;
+        case '<=':
+          result = lines.length <= num;
+          break;
+        case '==':
+          result = lines.length === num;
+          break;
+        case '!=':
+          result = lines.length !== num;
+          break;
+      }
+      return { result, score: result ? 1 : 0 };
+    }
+
+    // Pattern: json.field == "value" or json.field == value
+    const jsonFieldMatch = expr.match(
+      /^json\.(\w+(?:\.\w+)*)\s*(>=|<=|>|<|==|!=)\s*(?:["'](.+?)["']|(\d+(?:\.\d+)?)|(true|false|null))$/
+    );
+    if (jsonFieldMatch && json !== null) {
+      const path = jsonFieldMatch[1];
+      const op = jsonFieldMatch[2];
+      let compareValue: unknown = jsonFieldMatch[3] ?? jsonFieldMatch[4] ?? jsonFieldMatch[5];
+
+      // Parse numbers and booleans
+      if (compareValue === 'true') compareValue = true;
+      else if (compareValue === 'false') compareValue = false;
+      else if (compareValue === 'null') compareValue = null;
+      else if (jsonFieldMatch[4]) compareValue = Number.parseFloat(jsonFieldMatch[4]);
+
+      // Navigate JSON path
+      let fieldValue: unknown = json;
+      for (const key of path.split('.')) {
+        if (fieldValue && typeof fieldValue === 'object' && key in fieldValue) {
+          fieldValue = (fieldValue as Record<string, unknown>)[key];
+        } else {
+          fieldValue = undefined;
+          break;
+        }
+      }
+
+      let result = false;
+      switch (op) {
+        case '==':
+          result = fieldValue === compareValue;
+          break;
+        case '!=':
+          result = fieldValue !== compareValue;
+          break;
+        case '>':
+          result = typeof fieldValue === 'number' && fieldValue > (compareValue as number);
+          break;
+        case '<':
+          result = typeof fieldValue === 'number' && fieldValue < (compareValue as number);
+          break;
+        case '>=':
+          result = typeof fieldValue === 'number' && fieldValue >= (compareValue as number);
+          break;
+        case '<=':
+          result = typeof fieldValue === 'number' && fieldValue <= (compareValue as number);
+          break;
+      }
+      return { result, score: result ? 1 : 0 };
+    }
+
+    // Pattern: json != null (check if valid JSON)
+    if (expr === 'json != null') {
+      const result = json !== null;
+      return { result, score: result ? 1 : 0 };
+    }
+
+    // Pattern: response == expected
+    if (expr === 'response == expected' && expected !== undefined) {
+      const result = response === expected;
+      return { result, score: result ? 1 : 0 };
+    }
+
+    // Pattern: response.trim() == expected
+    if (expr === 'response.trim() == expected' && expected !== undefined) {
+      const result = response.trim() === expected;
+      return { result, score: result ? 1 : 0 };
+    }
+
+    // Unknown expression pattern
+    throw new Error(`Unsupported expression pattern: ${expr}`);
+  } catch (error) {
+    throw new Error(`Expression evaluation failed: ${(error as Error).message}`);
+  }
+}
+
+export class InlineEvaluator implements Evaluator {
+  readonly type = 'inline';
+
+  async evaluate(
+    response: string,
+    expected: Expected,
+    _context?: EvaluatorContext
+  ): Promise<EvaluatorResult> {
+    if (expected.type !== 'inline') {
+      throw new Error('Invalid expected type for InlineEvaluator');
+    }
+
+    const expression = expected.expression;
+    const expectedValue = expected.value;
+
+    // Build context
+    let json: unknown = null;
+    try {
+      json = JSON.parse(response);
+    } catch {
+      // Not valid JSON, that's fine
+    }
+
+    const context: ExpressionContext = {
+      response,
+      expected: expectedValue,
+      length: response.length,
+      words: response.split(/\s+/).filter((w) => w.length > 0),
+      lines: response.split('\n').filter((l) => l.trim().length > 0),
+      json,
+    };
+
+    try {
+      const { result, score } = evaluateExpression(expression, context);
+
+      return {
+        passed: result,
+        score,
+        reason: result ? `Expression passed: ${expression}` : `Expression failed: ${expression}`,
+        details: {
+          expression,
+          expectedValue,
+          responseLength: response.length,
+          wordCount: context.words.length,
+          lineCount: context.lines.length,
+          isValidJson: json !== null,
+        },
+      };
+    } catch (error) {
+      return {
+        passed: false,
+        score: 0,
+        reason: `Inline matcher error: ${(error as Error).message}`,
+        details: {
+          expression,
+          error: (error as Error).message,
+        },
+      };
+    }
+  }
+}
+
+/**
+ * List of supported expression patterns for documentation
+ */
+export const SUPPORTED_EXPRESSIONS = [
+  'response.includes("text")',
+  '!response.includes("text")',
+  'response.startsWith("text")',
+  'response.endsWith("text")',
+  'response.toLowerCase().includes("text")',
+  'response.match(/regex/)',
+  '!response.match(/regex/)',
+  'length > N / length < N / length >= N / length <= N / length == N',
+  'words.length > N',
+  'lines.length > N',
+  'json.field == "value"',
+  'json.field > N',
+  'json != null',
+  'response == expected',
+  'expression1 && expression2',
+  'expression1 || expression2',
+];

package/src/evaluators/llm-grader.ts

@@ -5,22 +5,27 @@
 import type { Expected } from '../scenario/schema';
 import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types';
 
-const GRADER_PROMPT = `You are an evaluator grading an AI response based on a rubric.
+const GRADER_PROMPT = `You are a strict JSON-only evaluator. You grade AI responses based on rubrics.
 
-## RUBRIC
+RUBRIC:
 {{rubric}}
 
-## RESPONSE TO EVALUATE
+RESPONSE TO EVALUATE:
 {{response}}
 
-## INSTRUCTIONS
-Score the response from 0.0 to 1.0 based on the rubric.
-Be objective and consistent in your scoring.
+TASK: Score the response from 0.0 to 1.0 based on the rubric above.
 
-Respond with ONLY a JSON object in this exact format:
-{"score": <number between 0 and 1>, "reason": "<brief explanation of score>"}
+OUTPUT FORMAT: You MUST respond with ONLY this exact JSON structure, nothing else:
+{"score":0.0,"reason":"explanation"}
 
-Do not include any other text, markdown, or formatting.`;
+RULES:
+- Output ONLY valid JSON, no markdown, no code blocks, no extra text
+- "score" must be a number between 0.0 and 1.0
+- "reason" must be a brief string explaining the score
+- Do NOT wrap in \`\`\`json or any formatting
+- Your entire response must be parseable by JSON.parse()
+
+JSON OUTPUT:`;
 
 export class LLMGraderEvaluator implements Evaluator {
   readonly type = 'llm_grader';
@@ -44,11 +49,13 @@ export class LLMGraderEvaluator implements Evaluator {
     );
 
     try {
+      // Note: Some models (like o1, o3, gpt-5-mini, reasoning models) only support temperature=1
+      // We omit temperature to let the API use its default for maximum compatibility
+      // Use higher maxTokens for reasoning models which use tokens for internal "thinking"
       const result = await context.client.generate({
         prompt,
         model: expected.model,
-        temperature: 0,
-        maxTokens: 200,
+        maxTokens: 1000,
       });
 
       const parsed = this.parseGraderResponse(result.text);
@@ -76,9 +83,25 @@ export class LLMGraderEvaluator implements Evaluator {
   }
 
   private parseGraderResponse(text: string): { score: number; reason?: string } {
-    const jsonMatch = text.match(/\{[\s\S]*?\}/);
+    // Clean up the response - remove markdown code blocks if present
+    const cleanedText = text
+      .replace(/```json\s*/gi, '')
+      .replace(/```\s*/g, '')
+      .trim();
+
+    // Try to find JSON object in the response
+    const jsonMatch = cleanedText.match(/\{[\s\S]*?\}/);
+
     if (!jsonMatch) {
-      throw new Error('No JSON found in grader response');
+      // Fallback: try to extract score from plain text patterns like "Score: 0.8" or "0.85"
+      const scoreMatch = cleanedText.match(/(?:score[:\s]*)?(\d+\.?\d*)/i);
+      if (scoreMatch) {
+        const score = Number(scoreMatch[1]);
+        if (!Number.isNaN(score) && score >= 0 && score <= 1) {
+          return { score, reason: cleanedText };
+        }
+      }
+      throw new Error(`No JSON found in grader response: ${text.substring(0, 100)}...`);
     }
 
     try {
@@ -94,6 +117,15 @@ export class LLMGraderEvaluator implements Evaluator {
         reason: parsed.reason,
       };
     } catch (error) {
+      // If JSON parsing fails, try extracting score directly
+      const scoreMatch = jsonMatch[0].match(/"score"[:\s]*(\d+\.?\d*)/i);
+      if (scoreMatch) {
+        const score = Number(scoreMatch[1]);
+        if (!Number.isNaN(score) && score >= 0 && score <= 1) {
+          const reasonMatch = jsonMatch[0].match(/"reason"[:\s]*"([^"]+)"/i);
+          return { score, reason: reasonMatch?.[1] };
+        }
+      }
       throw new Error(`Failed to parse grader response: ${(error as Error).message}`);
     }
   }

package/src/evaluators/not-contains.test.ts

@@ -0,0 +1,105 @@
+/**
+ * Tests for NotContainsEvaluator
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { NotContainsEvaluator } from './not-contains';
+
+describe('NotContainsEvaluator', () => {
+  const evaluator = new NotContainsEvaluator();
+
+  test('passes when no forbidden values are present (mode: all)', async () => {
+    const result = await evaluator.evaluate('The colors are green and purple.', {
+      type: 'not_contains',
+      values: ['red', 'blue', 'yellow'],
+      mode: 'all',
+    });
+    expect(result.passed).toBe(true);
+    expect(result.score).toBe(1);
+  });
+
+  test('fails when any forbidden value is present (mode: all)', async () => {
+    const result = await evaluator.evaluate('The colors are red and green.', {
+      type: 'not_contains',
+      values: ['red', 'blue', 'yellow'],
+      mode: 'all',
+    });
+    expect(result.passed).toBe(false);
+    expect(result.score).toBeCloseTo(0.67, 1);
+  });
+
+  test('fails when all forbidden values are present (mode: all)', async () => {
+    const result = await evaluator.evaluate('The colors are red, blue, and yellow.', {
+      type: 'not_contains',
+      values: ['red', 'blue', 'yellow'],
+      mode: 'all',
+    });
+    expect(result.passed).toBe(false);
+    expect(result.score).toBe(0);
+  });
+
+  test('passes when at least one forbidden value is absent (mode: any)', async () => {
+    const result = await evaluator.evaluate('The colors are red and blue.', {
+      type: 'not_contains',
+      values: ['red', 'blue', 'yellow'],
+      mode: 'any',
+    });
+    expect(result.passed).toBe(true);
+    expect(result.score).toBeCloseTo(0.33, 1);
+  });
+
+  test('fails when all forbidden values are present (mode: any)', async () => {
+    const result = await evaluator.evaluate('I have red, blue, and yellow paint.', {
+      type: 'not_contains',
+      values: ['red', 'blue', 'yellow'],
+      mode: 'any',
+    });
+    expect(result.passed).toBe(false);
+    expect(result.score).toBe(0);
+  });
+
+  test('is case insensitive', async () => {
+    const result = await evaluator.evaluate('GREEN PURPLE ORANGE', {
+      type: 'not_contains',
+      values: ['red', 'blue', 'yellow'],
+      mode: 'all',
+    });
+    expect(result.passed).toBe(true);
+  });
+
+  test('detects values case insensitively', async () => {
+    const result = await evaluator.evaluate('I have RED paint', {
+      type: 'not_contains',
+      values: ['red'],
+      mode: 'all',
+    });
+    expect(result.passed).toBe(false);
+  });
+
+  test('handles empty values array', async () => {
+    const result = await evaluator.evaluate('Any response text here', {
+      type: 'not_contains',
+      values: [],
+      mode: 'all',
+    });
+    expect(result.passed).toBe(true);
+    expect(result.score).toBe(1);
+  });
+
+  test('provides detailed results in details field', async () => {
+    const result = await evaluator.evaluate('The answer is red and green.', {
+      type: 'not_contains',
+      values: ['red', 'blue'],
+      mode: 'all',
+    });
+    expect(result.details).toEqual({
+      mode: 'all',
+      results: [
+        { value: 'red', found: true },
+        { value: 'blue', found: false },
+      ],
+      notFoundCount: 1,
+      totalCount: 2,
+    });
+  });
+});

package/src/evaluators/not-contains.ts

@@ -0,0 +1,45 @@
+/**
+ * Not Contains evaluator - checks if response does NOT contain specific values
+ */
+
+import type { Expected } from '../scenario/schema';
+import type { Evaluator, EvaluatorResult } from './types';
+
+export class NotContainsEvaluator implements Evaluator {
+  readonly type = 'not_contains';
+
+  async evaluate(response: string, expected: Expected): Promise<EvaluatorResult> {
+    if (expected.type !== 'not_contains') {
+      throw new Error('Invalid expected type for NotContainsEvaluator');
+    }
+
+    const normalizedResponse = response.toLowerCase();
+    const results = expected.values.map((value) => ({
+      value,
+      found: normalizedResponse.includes(value.toLowerCase()),
+    }));
+
+    const notFoundCount = results.filter((r) => !r.found).length;
+
+    // mode: 'all' means ALL values must be absent
+    // mode: 'any' means AT LEAST ONE value must be absent
+    const passed =
+      expected.mode === 'all' ? notFoundCount === expected.values.length : notFoundCount > 0;
+
+    const score = expected.values.length > 0 ? notFoundCount / expected.values.length : 1;
+
+    return {
+      passed,
+      score,
+      reason: passed
+        ? `Correctly absent: ${notFoundCount}/${expected.values.length} values (mode: ${expected.mode})`
+        : `Found forbidden values (mode: ${expected.mode})`,
+      details: {
+        mode: expected.mode,
+        results,
+        notFoundCount,
+        totalCount: expected.values.length,
+      },
+    };
+  }
+}