@artemiskit/core 0.1.2

package/src/runner/executor.ts

@@ -0,0 +1,251 @@
+/**
+ * Test case executor
+ */
+
+import type { CaseRedactionInfo, CaseResult } from '../artifacts/types';
+import { getEvaluator } from '../evaluators';
+import { type RedactionConfig, Redactor } from '../redaction';
+import type { TestCase } from '../scenario/schema';
+import { mergeVariables, substituteVariables } from '../scenario/variables';
+import type { ExecutorContext } from './types';
+
+/**
+ * Merge redaction configs with priority: CLI > case > scenario
+ */
+function mergeRedactionConfig(
+  scenarioConfig?: RedactionConfig,
+  caseConfig?: RedactionConfig,
+  cliConfig?: RedactionConfig
+): RedactionConfig {
+  // CLI config takes highest priority if enabled is explicitly set
+  if (cliConfig?.enabled !== undefined) {
+    return {
+      enabled: cliConfig.enabled,
+      patterns: cliConfig.patterns ?? caseConfig?.patterns ?? scenarioConfig?.patterns,
+      redactPrompts:
+        cliConfig.redactPrompts ??
+        caseConfig?.redactPrompts ??
+        scenarioConfig?.redactPrompts ??
+        true,
+      redactResponses:
+        cliConfig.redactResponses ??
+        caseConfig?.redactResponses ??
+        scenarioConfig?.redactResponses ??
+        true,
+      redactMetadata:
+        cliConfig.redactMetadata ??
+        caseConfig?.redactMetadata ??
+        scenarioConfig?.redactMetadata ??
+        false,
+      replacement:
+        cliConfig.replacement ??
+        caseConfig?.replacement ??
+        scenarioConfig?.replacement ??
+        '[REDACTED]',
+    };
+  }
+
+  // Case config takes priority over scenario
+  if (caseConfig?.enabled !== undefined) {
+    return {
+      enabled: caseConfig.enabled,
+      patterns: caseConfig.patterns ?? scenarioConfig?.patterns,
+      redactPrompts: caseConfig.redactPrompts ?? scenarioConfig?.redactPrompts ?? true,
+      redactResponses: caseConfig.redactResponses ?? scenarioConfig?.redactResponses ?? true,
+      redactMetadata: caseConfig.redactMetadata ?? scenarioConfig?.redactMetadata ?? false,
+      replacement: caseConfig.replacement ?? scenarioConfig?.replacement ?? '[REDACTED]',
+    };
+  }
+
+  // Fall back to scenario config
+  if (scenarioConfig?.enabled) {
+    return {
+      enabled: scenarioConfig.enabled,
+      patterns: scenarioConfig.patterns,
+      redactPrompts: scenarioConfig.redactPrompts ?? true,
+      redactResponses: scenarioConfig.redactResponses ?? true,
+      redactMetadata: scenarioConfig.redactMetadata ?? false,
+      replacement: scenarioConfig.replacement ?? '[REDACTED]',
+    };
+  }
+
+  // Default: disabled
+  return {
+    enabled: false,
+    redactPrompts: true,
+    redactResponses: true,
+    redactMetadata: false,
+    replacement: '[REDACTED]',
+  };
+}
+
+/**
+ * Execute a single test case
+ */
+export async function executeCase(
+  testCase: TestCase,
+  context: ExecutorContext
+): Promise<CaseResult> {
+  const { timeout, retries = 0 } = context;
+  const caseStartTime = Date.now();
+
+  let lastError: Error | null = null;
+
+  for (let attempt = 0; attempt <= retries; attempt++) {
+    try {
+      const result = await executeCaseAttempt(testCase, context, timeout);
+      return result;
+    } catch (error) {
+      lastError = error as Error;
+      if (attempt < retries) {
+        // Wait before retry with exponential backoff
+        await sleep(2 ** attempt * 1000);
+      }
+    }
+  }
+
+  // All retries failed
+  const latencyMs = Date.now() - caseStartTime;
+  return {
+    id: testCase.id,
+    name: testCase.name,
+    ok: false,
+    score: 0,
+    matcherType: testCase.expected.type,
+    reason: `Failed after ${retries + 1} attempts: ${lastError?.message}`,
+    latencyMs,
+    tokens: { prompt: 0, completion: 0, total: 0 },
+    prompt: testCase.prompt,
+    response: '',
+    expected: testCase.expected,
+    tags: testCase.tags,
+    error: lastError?.message,
+  };
+}
+
+async function executeCaseAttempt(
+  testCase: TestCase,
+  context: ExecutorContext,
+  timeout?: number
+): Promise<CaseResult> {
+  const { client, scenario, redaction: cliRedaction } = context;
+
+  // Merge scenario-level and case-level variables (case overrides scenario)
+  const variables = mergeVariables(scenario.variables, testCase.variables);
+
+  // Apply variable substitution to prompt
+  let prompt = substituteVariables(testCase.prompt, variables);
+
+  // Build prompt with system prompt if present
+  if (scenario.setup?.systemPrompt && typeof prompt === 'string') {
+    const systemPrompt = substituteVariables(scenario.setup.systemPrompt, variables);
+    prompt = [
+      { role: 'system' as const, content: systemPrompt },
+      { role: 'user' as const, content: prompt },
+    ];
+  }
+
+  // Generate response with optional timeout
+  const generatePromise = client.generate({
+    prompt,
+    model: testCase.model || scenario.model,
+    temperature: scenario.temperature,
+    maxTokens: scenario.maxTokens,
+    seed: scenario.seed,
+  });
+
+  const result = timeout
+    ? await Promise.race([generatePromise, createTimeout(timeout)])
+    : await generatePromise;
+
+  // Evaluate response
+  const evaluator = getEvaluator(testCase.expected.type);
+  const evalResult = await evaluator.evaluate(result.text, testCase.expected, {
+    client,
+    testCase,
+  });
+
+  // Determine effective redaction config (CLI > case > scenario)
+  const effectiveRedaction = mergeRedactionConfig(
+    scenario.redaction,
+    testCase.redaction,
+    cliRedaction
+  );
+
+  // Apply redaction if enabled
+  let finalPrompt: string | object = testCase.prompt;
+  let finalResponse = result.text;
+  let redactionInfo: CaseRedactionInfo | undefined;
+
+  if (effectiveRedaction.enabled) {
+    const redactor = new Redactor(effectiveRedaction);
+
+    let promptRedacted = false;
+    let responseRedacted = false;
+    let totalRedactions = 0;
+
+    // Redact prompt if configured
+    if (effectiveRedaction.redactPrompts) {
+      if (typeof finalPrompt === 'string') {
+        const promptResult = redactor.redactPrompt(finalPrompt);
+        finalPrompt = promptResult.text;
+        promptRedacted = promptResult.wasRedacted;
+        totalRedactions += promptResult.redactionCount;
+      } else if (Array.isArray(finalPrompt)) {
+        // Handle chat message array
+        finalPrompt = finalPrompt.map((msg) => {
+          if (typeof msg === 'object' && 'content' in msg && typeof msg.content === 'string') {
+            const promptResult = redactor.redactPrompt(msg.content);
+            if (promptResult.wasRedacted) {
+              promptRedacted = true;
+              totalRedactions += promptResult.redactionCount;
+            }
+            return { ...msg, content: promptResult.text };
+          }
+          return msg;
+        });
+      }
+    }
+
+    // Redact response if configured
+    if (effectiveRedaction.redactResponses) {
+      const responseResult = redactor.redactResponse(finalResponse);
+      finalResponse = responseResult.text;
+      responseRedacted = responseResult.wasRedacted;
+      totalRedactions += responseResult.redactionCount;
+    }
+
+    redactionInfo = {
+      redacted: promptRedacted || responseRedacted,
+      promptRedacted,
+      responseRedacted,
+      redactionCount: totalRedactions,
+    };
+  }
+
+  return {
+    id: testCase.id,
+    name: testCase.name,
+    ok: evalResult.passed,
+    score: evalResult.score,
+    matcherType: testCase.expected.type,
+    reason: evalResult.reason,
+    latencyMs: result.latencyMs,
+    tokens: result.tokens,
+    prompt: finalPrompt,
+    response: finalResponse,
+    expected: testCase.expected,
+    tags: testCase.tags,
+    redaction: redactionInfo,
+  };
+}
+
+function createTimeout(ms: number): Promise<never> {
+  return new Promise((_, reject) => {
+    setTimeout(() => reject(new Error(`Timeout after ${ms}ms`)), ms);
+  });
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/src/runner/index.ts

@@ -0,0 +1,7 @@
+/**
+ * Runner module exports
+ */
+
+export * from './types';
+export { executeCase } from './executor';
+export { runScenario, runScenarios } from './runner';

package/src/runner/runner.ts

@@ -0,0 +1,153 @@
+/**
+ * Scenario runner - main entry point for running test scenarios
+ */
+
+import { createRunManifest } from '../artifacts/manifest';
+import type { CaseResult, ManifestRedactionInfo } from '../artifacts/types';
+import { Redactor } from '../redaction';
+import { executeCase } from './executor';
+import type { RunOptions, RunResult } from './types';
+
+/**
+ * Run a test scenario
+ */
+export async function runScenario(options: RunOptions): Promise<RunResult> {
+  const {
+    scenario,
+    client,
+    project = process.env.ARTEMIS_PROJECT || 'default',
+    resolvedConfig,
+    tags,
+    concurrency = 1,
+    timeout,
+    retries,
+    redaction,
+    onCaseComplete,
+    onProgress,
+  } = options;
+
+  // Filter cases by tags if specified
+  let cases = scenario.cases;
+  if (tags && tags.length > 0) {
+    cases = cases.filter((c) => tags.some((tag) => c.tags.includes(tag)));
+    onProgress?.(`Filtered to ${cases.length} cases by tags: ${tags.join(', ')}`);
+  }
+
+  if (cases.length === 0) {
+    throw new Error('No test cases to run after filtering');
+  }
+
+  onProgress?.(`Running ${cases.length} test cases...`);
+
+  const startTime = new Date();
+  const results: CaseResult[] = [];
+
+  if (concurrency === 1) {
+    // Sequential execution
+    for (let i = 0; i < cases.length; i++) {
+      const testCase = cases[i];
+      const result = await executeCase(testCase, {
+        client,
+        scenario,
+        timeout: testCase.timeout || timeout,
+        retries: testCase.retries ?? retries,
+        redaction,
+      });
+      results.push(result);
+      onCaseComplete?.(result, i, cases.length);
+    }
+  } else {
+    // Concurrent execution with limited parallelism
+    const chunks = chunkArray(cases, concurrency);
+    let completed = 0;
+
+    for (const chunk of chunks) {
+      const chunkResults = await Promise.all(
+        chunk.map(async (testCase) => {
+          const result = await executeCase(testCase, {
+            client,
+            scenario,
+            timeout: testCase.timeout || timeout,
+            retries: testCase.retries ?? retries,
+            redaction,
+          });
+          completed++;
+          onCaseComplete?.(result, completed - 1, cases.length);
+          return result;
+        })
+      );
+      results.push(...chunkResults);
+    }
+  }
+
+  const endTime = new Date();
+
+  // Calculate redaction metadata if any redaction occurred
+  let redactionInfo: ManifestRedactionInfo | undefined;
+  const effectiveRedaction = redaction ?? scenario.redaction;
+
+  if (effectiveRedaction?.enabled) {
+    const redactor = new Redactor(effectiveRedaction);
+    const promptsRedacted = results.filter((r) => r.redaction?.promptRedacted).length;
+    const responsesRedacted = results.filter((r) => r.redaction?.responseRedacted).length;
+    const totalRedactions = results.reduce((sum, r) => sum + (r.redaction?.redactionCount ?? 0), 0);
+
+    redactionInfo = {
+      enabled: true,
+      patternsUsed: redactor.patternNames,
+      replacement: redactor.replacement,
+      summary: {
+        promptsRedacted,
+        responsesRedacted,
+        totalRedactions,
+      },
+    };
+  }
+
+  // Create manifest
+  const manifest = createRunManifest({
+    project,
+    config: {
+      scenario: scenario.name,
+      provider: client.provider,
+      model: resolvedConfig?.model || scenario.model,
+      temperature: resolvedConfig?.temperature ?? scenario.temperature,
+      seed: scenario.seed,
+    },
+    resolvedConfig,
+    cases: results,
+    startTime,
+    endTime,
+    redaction: redactionInfo,
+  });
+
+  const success = manifest.metrics.failed_cases === 0;
+
+  return {
+    manifest,
+    cases: results,
+    success,
+  };
+}
+
+/**
+ * Run multiple scenarios
+ */
+export async function runScenarios(optionsList: RunOptions[]): Promise<RunResult[]> {
+  const results: RunResult[] = [];
+
+  for (const options of optionsList) {
+    const result = await runScenario(options);
+    results.push(result);
+  }
+
+  return results;
+}
+
+function chunkArray<T>(array: T[], size: number): T[][] {
+  const chunks: T[][] = [];
+  for (let i = 0; i < array.length; i += size) {
+    chunks.push(array.slice(i, i + size));
+  }
+  return chunks;
+}

package/src/runner/types.ts

@@ -0,0 +1,60 @@
+/**
+ * Runner types and interfaces
+ */
+
+import type { ModelClient } from '../adapters/types';
+import type { CaseResult, ResolvedConfig, RunManifest } from '../artifacts/types';
+import type { RedactionConfig } from '../redaction/types';
+import type { Scenario } from '../scenario/schema';
+
+/**
+ * Options for running a scenario
+ */
+export interface RunOptions {
+  /** The scenario to run */
+  scenario: Scenario;
+  /** Model client to use */
+  client: ModelClient;
+  /** Project name for the manifest */
+  project?: string;
+  /** Resolved configuration with source tracking */
+  resolvedConfig?: ResolvedConfig;
+  /** Filter cases by tags */
+  tags?: string[];
+  /** Number of concurrent requests */
+  concurrency?: number;
+  /** Timeout per case in milliseconds */
+  timeout?: number;
+  /** Number of retries per case */
+  retries?: number;
+  /** Redaction configuration (CLI overrides scenario) */
+  redaction?: RedactionConfig;
+  /** Callback for each case result */
+  onCaseComplete?: (result: CaseResult, index: number, total: number) => void;
+  /** Callback for progress updates */
+  onProgress?: (message: string) => void;
+}
+
+/**
+ * Result of a scenario run
+ */
+export interface RunResult {
+  /** The generated manifest */
+  manifest: RunManifest;
+  /** Individual case results */
+  cases: CaseResult[];
+  /** Whether all cases passed */
+  success: boolean;
+}
+
+/**
+ * Context passed to case executor
+ */
+export interface ExecutorContext {
+  client: ModelClient;
+  scenario: Scenario;
+  timeout?: number;
+  retries?: number;
+  /** Redaction configuration for this execution */
+  redaction?: RedactionConfig;
+}

package/src/scenario/index.ts

@@ -0,0 +1,7 @@
+/**
+ * Scenario module exports
+ */
+
+export * from './schema';
+export * from './parser';
+export * from './variables';

package/src/scenario/parser.test.ts

@@ -0,0 +1,99 @@
+/**
+ * Tests for scenario parser
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { parseScenarioString, validateScenario } from './parser';
+
+describe('parseScenarioString', () => {
+  test('parses valid YAML scenario', () => {
+    const yaml = `
+name: Test Scenario
+description: A test scenario
+version: "1.0"
+provider: openai
+model: gpt-4
+cases:
+  - id: test1
+    prompt: "Hello"
+    expected:
+      type: exact
+      value: "Hi"
+`;
+    const scenario = parseScenarioString(yaml);
+    expect(scenario.name).toBe('Test Scenario');
+    expect(scenario.cases.length).toBe(1);
+    expect(scenario.cases[0].id).toBe('test1');
+  });
+
+  test('throws on invalid YAML', () => {
+    const yaml = `
+name: Test
+cases: not an array
+`;
+    expect(() => parseScenarioString(yaml)).toThrow();
+  });
+
+  test('throws on missing required fields', () => {
+    const yaml = `
+description: Missing name and cases
+`;
+    expect(() => parseScenarioString(yaml)).toThrow();
+  });
+
+  test('parses different expected types', () => {
+    const yaml = `
+name: Multi-matcher Test
+cases:
+  - id: regex-test
+    prompt: "Test"
+    expected:
+      type: regex
+      pattern: "^\\\\w+$"
+  - id: fuzzy-test
+    prompt: "Test"
+    expected:
+      type: fuzzy
+      value: "test"
+      threshold: 0.8
+  - id: contains-test
+    prompt: "Test"
+    expected:
+      type: contains
+      values:
+        - foo
+        - bar
+      mode: all
+`;
+    const scenario = parseScenarioString(yaml);
+    expect(scenario.cases.length).toBe(3);
+    expect(scenario.cases[0].expected.type).toBe('regex');
+    expect(scenario.cases[1].expected.type).toBe('fuzzy');
+    expect(scenario.cases[2].expected.type).toBe('contains');
+  });
+});
+
+describe('validateScenario', () => {
+  test('validates correct scenario object', () => {
+    const scenario = {
+      name: 'Test',
+      cases: [
+        {
+          id: 'test1',
+          prompt: 'Hello',
+          expected: { type: 'exact', value: 'Hi' },
+        },
+      ],
+    };
+    const validated = validateScenario(scenario);
+    expect(validated.name).toBe('Test');
+  });
+
+  test('throws on invalid scenario', () => {
+    const scenario = {
+      name: 'Test',
+      cases: [],
+    };
+    expect(() => validateScenario(scenario)).toThrow();
+  });
+});

package/src/scenario/parser.ts

@@ -0,0 +1,108 @@
+/**
+ * Scenario parser for YAML files
+ */
+
+import { readFile } from 'node:fs/promises';
+import { parse as parseYaml } from 'yaml';
+import { ArtemisError } from '../utils/errors';
+import { type Scenario, ScenarioSchema } from './schema';
+
+/**
+ * Expand environment variables in config values
+ * Supports ${VAR} and ${VAR:-default} syntax
+ */
+function expandEnvVars(obj: unknown): unknown {
+  if (typeof obj === 'string') {
+    return obj.replace(/\$\{([^}]+)\}/g, (_, expr) => {
+      const [varName, defaultValue] = expr.split(':-');
+      return process.env[varName] || defaultValue || '';
+    });
+  }
+
+  if (Array.isArray(obj)) {
+    return obj.map(expandEnvVars);
+  }
+
+  if (obj && typeof obj === 'object') {
+    const result: Record<string, unknown> = {};
+    for (const [key, value] of Object.entries(obj)) {
+      result[key] = expandEnvVars(value);
+    }
+    return result;
+  }
+
+  return obj;
+}
+
+/**
+ * Parse a scenario from a YAML file
+ */
+export async function parseScenarioFile(filePath: string): Promise<Scenario> {
+  try {
+    const content = await readFile(filePath, 'utf-8');
+    return parseScenarioString(content, filePath);
+  } catch (error) {
+    if (error instanceof ArtemisError) {
+      throw error;
+    }
+    throw new ArtemisError(`Failed to read scenario file: ${filePath}`, 'SCENARIO_READ_ERROR', {
+      cause: error as Error,
+    });
+  }
+}
+
+/**
+ * Parse a scenario from a YAML string
+ */
+export function parseScenarioString(content: string, source?: string): Scenario {
+  try {
+    const raw = parseYaml(content);
+
+    // Expand environment variables before validation
+    const expanded = expandEnvVars(raw);
+
+    const result = ScenarioSchema.safeParse(expanded);
+
+    if (!result.success) {
+      const issues = result.error.issues
+        .map((i) => `  - ${i.path.join('.')}: ${i.message}`)
+        .join('\n');
+
+      throw new ArtemisError(
+        `Invalid scenario${source ? ` in ${source}` : ''}:\n${issues}`,
+        'SCENARIO_VALIDATION_ERROR',
+        { zodError: result.error }
+      );
+    }
+
+    return result.data;
+  } catch (error) {
+    if (error instanceof ArtemisError) {
+      throw error;
+    }
+    throw new ArtemisError(
+      `Failed to parse scenario YAML${source ? ` from ${source}` : ''}`,
+      'SCENARIO_PARSE_ERROR',
+      { cause: error as Error }
+    );
+  }
+}
+
+/**
+ * Validate a scenario object
+ */
+export function validateScenario(scenario: unknown): Scenario {
+  const result = ScenarioSchema.safeParse(scenario);
+
+  if (!result.success) {
+    const issues = result.error.issues
+      .map((i) => `  - ${i.path.join('.')}: ${i.message}`)
+      .join('\n');
+
+    throw new ArtemisError(`Invalid scenario:\n${issues}`, 'SCENARIO_VALIDATION_ERROR', {
+      zodError: result.error,
+    });
+  }
+
+  return result.data;
+}