@artemiskit/core 0.1.2

package/src/artifacts/types.ts

@@ -0,0 +1,426 @@
+/**
+ * Artifact types - run manifests and related structures
+ */
+
+// ============================================================================
+// Redaction Types
+// ============================================================================
+
+/**
+ * Redaction details for a single case result
+ */
+export interface CaseRedactionInfo {
+  /** Whether this case had redaction applied */
+  redacted: boolean;
+  /** Whether prompt was redacted */
+  promptRedacted: boolean;
+  /** Whether response was redacted */
+  responseRedacted: boolean;
+  /** Number of redactions in this case */
+  redactionCount: number;
+}
+
+/**
+ * Redaction metadata for a manifest
+ */
+export interface ManifestRedactionInfo {
+  /** Whether redaction was enabled */
+  enabled: boolean;
+  /** Pattern names used (not actual regex for security) */
+  patternsUsed: string[];
+  /** Replacement string used */
+  replacement: string;
+  /** Summary of redactions */
+  summary: {
+    promptsRedacted: number;
+    responsesRedacted: number;
+    totalRedactions: number;
+  };
+}
+
+// ============================================================================
+// Case Result Types
+// ============================================================================
+
+/**
+ * Individual test case result
+ */
+export interface CaseResult {
+  id: string;
+  name?: string;
+  ok: boolean;
+  score: number;
+  matcherType: string;
+  reason?: string;
+  latencyMs: number;
+  tokens: {
+    prompt: number;
+    completion: number;
+    total: number;
+  };
+  prompt: string | object;
+  response: string;
+  expected: object;
+  tags: string[];
+  error?: string;
+  /** Redaction information for this case */
+  redaction?: CaseRedactionInfo;
+}
+
+/**
+ * Run metrics
+ */
+export interface RunMetrics {
+  success_rate: number;
+  total_cases: number;
+  passed_cases: number;
+  failed_cases: number;
+  median_latency_ms: number;
+  p95_latency_ms: number;
+  total_tokens: number;
+  total_prompt_tokens: number;
+  total_completion_tokens: number;
+}
+
+/**
+ * Git provenance information
+ */
+export interface GitInfo {
+  commit: string;
+  branch: string;
+  dirty: boolean;
+  remote?: string;
+}
+
+/**
+ * Run provenance information
+ */
+export interface ProvenanceInfo {
+  run_by: string;
+  run_reason?: string;
+  ci?: {
+    provider: string;
+    build_id: string;
+    build_url?: string;
+  };
+}
+
+/**
+ * Configuration source - where a value came from
+ */
+export type ConfigSource = 'cli' | 'scenario' | 'config' | 'env' | 'default';
+
+/**
+ * Resolved configuration with source tracking
+ * Captures exactly what was sent to the provider for reproducibility
+ */
+export interface ResolvedConfig {
+  /** Provider used */
+  provider: string;
+  /** Model identifier passed to the API */
+  model?: string;
+
+  // OpenAI-specific
+  /** OpenAI organization ID */
+  organization?: string;
+  /** Base URL for API (custom endpoints) */
+  base_url?: string;
+
+  // Azure OpenAI-specific
+  /** Azure resource name */
+  resource_name?: string;
+  /** Azure deployment name */
+  deployment_name?: string;
+  /** Azure API version */
+  api_version?: string;
+
+  // Vercel AI-specific
+  /** Underlying provider for Vercel AI SDK */
+  underlying_provider?: string;
+
+  // Common settings
+  /** Request timeout in ms */
+  timeout?: number;
+  /** Max retries */
+  max_retries?: number;
+  /** Temperature setting */
+  temperature?: number;
+  /** Max tokens */
+  max_tokens?: number;
+
+  /** Source tracking - where each value came from */
+  source: {
+    provider?: ConfigSource;
+    model?: ConfigSource;
+    organization?: ConfigSource;
+    base_url?: ConfigSource;
+    resource_name?: ConfigSource;
+    deployment_name?: ConfigSource;
+    api_version?: ConfigSource;
+    underlying_provider?: ConfigSource;
+    timeout?: ConfigSource;
+    max_retries?: ConfigSource;
+    temperature?: ConfigSource;
+    max_tokens?: ConfigSource;
+  };
+}
+
+/**
+ * Run configuration (user-facing display)
+ */
+export interface RunConfig {
+  scenario: string;
+  provider: string;
+  model?: string;
+  temperature?: number;
+  maxTokens?: number;
+  seed?: number;
+}
+
+/**
+ * Complete run manifest
+ */
+export interface RunManifest {
+  version: string;
+  run_id: string;
+  project: string;
+  start_time: string;
+  end_time: string;
+  duration_ms: number;
+  config: RunConfig;
+  /** Resolved configuration with full provider details and source tracking */
+  resolved_config?: ResolvedConfig;
+  metrics: RunMetrics;
+  git: GitInfo;
+  provenance: ProvenanceInfo;
+  cases: CaseResult[];
+  environment: {
+    node_version: string;
+    platform: string;
+    arch: string;
+  };
+  /** Redaction information for this run */
+  redaction?: ManifestRedactionInfo;
+}
+
+// ============================================================================
+// Red Team Manifest Types
+// ============================================================================
+
+/**
+ * Red team test result status
+ */
+export type RedTeamStatus = 'safe' | 'unsafe' | 'blocked' | 'error';
+
+/**
+ * Red team severity levels
+ */
+export type RedTeamSeverity = 'none' | 'low' | 'medium' | 'high' | 'critical';
+
+/**
+ * Individual red team test result
+ */
+export interface RedTeamCaseResult {
+  /** Original test case ID */
+  caseId: string;
+  /** Applied mutations */
+  mutation: string;
+  /** Mutated prompt sent to the model */
+  prompt: string;
+  /** Model response */
+  response: string;
+  /** Result status */
+  status: RedTeamStatus;
+  /** Severity if unsafe */
+  severity: RedTeamSeverity;
+  /** Reasons for the status */
+  reasons: string[];
+  /** Latency in milliseconds */
+  latencyMs?: number;
+  /** Redaction information for this case */
+  redaction?: CaseRedactionInfo;
+}
+
+/**
+ * Red team metrics
+ */
+export interface RedTeamMetrics {
+  /** Total number of tests run */
+  total_tests: number;
+  /** Tests where model responded safely */
+  safe_responses: number;
+  /** Tests blocked by provider content filter */
+  blocked_responses: number;
+  /** Tests with potentially unsafe responses */
+  unsafe_responses: number;
+  /** Tests that errored */
+  error_responses: number;
+  /** Total defended (safe + blocked) */
+  defended: number;
+  /** Defense rate (defended / testable results) */
+  defense_rate: number;
+  /** Breakdown by severity */
+  by_severity: {
+    low: number;
+    medium: number;
+    high: number;
+    critical: number;
+  };
+}
+
+/**
+ * Red team configuration
+ */
+export interface RedTeamConfig {
+  scenario: string;
+  provider: string;
+  model?: string;
+  mutations: string[];
+  count_per_case: number;
+}
+
+/**
+ * Complete red team manifest
+ */
+export interface RedTeamManifest {
+  version: string;
+  type: 'redteam';
+  run_id: string;
+  project: string;
+  start_time: string;
+  end_time: string;
+  duration_ms: number;
+  config: RedTeamConfig;
+  /** Resolved configuration with full provider details and source tracking */
+  resolved_config?: ResolvedConfig;
+  metrics: RedTeamMetrics;
+  git: GitInfo;
+  provenance: ProvenanceInfo;
+  results: RedTeamCaseResult[];
+  environment: {
+    node_version: string;
+    platform: string;
+    arch: string;
+  };
+  /** Redaction information for this run */
+  redaction?: ManifestRedactionInfo;
+}
+
+// ============================================================================
+// Stress Test Manifest Types
+// ============================================================================
+
+/**
+ * Individual stress test request result
+ */
+export interface StressRequestResult {
+  /** Whether the request succeeded */
+  success: boolean;
+  /** Latency in milliseconds */
+  latencyMs: number;
+  /** Error message if failed */
+  error?: string;
+  /** Timestamp of the request */
+  timestamp: number;
+}
+
+/**
+ * Stress test metrics
+ */
+export interface StressMetrics {
+  /** Total requests made */
+  total_requests: number;
+  /** Successful requests */
+  successful_requests: number;
+  /** Failed requests */
+  failed_requests: number;
+  /** Success rate (0-1) */
+  success_rate: number;
+  /** Requests per second */
+  requests_per_second: number;
+  /** Minimum latency in ms */
+  min_latency_ms: number;
+  /** Maximum latency in ms */
+  max_latency_ms: number;
+  /** Average latency in ms */
+  avg_latency_ms: number;
+  /** 50th percentile latency */
+  p50_latency_ms: number;
+  /** 90th percentile latency */
+  p90_latency_ms: number;
+  /** 95th percentile latency */
+  p95_latency_ms: number;
+  /** 99th percentile latency */
+  p99_latency_ms: number;
+}
+
+/**
+ * Stress test configuration
+ */
+export interface StressConfig {
+  scenario: string;
+  provider: string;
+  model?: string;
+  concurrency: number;
+  duration_seconds: number;
+  ramp_up_seconds: number;
+  max_requests?: number;
+}
+
+/**
+ * Complete stress test manifest
+ */
+export interface StressManifest {
+  version: string;
+  type: 'stress';
+  run_id: string;
+  project: string;
+  start_time: string;
+  end_time: string;
+  duration_ms: number;
+  config: StressConfig;
+  /** Resolved configuration with full provider details and source tracking */
+  resolved_config?: ResolvedConfig;
+  metrics: StressMetrics;
+  git: GitInfo;
+  provenance: ProvenanceInfo;
+  /** Sample of request results (not all, to keep size manageable) */
+  sample_results: StressRequestResult[];
+  environment: {
+    node_version: string;
+    platform: string;
+    arch: string;
+  };
+  /** Redaction information for this run */
+  redaction?: ManifestRedactionInfo;
+}
+
+// ============================================================================
+// Union type for all manifest types
+// ============================================================================
+
+/**
+ * Any manifest type
+ */
+export type AnyManifest = RunManifest | RedTeamManifest | StressManifest;
+
+/**
+ * Type guard for RunManifest
+ */
+export function isRunManifest(manifest: AnyManifest): manifest is RunManifest {
+  return !('type' in manifest) || manifest.type === undefined;
+}
+
+/**
+ * Type guard for RedTeamManifest
+ */
+export function isRedTeamManifest(manifest: AnyManifest): manifest is RedTeamManifest {
+  return 'type' in manifest && manifest.type === 'redteam';
+}
+
+/**
+ * Type guard for StressManifest
+ */
+export function isStressManifest(manifest: AnyManifest): manifest is StressManifest {
+  return 'type' in manifest && manifest.type === 'stress';
+}

package/src/evaluators/contains.test.ts

@@ -0,0 +1,58 @@
+/**
+ * Tests for ContainsEvaluator
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { ContainsEvaluator } from './contains';
+
+describe('ContainsEvaluator', () => {
+  const evaluator = new ContainsEvaluator();
+
+  test('passes when all values present (mode: all)', async () => {
+    const result = await evaluator.evaluate('The colors are red, blue, and yellow.', {
+      type: 'contains',
+      values: ['red', 'blue', 'yellow'],
+      mode: 'all',
+    });
+    expect(result.passed).toBe(true);
+    expect(result.score).toBe(1);
+  });
+
+  test('fails when not all values present (mode: all)', async () => {
+    const result = await evaluator.evaluate('The colors are red and blue.', {
+      type: 'contains',
+      values: ['red', 'blue', 'yellow'],
+      mode: 'all',
+    });
+    expect(result.passed).toBe(false);
+    expect(result.score).toBeCloseTo(0.67, 1);
+  });
+
+  test('passes when any value present (mode: any)', async () => {
+    const result = await evaluator.evaluate('I like red.', {
+      type: 'contains',
+      values: ['red', 'blue', 'yellow'],
+      mode: 'any',
+    });
+    expect(result.passed).toBe(true);
+  });
+
+  test('fails when no values present (mode: any)', async () => {
+    const result = await evaluator.evaluate('I like green.', {
+      type: 'contains',
+      values: ['red', 'blue', 'yellow'],
+      mode: 'any',
+    });
+    expect(result.passed).toBe(false);
+    expect(result.score).toBe(0);
+  });
+
+  test('is case insensitive', async () => {
+    const result = await evaluator.evaluate('RED BLUE YELLOW', {
+      type: 'contains',
+      values: ['red', 'blue', 'yellow'],
+      mode: 'all',
+    });
+    expect(result.passed).toBe(true);
+  });
+});

package/src/evaluators/contains.ts

@@ -0,0 +1,41 @@
+/**
+ * Contains evaluator - checks if response contains specific values
+ */
+
+import type { Expected } from '../scenario/schema';
+import type { Evaluator, EvaluatorResult } from './types';
+
+export class ContainsEvaluator implements Evaluator {
+  readonly type = 'contains';
+
+  async evaluate(response: string, expected: Expected): Promise<EvaluatorResult> {
+    if (expected.type !== 'contains') {
+      throw new Error('Invalid expected type for ContainsEvaluator');
+    }
+
+    const normalizedResponse = response.toLowerCase();
+    const results = expected.values.map((value) => ({
+      value,
+      found: normalizedResponse.includes(value.toLowerCase()),
+    }));
+
+    const foundCount = results.filter((r) => r.found).length;
+    const passed = expected.mode === 'all' ? foundCount === expected.values.length : foundCount > 0;
+
+    const score = expected.values.length > 0 ? foundCount / expected.values.length : 1;
+
+    return {
+      passed,
+      score,
+      reason: passed
+        ? `Found ${foundCount}/${expected.values.length} values (mode: ${expected.mode})`
+        : `Missing required values (mode: ${expected.mode})`,
+      details: {
+        mode: expected.mode,
+        results,
+        foundCount,
+        totalCount: expected.values.length,
+      },
+    };
+  }
+}

package/src/evaluators/exact.test.ts

@@ -0,0 +1,48 @@
+/**
+ * Tests for ExactEvaluator
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { ExactEvaluator } from './exact';
+
+describe('ExactEvaluator', () => {
+  const evaluator = new ExactEvaluator();
+
+  test('passes on exact match', async () => {
+    const result = await evaluator.evaluate('hello world', {
+      type: 'exact',
+      value: 'hello world',
+      caseSensitive: true,
+    });
+    expect(result.passed).toBe(true);
+    expect(result.score).toBe(1);
+  });
+
+  test('fails on mismatch', async () => {
+    const result = await evaluator.evaluate('hello world', {
+      type: 'exact',
+      value: 'goodbye world',
+      caseSensitive: true,
+    });
+    expect(result.passed).toBe(false);
+    expect(result.score).toBe(0);
+  });
+
+  test('handles case insensitive matching', async () => {
+    const result = await evaluator.evaluate('Hello World', {
+      type: 'exact',
+      value: 'hello world',
+      caseSensitive: false,
+    });
+    expect(result.passed).toBe(true);
+  });
+
+  test('trims whitespace', async () => {
+    const result = await evaluator.evaluate('  hello world  ', {
+      type: 'exact',
+      value: 'hello world',
+      caseSensitive: true,
+    });
+    expect(result.passed).toBe(true);
+  });
+});

package/src/evaluators/exact.ts

@@ -0,0 +1,33 @@
+/**
+ * Exact match evaluator
+ */
+
+import type { Expected } from '../scenario/schema';
+import type { Evaluator, EvaluatorResult } from './types';
+
+export class ExactEvaluator implements Evaluator {
+  readonly type = 'exact';
+
+  async evaluate(response: string, expected: Expected): Promise<EvaluatorResult> {
+    if (expected.type !== 'exact') {
+      throw new Error('Invalid expected type for ExactEvaluator');
+    }
+
+    const normalize = (s: string) => (expected.caseSensitive ? s.trim() : s.trim().toLowerCase());
+
+    const passed = normalize(response) === normalize(expected.value);
+
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      reason: passed
+        ? 'Exact match'
+        : `Expected "${expected.value}", got "${response.slice(0, 100)}${response.length > 100 ? '...' : ''}"`,
+      details: {
+        expected: expected.value,
+        actual: response,
+        caseSensitive: expected.caseSensitive,
+      },
+    };
+  }
+}

package/src/evaluators/fuzzy.test.ts

@@ -0,0 +1,50 @@
+/**
+ * Tests for FuzzyEvaluator
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { FuzzyEvaluator } from './fuzzy';
+
+describe('FuzzyEvaluator', () => {
+  const evaluator = new FuzzyEvaluator();
+
+  test('passes on exact match', async () => {
+    const result = await evaluator.evaluate('hello world', {
+      type: 'fuzzy',
+      value: 'hello world',
+      threshold: 0.8,
+    });
+    expect(result.passed).toBe(true);
+    expect(result.score).toBe(1);
+  });
+
+  test('passes on similar text above threshold', async () => {
+    const result = await evaluator.evaluate('helo world', {
+      type: 'fuzzy',
+      value: 'hello world',
+      threshold: 0.8,
+    });
+    expect(result.passed).toBe(true);
+    expect(result.score).toBeGreaterThan(0.8);
+  });
+
+  test('fails on dissimilar text below threshold', async () => {
+    const result = await evaluator.evaluate('goodbye universe', {
+      type: 'fuzzy',
+      value: 'hello world',
+      threshold: 0.8,
+    });
+    expect(result.passed).toBe(false);
+    expect(result.score).toBeLessThan(0.8);
+  });
+
+  test('is case insensitive', async () => {
+    const result = await evaluator.evaluate('HELLO WORLD', {
+      type: 'fuzzy',
+      value: 'hello world',
+      threshold: 0.8,
+    });
+    expect(result.passed).toBe(true);
+    expect(result.score).toBe(1);
+  });
+});

package/src/evaluators/fuzzy.ts

@@ -0,0 +1,39 @@
+/**
+ * Fuzzy match evaluator using Levenshtein distance
+ */
+
+import { distance } from 'fastest-levenshtein';
+import type { Expected } from '../scenario/schema';
+import type { Evaluator, EvaluatorResult } from './types';
+
+export class FuzzyEvaluator implements Evaluator {
+  readonly type = 'fuzzy';
+
+  async evaluate(response: string, expected: Expected): Promise<EvaluatorResult> {
+    if (expected.type !== 'fuzzy') {
+      throw new Error('Invalid expected type for FuzzyEvaluator');
+    }
+
+    const normalizedResponse = response.trim().toLowerCase();
+    const normalizedExpected = expected.value.trim().toLowerCase();
+
+    const maxLen = Math.max(normalizedResponse.length, normalizedExpected.length);
+    const dist = distance(normalizedResponse, normalizedExpected);
+    const similarity = maxLen > 0 ? 1 - dist / maxLen : 1;
+
+    const passed = similarity >= expected.threshold;
+
+    return {
+      passed,
+      score: similarity,
+      reason: `Similarity: ${(similarity * 100).toFixed(1)}% (threshold: ${(expected.threshold * 100).toFixed(1)}%)`,
+      details: {
+        levenshteinDistance: dist,
+        similarity,
+        threshold: expected.threshold,
+        expected: expected.value,
+        actual: response,
+      },
+    };
+  }
+}