@artemiskit/core 0.1.2

package/src/scenario/schema.ts

@@ -0,0 +1,176 @@
+/**
+ * Scenario schema definitions using Zod
+ */
+
+import { z } from 'zod';
+import { RedactionConfigSchema as BaseRedactionConfigSchema } from '../redaction/types';
+
+/**
+ * Provider schema - supports all providers
+ */
+export const ProviderSchema = z.enum([
+  'openai',
+  'azure-openai',
+  'vercel-ai',
+  'anthropic',
+  'google',
+  'mistral',
+  'cohere',
+  'huggingface',
+  'ollama',
+  'custom',
+]);
+
+/**
+ * Provider config schema - optional overrides for provider settings
+ * Supports ${ENV_VAR} and ${ENV_VAR:-default} syntax for values
+ * All fields are optional - only specified fields override defaults
+ */
+export const ProviderConfigSchema = z
+  .object({
+    // Common fields
+    apiKey: z.string().optional(),
+    baseUrl: z.string().optional(),
+    defaultModel: z.string().optional(),
+    timeout: z.number().optional(),
+    maxRetries: z.number().optional(),
+
+    // OpenAI specific
+    organization: z.string().optional(),
+
+    // Azure OpenAI specific
+    resourceName: z.string().optional(),
+    deploymentName: z.string().optional(),
+    apiVersion: z.string().optional(),
+
+    // Vercel AI specific
+    underlyingProvider: z.enum(['openai', 'azure', 'anthropic', 'google', 'mistral']).optional(),
+  })
+  .optional();
+
+/**
+ * Expected result types - how to evaluate responses
+ */
+export const ExpectedSchema = z.discriminatedUnion('type', [
+  z.object({
+    type: z.literal('exact'),
+    value: z.string(),
+    caseSensitive: z.boolean().optional().default(true),
+  }),
+
+  z.object({
+    type: z.literal('regex'),
+    pattern: z.string(),
+    flags: z.string().optional(),
+  }),
+
+  z.object({
+    type: z.literal('fuzzy'),
+    value: z.string(),
+    threshold: z.number().min(0).max(1).default(0.8),
+  }),
+
+  z.object({
+    type: z.literal('llm_grader'),
+    rubric: z.string(),
+    model: z.string().optional(),
+    provider: ProviderSchema.optional(),
+    threshold: z.number().min(0).max(1).default(0.7),
+  }),
+
+  z.object({
+    type: z.literal('contains'),
+    values: z.array(z.string()),
+    mode: z.enum(['all', 'any']).default('all'),
+  }),
+
+  z.object({
+    type: z.literal('json_schema'),
+    schema: z.record(z.unknown()),
+  }),
+
+  z.object({
+    type: z.literal('custom'),
+    evaluator: z.string(),
+    config: z.record(z.unknown()).optional(),
+  }),
+]);
+
+/**
+ * Chat message schema
+ */
+export const ChatMessageSchema = z.object({
+  role: z.enum(['system', 'user', 'assistant']),
+  content: z.string(),
+});
+
+/**
+ * Variables schema - key-value pairs for template substitution
+ */
+export const VariablesSchema = z.record(z.string(), z.union([z.string(), z.number(), z.boolean()]));
+
+/**
+ * Redaction configuration schema for scenario-level settings
+ * Re-exported from redaction module, made optional for scenario context
+ */
+const RedactionConfigSchema = BaseRedactionConfigSchema.optional();
+
+/**
+ * Test case schema
+ */
+export const TestCaseSchema = z.object({
+  id: z.string(),
+  name: z.string().optional(),
+  description: z.string().optional(),
+  prompt: z.union([z.string(), z.array(ChatMessageSchema)]),
+  expected: ExpectedSchema,
+  tags: z.array(z.string()).optional().default([]),
+  metadata: z.record(z.unknown()).optional().default({}),
+  timeout: z.number().optional(),
+  retries: z.number().optional().default(0),
+  provider: ProviderSchema.optional(),
+  model: z.string().optional(),
+  variables: VariablesSchema.optional(),
+  /** Case-level redaction config (overrides scenario-level) */
+  redaction: RedactionConfigSchema,
+});
+
+/**
+ * Scenario schema - a collection of test cases
+ */
+export const ScenarioSchema = z.object({
+  name: z.string(),
+  description: z.string().optional(),
+  version: z.string().optional().default('1.0'),
+  provider: ProviderSchema.optional(),
+  model: z.string().optional(),
+  providerConfig: ProviderConfigSchema,
+  seed: z.number().optional(),
+  temperature: z.number().min(0).max(2).optional(),
+  maxTokens: z.number().optional(),
+  tags: z.array(z.string()).optional().default([]),
+  variables: VariablesSchema.optional(),
+  /** Scenario-level redaction configuration */
+  redaction: RedactionConfigSchema,
+  setup: z
+    .object({
+      systemPrompt: z.string().optional(),
+      functions: z.array(z.unknown()).optional(),
+    })
+    .optional(),
+  cases: z.array(TestCaseSchema).min(1),
+  teardown: z
+    .object({
+      cleanup: z.boolean().optional(),
+    })
+    .optional(),
+});
+
+export type Expected = z.infer<typeof ExpectedSchema>;
+export type TestCase = z.infer<typeof TestCaseSchema>;
+export type Scenario = z.infer<typeof ScenarioSchema>;
+export type Provider = z.infer<typeof ProviderSchema>;
+export type ProviderConfig = z.infer<typeof ProviderConfigSchema>;
+export type ChatMessageType = z.infer<typeof ChatMessageSchema>;
+export type Variables = z.infer<typeof VariablesSchema>;
+export type ScenarioRedactionConfig = z.infer<typeof RedactionConfigSchema>;

package/src/scenario/variables.test.ts

@@ -0,0 +1,150 @@
+/**
+ * Tests for variable substitution
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { mergeVariables, substituteString, substituteVariables } from './variables';
+
+describe('substituteString', () => {
+  test('substitutes single variable', () => {
+    const result = substituteString('Hello {{name}}!', { name: 'World' });
+    expect(result).toBe('Hello World!');
+  });
+
+  test('substitutes multiple variables', () => {
+    const result = substituteString('{{greeting}} {{name}}!', {
+      greeting: 'Hello',
+      name: 'World',
+    });
+    expect(result).toBe('Hello World!');
+  });
+
+  test('substitutes same variable multiple times', () => {
+    const result = substituteString('{{x}} + {{x}} = 2{{x}}', { x: '1' });
+    expect(result).toBe('1 + 1 = 21');
+  });
+
+  test('leaves unmatched variables as-is', () => {
+    const result = substituteString('Hello {{name}} and {{unknown}}!', { name: 'World' });
+    expect(result).toBe('Hello World and {{unknown}}!');
+  });
+
+  test('handles number values', () => {
+    const result = substituteString('Count: {{count}}', { count: 42 });
+    expect(result).toBe('Count: 42');
+  });
+
+  test('handles boolean values', () => {
+    const result = substituteString('Enabled: {{enabled}}', { enabled: true });
+    expect(result).toBe('Enabled: true');
+  });
+
+  test('handles empty variables object', () => {
+    const result = substituteString('Hello {{name}}!', {});
+    expect(result).toBe('Hello {{name}}!');
+  });
+
+  test('handles string without variables', () => {
+    const result = substituteString('Hello World!', { name: 'Test' });
+    expect(result).toBe('Hello World!');
+  });
+});
+
+describe('substituteVariables', () => {
+  test('substitutes in simple object', () => {
+    const result = substituteVariables({ message: 'Hello {{name}}!' }, { name: 'World' });
+    expect(result).toEqual({ message: 'Hello World!' });
+  });
+
+  test('substitutes in nested object', () => {
+    const result = substituteVariables(
+      {
+        outer: {
+          inner: 'Value is {{value}}',
+        },
+      },
+      { value: '42' }
+    );
+    expect(result).toEqual({
+      outer: {
+        inner: 'Value is 42',
+      },
+    });
+  });
+
+  test('substitutes in arrays', () => {
+    const result = substituteVariables(['Hello {{name}}', 'Goodbye {{name}}'], { name: 'World' });
+    expect(result).toEqual(['Hello World', 'Goodbye World']);
+  });
+
+  test('substitutes in array of objects', () => {
+    const result = substituteVariables(
+      [
+        { role: 'user', content: 'My name is {{name}}' },
+        { role: 'assistant', content: 'Hello {{name}}!' },
+      ],
+      { name: 'Alice' }
+    );
+    expect(result).toEqual([
+      { role: 'user', content: 'My name is Alice' },
+      { role: 'assistant', content: 'Hello Alice!' },
+    ]);
+  });
+
+  test('preserves non-string values', () => {
+    const result = substituteVariables(
+      {
+        name: '{{product}}',
+        count: 42,
+        enabled: true,
+        items: [1, 2, 3],
+      },
+      { product: 'Widget' }
+    );
+    expect(result).toEqual({
+      name: 'Widget',
+      count: 42,
+      enabled: true,
+      items: [1, 2, 3],
+    });
+  });
+
+  test('handles empty variables', () => {
+    const obj = { message: 'Hello {{name}}!' };
+    const result = substituteVariables(obj, {});
+    expect(result).toEqual({ message: 'Hello {{name}}!' });
+  });
+
+  test('returns primitive values unchanged', () => {
+    expect(substituteVariables(42, { x: '1' })).toBe(42);
+    expect(substituteVariables(true, { x: '1' })).toBe(true);
+    expect(substituteVariables(null, { x: '1' })).toBe(null);
+  });
+});
+
+describe('mergeVariables', () => {
+  test('merges scenario and case variables', () => {
+    const result = mergeVariables({ a: '1', b: '2' }, { c: '3' });
+    expect(result).toEqual({ a: '1', b: '2', c: '3' });
+  });
+
+  test('case variables override scenario variables', () => {
+    const result = mergeVariables({ name: 'Scenario', value: 'original' }, { name: 'Case' });
+    expect(result).toEqual({ name: 'Case', value: 'original' });
+  });
+
+  test('handles undefined scenario variables', () => {
+    const result = mergeVariables(undefined, { name: 'Case' });
+    expect(result).toEqual({ name: 'Case' });
+  });
+
+  test('handles undefined case variables', () => {
+    const result = mergeVariables({ name: 'Scenario' }, undefined);
+    expect(result).toEqual({ name: 'Scenario' });
+  });
+
+  test('handles both undefined', () => {
+    const result = mergeVariables(undefined, undefined);
+    expect(result).toEqual({});
+  });
+});

package/src/scenario/variables.ts

@@ -0,0 +1,60 @@
+/**
+ * Variable substitution for scenario templates
+ * Supports {{variable}} syntax in strings
+ */
+
+import type { Variables } from './schema';
+
+/**
+ * Substitute variables in a string using {{variable}} syntax
+ */
+export function substituteString(str: string, variables: Variables): string {
+  return str.replace(/\{\{(\w+)\}\}/g, (match, varName) => {
+    if (varName in variables) {
+      return String(variables[varName]);
+    }
+    // Leave unmatched variables as-is
+    return match;
+  });
+}
+
+/**
+ * Recursively substitute variables in an object
+ * Only substitutes in string values, preserving other types
+ */
+export function substituteVariables<T>(obj: T, variables: Variables): T {
+  if (!variables || Object.keys(variables).length === 0) {
+    return obj;
+  }
+
+  if (typeof obj === 'string') {
+    return substituteString(obj, variables) as T;
+  }
+
+  if (Array.isArray(obj)) {
+    return obj.map((item) => substituteVariables(item, variables)) as T;
+  }
+
+  if (obj && typeof obj === 'object') {
+    const result: Record<string, unknown> = {};
+    for (const [key, value] of Object.entries(obj)) {
+      result[key] = substituteVariables(value, variables);
+    }
+    return result as T;
+  }
+
+  return obj;
+}
+
+/**
+ * Merge variables with case-level overriding scenario-level
+ */
+export function mergeVariables(
+  scenarioVars: Variables | undefined,
+  caseVars: Variables | undefined
+): Variables {
+  return {
+    ...scenarioVars,
+    ...caseVars,
+  };
+}

package/src/storage/factory.ts

@@ -0,0 +1,52 @@
+/**
+ * Storage factory for creating storage adapters
+ */
+
+import { ArtemisError } from '../utils/errors';
+import { LocalStorageAdapter } from './local';
+import { SupabaseStorageAdapter } from './supabase';
+import type { StorageAdapter, StorageConfig } from './types';
+
+/**
+ * Create a storage adapter from configuration
+ */
+export function createStorageAdapter(config: StorageConfig): StorageAdapter {
+  switch (config.type) {
+    case 'supabase':
+      if (!config.url || !config.anonKey) {
+        throw new ArtemisError(
+          'Supabase storage requires url and anonKey configuration',
+          'CONFIG_ERROR'
+        );
+      }
+      return new SupabaseStorageAdapter({
+        url: config.url,
+        anonKey: config.anonKey,
+        bucket: config.bucket,
+      });
+
+    case 'local':
+      return new LocalStorageAdapter(config.basePath);
+
+    default:
+      throw new ArtemisError(`Unknown storage type: ${config.type}`, 'CONFIG_ERROR');
+  }
+}
+
+/**
+ * Create storage adapter from environment variables
+ */
+export function createStorageFromEnv(): StorageAdapter {
+  const supabaseUrl = process.env.SUPABASE_URL;
+  const supabaseKey = process.env.SUPABASE_ANON_KEY;
+
+  if (supabaseUrl && supabaseKey) {
+    return new SupabaseStorageAdapter({
+      url: supabaseUrl,
+      anonKey: supabaseKey,
+      bucket: process.env.SUPABASE_BUCKET,
+    });
+  }
+
+  return new LocalStorageAdapter(process.env.ARTEMIS_STORAGE_PATH || './artemis-runs');
+}

package/src/storage/index.ts

@@ -0,0 +1,8 @@
+/**
+ * Storage module exports
+ */
+
+export * from './types';
+export { createStorageAdapter, createStorageFromEnv } from './factory';
+export { SupabaseStorageAdapter, type SupabaseStorageConfig } from './supabase';
+export { LocalStorageAdapter } from './local';

package/src/storage/local.test.ts

@@ -0,0 +1,165 @@
+/**
+ * Tests for LocalStorageAdapter
+ */
+
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import { mkdir, rm } from 'node:fs/promises';
+import type { RunManifest } from '../artifacts/types';
+import { LocalStorageAdapter } from './local';
+
+const TEST_DIR = './test-artemis-runs';
+
+describe('LocalStorageAdapter', () => {
+  let storage: LocalStorageAdapter;
+
+  const mockManifest: RunManifest = {
+    version: '1.0',
+    run_id: 'test-run-123',
+    project: 'test-project',
+    start_time: '2024-01-01T00:00:00.000Z',
+    end_time: '2024-01-01T00:01:00.000Z',
+    duration_ms: 60000,
+    config: {
+      scenario: 'test-scenario',
+      provider: 'openai',
+      model: 'gpt-4',
+    },
+    metrics: {
+      success_rate: 0.8,
+      total_cases: 10,
+      passed_cases: 8,
+      failed_cases: 2,
+      median_latency_ms: 150,
+      p95_latency_ms: 300,
+      total_tokens: 1000,
+      total_prompt_tokens: 600,
+      total_completion_tokens: 400,
+    },
+    cases: [],
+    environment: {
+      node_version: '20.0.0',
+      os: 'darwin',
+      arch: 'arm64',
+    },
+    provenance: {
+      run_by: 'test-user',
+    },
+  };
+
+  beforeAll(async () => {
+    storage = new LocalStorageAdapter(TEST_DIR);
+    await mkdir(TEST_DIR, { recursive: true });
+  });
+
+  afterAll(async () => {
+    await rm(TEST_DIR, { recursive: true, force: true });
+  });
+
+  test('saves manifest to filesystem', async () => {
+    const filePath = await storage.save(mockManifest);
+    expect(filePath).toContain('test-run-123.json');
+  });
+
+  test('loads manifest from filesystem', async () => {
+    // Save first
+    await storage.save(mockManifest);
+
+    // Load back
+    const loaded = await storage.load('test-run-123');
+    expect(loaded.run_id).toBe('test-run-123');
+    expect(loaded.project).toBe('test-project');
+    expect(loaded.config.scenario).toBe('test-scenario');
+  });
+
+  test('loadRun returns RunManifest', async () => {
+    await storage.save(mockManifest);
+    const loaded = await storage.loadRun('test-run-123');
+    expect(loaded.metrics.success_rate).toBe(0.8);
+  });
+
+  test('throws error for non-existent run', async () => {
+    await expect(storage.load('non-existent-run')).rejects.toThrow('Run not found');
+  });
+
+  test('lists runs', async () => {
+    // Save a manifest
+    await storage.save(mockManifest);
+
+    // List runs
+    const runs = await storage.list();
+    expect(runs.length).toBeGreaterThanOrEqual(1);
+
+    const testRun = runs.find((r) => r.runId === 'test-run-123');
+    expect(testRun).toBeDefined();
+    expect(testRun?.scenario).toBe('test-scenario');
+    expect(testRun?.successRate).toBe(0.8);
+  });
+
+  test('lists runs with project filter', async () => {
+    await storage.save(mockManifest);
+
+    const runs = await storage.list({ project: 'test-project' });
+    expect(runs.every((r) => r.runId === 'test-run-123' || true)).toBe(true);
+  });
+
+  test('lists runs with scenario filter', async () => {
+    await storage.save(mockManifest);
+
+    const runs = await storage.list({ scenario: 'test-scenario' });
+    expect(runs.length).toBeGreaterThanOrEqual(1);
+  });
+
+  test('lists runs with limit', async () => {
+    await storage.save(mockManifest);
+    await storage.save({ ...mockManifest, run_id: 'test-run-456' });
+
+    const runs = await storage.list({ limit: 1 });
+    expect(runs.length).toBe(1);
+  });
+
+  test('deletes run', async () => {
+    const manifest = { ...mockManifest, run_id: 'to-delete-123' };
+    await storage.save(manifest);
+
+    // Verify it exists
+    const loaded = await storage.load('to-delete-123');
+    expect(loaded.run_id).toBe('to-delete-123');
+
+    // Delete
+    await storage.delete('to-delete-123');
+
+    // Verify it's gone
+    await expect(storage.load('to-delete-123')).rejects.toThrow('Run not found');
+  });
+
+  test('compares two runs', async () => {
+    const baseline = { ...mockManifest, run_id: 'baseline-run' };
+    const current = {
+      ...mockManifest,
+      run_id: 'current-run',
+      metrics: {
+        ...mockManifest.metrics,
+        success_rate: 0.9,
+        median_latency_ms: 120,
+        total_tokens: 1100,
+      },
+    };
+
+    await storage.save(baseline);
+    await storage.save(current);
+
+    const comparison = await storage.compare('baseline-run', 'current-run');
+
+    expect(comparison.baseline.run_id).toBe('baseline-run');
+    expect(comparison.current.run_id).toBe('current-run');
+    expect(comparison.delta.successRate).toBeCloseTo(0.1, 2);
+    expect(comparison.delta.latency).toBe(-30);
+    expect(comparison.delta.tokens).toBe(100);
+  });
+
+  test('handles empty storage gracefully', async () => {
+    const emptyStorage = new LocalStorageAdapter('./empty-test-dir');
+    const runs = await emptyStorage.list();
+    expect(runs).toEqual([]);
+  });
+});