@artemiskit/cli 0.1.5 → 0.1.7

package/src/__tests__/helpers/mock-adapter.ts

@@ -0,0 +1,108 @@
+/**
+ * Mock adapter for CLI integration tests
+ *
+ * This module provides mock types and adapters for testing CLI commands
+ * without making actual LLM API calls.
+ */
+
+/** Message format for chat interactions */
+export interface MockMessage {
+  role: 'system' | 'user' | 'assistant';
+  content: string;
+}
+
+/** Response from the mock adapter */
+export interface MockLLMResponse {
+  content: string;
+  usage: { input: number; output: number };
+}
+
+/** Mock adapter interface */
+export interface MockLLMAdapter {
+  chat: (messages: MockMessage[]) => Promise<MockLLMResponse>;
+}
+
+export interface MockResponse {
+  content: string;
+  latencyMs?: number;
+  tokens?: { input: number; output: number };
+}
+
+export interface MockAdapterOptions {
+  responses?: Map<string, MockResponse>;
+  defaultResponse?: MockResponse;
+  shouldFail?: boolean;
+  failureMessage?: string;
+}
+
+/**
+ * Creates a mock LLM adapter for testing
+ */
+export function createMockAdapter(options: MockAdapterOptions = {}): MockLLMAdapter {
+  const {
+    responses = new Map(),
+    defaultResponse = {
+      content: 'Hello! How can I help you today?',
+      latencyMs: 100,
+      tokens: { input: 10, output: 15 },
+    },
+    shouldFail = false,
+    failureMessage = 'Mock adapter failure',
+  } = options;
+
+  return {
+    chat: async (messages: MockMessage[]): Promise<MockLLMResponse> => {
+      if (shouldFail) {
+        throw new Error(failureMessage);
+      }
+
+      // Get the last user message to determine response
+      const lastUserMessage = [...messages].reverse().find((m) => m.role === 'user');
+      const prompt = lastUserMessage?.content || '';
+
+      // Check for specific response mapping
+      const mockResponse = responses.get(prompt) || defaultResponse;
+
+      // Simulate latency
+      if (mockResponse.latencyMs) {
+        await new Promise((resolve) => setTimeout(resolve, Math.min(mockResponse.latencyMs, 50)));
+      }
+
+      return {
+        content: mockResponse.content,
+        usage: mockResponse.tokens || { input: 10, output: 15 },
+      };
+    },
+  };
+}
+
+/**
+ * Preset responses for common test scenarios
+ */
+export const mockResponses = {
+  greeting: {
+    content: 'Hello! How can I help you today?',
+    latencyMs: 50,
+    tokens: { input: 5, output: 8 },
+  },
+  capitals: {
+    content: 'The capital of France is Paris.',
+    latencyMs: 75,
+    tokens: { input: 10, output: 8 },
+  },
+  math: {
+    content: 'The answer is 4.',
+    latencyMs: 30,
+    tokens: { input: 8, output: 5 },
+  },
+  json: {
+    content: '{"name": "John", "age": 30}',
+    latencyMs: 60,
+    tokens: { input: 15, output: 12 },
+  },
+  code: {
+    content: 'function add(a, b) { return a + b; }',
+    latencyMs: 80,
+    tokens: { input: 20, output: 15 },
+  },
+};

package/src/__tests__/helpers/test-utils.ts

@@ -0,0 +1,205 @@
+/**
+ * Test utilities for CLI integration tests
+ */
+
+import { mkdir, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+/**
+ * Creates a temporary directory for test isolation
+ */
+export async function createTestDir(prefix = 'artemis-test'): Promise<string> {
+  const testDir = join(tmpdir(), `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+  await mkdir(testDir, { recursive: true });
+  return testDir;
+}
+
+/**
+ * Cleans up a test directory
+ */
+export async function cleanupTestDir(testDir: string): Promise<void> {
+  try {
+    await rm(testDir, { recursive: true, force: true });
+  } catch {
+    // Ignore cleanup errors
+  }
+}
+
+/**
+ * Creates a test scenario file
+ */
+export async function createScenarioFile(
+  dir: string,
+  name: string,
+  content: string
+): Promise<string> {
+  const scenariosDir = join(dir, 'scenarios');
+  await mkdir(scenariosDir, { recursive: true });
+  const filePath = join(scenariosDir, `${name}.yaml`);
+  await writeFile(filePath, content);
+  return filePath;
+}
+
+/**
+ * Creates a test config file
+ */
+export async function createConfigFile(
+  dir: string,
+  config: Record<string, unknown>
+): Promise<string> {
+  const filePath = join(dir, 'artemis.config.yaml');
+  const yaml = Object.entries(config)
+    .map(([key, value]) => {
+      if (typeof value === 'object' && value !== null) {
+        const nested = Object.entries(value as Record<string, unknown>)
+          .map(([k, v]) => `  ${k}: ${JSON.stringify(v)}`)
+          .join('\n');
+        return `${key}:\n${nested}`;
+      }
+      return `${key}: ${JSON.stringify(value)}`;
+    })
+    .join('\n');
+  await writeFile(filePath, yaml);
+  return filePath;
+}
+
+/**
+ * Sample scenario templates for testing
+ */
+export const scenarioTemplates = {
+  simple: `
+name: simple-test
+description: A simple test scenario
+
+cases:
+  - id: test-1
+    prompt: "Say hello"
+    expected:
+      type: contains
+      values:
+        - "hello"
+      mode: any
+`,
+
+  multiCase: `
+name: multi-case-test
+description: Multiple test cases
+
+cases:
+  - id: case-1
+    prompt: "What is 2+2?"
+    expected:
+      type: contains
+      values:
+        - "4"
+      mode: any
+
+  - id: case-2
+    prompt: "What is the capital of France?"
+    expected:
+      type: contains
+      values:
+        - "Paris"
+      mode: any
+
+  - id: case-3
+    prompt: "Say hello"
+    expected:
+      type: contains
+      values:
+        - "hello"
+      mode: any
+`,
+
+  withProvider: `
+name: provider-test
+description: Test with provider config
+provider: openai
+model: gpt-4o-mini
+
+cases:
+  - id: test-1
+    prompt: "Hello"
+    expected:
+      type: contains
+      values:
+        - "hello"
+      mode: any
+`,
+
+  exactMatch: `
+name: exact-match-test
+description: Test exact matching
+
+cases:
+  - id: exact-1
+    prompt: "Return exactly: hello world"
+    expected:
+      type: exact
+      value: "hello world"
+`,
+
+  regexMatch: `
+name: regex-test
+description: Test regex matching
+
+cases:
+  - id: regex-1
+    prompt: "Return a number"
+    expected:
+      type: regex
+      pattern: "\\\\d+"
+`,
+
+  failing: `
+name: failing-test
+description: A test that should fail
+
+cases:
+  - id: will-fail
+    prompt: "Say hello"
+    expected:
+      type: exact
+      value: "this will not match"
+`,
+};
+
+/**
+ * Captures console output during test execution
+ */
+export class OutputCapture {
+  private originalLog: typeof console.log = console.log;
+  private originalError: typeof console.error = console.error;
+  private logs: string[] = [];
+  private errors: string[] = [];
+
+  start(): void {
+    this.originalLog = console.log;
+    this.originalError = console.error;
+    this.logs = [];
+    this.errors = [];
+
+    console.log = (...args: unknown[]) => {
+      this.logs.push(args.map(String).join(' '));
+    };
+
+    console.error = (...args: unknown[]) => {
+      this.errors.push(args.map(String).join(' '));
+    };
+  }
+
+  stop(): { logs: string[]; errors: string[] } {
+    console.log = this.originalLog;
+    console.error = this.originalError;
+    return { logs: this.logs, errors: this.errors };
+  }
+
+  getOutput(): string {
+    return this.logs.join('\n');
+  }
+
+  getErrors(): string {
+    return this.errors.join('\n');
+  }
+}

package/src/__tests__/integration/compare-command.test.ts

@@ -0,0 +1,236 @@
+/**
+ * Integration tests for compare command
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { createStorage } from '../../utils/storage.js';
+import { cleanupTestDir, createTestDir } from '../helpers/test-utils.js';
+
+describe('Compare Command', () => {
+  let testDir: string;
+  let originalCwd: string;
+
+  beforeEach(async () => {
+    testDir = await createTestDir('compare-test');
+    originalCwd = process.cwd();
+    process.chdir(testDir);
+
+    // Create storage directory
+    await mkdir(join(testDir, 'artemis-runs', 'test-project'), { recursive: true });
+  });
+
+  afterEach(async () => {
+    process.chdir(originalCwd);
+    await cleanupTestDir(testDir);
+  });
+
+  describe('storage comparison', () => {
+    it('should compare two runs and calculate deltas', async () => {
+      const baselineManifest = {
+        run_id: 'baseline-001',
+        project: 'test-project',
+        config: { scenario: 'test-scenario' },
+        start_time: new Date('2026-01-15T10:00:00Z').toISOString(),
+        metrics: {
+          success_rate: 0.8,
+          passed_cases: 4,
+          failed_cases: 1,
+          total_tokens: 500,
+          median_latency_ms: 200,
+        },
+        cases: [],
+      };
+
+      const currentManifest = {
+        run_id: 'current-001',
+        project: 'test-project',
+        config: { scenario: 'test-scenario' },
+        start_time: new Date('2026-01-16T10:00:00Z').toISOString(),
+        metrics: {
+          success_rate: 1.0,
+          passed_cases: 5,
+          failed_cases: 0,
+          total_tokens: 450,
+          median_latency_ms: 150,
+        },
+        cases: [],
+      };
+
+      await writeFile(
+        join(testDir, 'artemis-runs', 'test-project', 'baseline-001.json'),
+        JSON.stringify(baselineManifest)
+      );
+      await writeFile(
+        join(testDir, 'artemis-runs', 'test-project', 'current-001.json'),
+        JSON.stringify(currentManifest)
+      );
+
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      const comparison = await storage.compare?.('baseline-001', 'current-001');
+
+      expect(comparison.baseline.metrics.success_rate).toBe(0.8);
+      expect(comparison.current.metrics.success_rate).toBe(1.0);
+      expect(comparison.delta.successRate).toBeCloseTo(0.2, 5); // 1.0 - 0.8
+      expect(comparison.delta.latency).toBe(-50); // 150 - 200 (improved)
+      expect(comparison.delta.tokens).toBe(-50); // 450 - 500 (reduced)
+    });
+
+    it('should detect regression when success rate drops', async () => {
+      const baselineManifest = {
+        run_id: 'baseline-002',
+        project: 'test-project',
+        config: { scenario: 'test-scenario' },
+        start_time: new Date('2026-01-15T10:00:00Z').toISOString(),
+        metrics: {
+          success_rate: 1.0,
+          passed_cases: 5,
+          failed_cases: 0,
+          total_tokens: 500,
+          median_latency_ms: 200,
+        },
+        cases: [],
+      };
+
+      const currentManifest = {
+        run_id: 'current-002',
+        project: 'test-project',
+        config: { scenario: 'test-scenario' },
+        start_time: new Date('2026-01-16T10:00:00Z').toISOString(),
+        metrics: {
+          success_rate: 0.6,
+          passed_cases: 3,
+          failed_cases: 2,
+          total_tokens: 600,
+          median_latency_ms: 300,
+        },
+        cases: [],
+      };
+
+      await writeFile(
+        join(testDir, 'artemis-runs', 'test-project', 'baseline-002.json'),
+        JSON.stringify(baselineManifest)
+      );
+      await writeFile(
+        join(testDir, 'artemis-runs', 'test-project', 'current-002.json'),
+        JSON.stringify(currentManifest)
+      );
+
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      const comparison = await storage.compare?.('baseline-002', 'current-002');
+
+      // Success rate dropped by 0.4 (40%)
+      expect(comparison.delta.successRate).toBeCloseTo(-0.4, 5);
+
+      // This would be a regression (threshold typically 5%)
+      const threshold = 0.05;
+      const hasRegression = comparison.delta.successRate < -threshold;
+      expect(hasRegression).toBe(true);
+    });
+
+    it('should handle identical runs', async () => {
+      const manifest = {
+        run_id: 'same-001',
+        project: 'test-project',
+        config: { scenario: 'test-scenario' },
+        start_time: new Date().toISOString(),
+        metrics: {
+          success_rate: 0.9,
+          passed_cases: 9,
+          failed_cases: 1,
+          total_tokens: 1000,
+          median_latency_ms: 250,
+        },
+        cases: [],
+      };
+
+      await writeFile(
+        join(testDir, 'artemis-runs', 'test-project', 'same-001.json'),
+        JSON.stringify(manifest)
+      );
+
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      const comparison = await storage.compare?.('same-001', 'same-001');
+
+      expect(comparison.delta.successRate).toBe(0);
+      expect(comparison.delta.latency).toBe(0);
+      expect(comparison.delta.tokens).toBe(0);
+    });
+
+    it('should throw error for non-existent baseline', async () => {
+      const currentManifest = {
+        run_id: 'exists-001',
+        project: 'test-project',
+        config: { scenario: 'test-scenario' },
+        start_time: new Date().toISOString(),
+        metrics: {
+          success_rate: 1.0,
+          passed_cases: 5,
+          failed_cases: 0,
+          total_tokens: 500,
+          median_latency_ms: 200,
+        },
+        cases: [],
+      };
+
+      await writeFile(
+        join(testDir, 'artemis-runs', 'test-project', 'exists-001.json'),
+        JSON.stringify(currentManifest)
+      );
+
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      await expect(storage.compare?.('non-existent', 'exists-001')).rejects.toThrow();
+    });
+
+    it('should throw error for non-existent current', async () => {
+      const baselineManifest = {
+        run_id: 'exists-002',
+        project: 'test-project',
+        config: { scenario: 'test-scenario' },
+        start_time: new Date().toISOString(),
+        metrics: {
+          success_rate: 1.0,
+          passed_cases: 5,
+          failed_cases: 0,
+          total_tokens: 500,
+          median_latency_ms: 200,
+        },
+        cases: [],
+      };
+
+      await writeFile(
+        join(testDir, 'artemis-runs', 'test-project', 'exists-002.json'),
+        JSON.stringify(baselineManifest)
+      );
+
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      await expect(storage.compare?.('exists-002', 'non-existent')).rejects.toThrow();
+    });
+  });
+});

package/src/__tests__/integration/config.test.ts

@@ -0,0 +1,125 @@
+/**
+ * Integration tests for CLI configuration loading
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { loadConfig } from '../../config/loader.js';
+import { cleanupTestDir, createTestDir } from '../helpers/test-utils.js';
+
+describe('Config Loader', () => {
+  let testDir: string;
+  let originalCwd: string;
+
+  beforeEach(async () => {
+    testDir = await createTestDir('config-test');
+    originalCwd = process.cwd();
+    process.chdir(testDir);
+  });
+
+  afterEach(async () => {
+    process.chdir(originalCwd);
+    await cleanupTestDir(testDir);
+  });
+
+  describe('loadConfig', () => {
+    it('should return null when no config file exists', async () => {
+      const config = await loadConfig();
+      expect(config).toBeNull();
+    });
+
+    it('should load artemis.config.yaml from current directory', async () => {
+      const configContent = `
+provider: openai
+model: gpt-4o-mini
+project: test-project
+`;
+      await writeFile(join(testDir, 'artemis.config.yaml'), configContent);
+
+      const config = await loadConfig();
+      expect(config).toBeDefined();
+      expect(config?.provider).toBe('openai');
+      expect(config?.model).toBe('gpt-4o-mini');
+      expect(config?.project).toBe('test-project');
+    });
+
+    it('should load artemis.config.yml (yml extension)', async () => {
+      const configContent = `
+provider: azure-openai
+model: gpt-4
+`;
+      await writeFile(join(testDir, 'artemis.config.yml'), configContent);
+
+      const config = await loadConfig();
+      expect(config).toBeDefined();
+      expect(config?.provider).toBe('azure-openai');
+    });
+
+    it('should load config from explicit path', async () => {
+      const customPath = join(testDir, 'custom-config.yaml');
+      const configContent = `
+provider: anthropic
+model: claude-3-sonnet
+`;
+      await writeFile(customPath, configContent);
+
+      const config = await loadConfig(customPath);
+      expect(config).toBeDefined();
+      expect(config?.provider).toBe('anthropic');
+      expect(config?.model).toBe('claude-3-sonnet');
+    });
+
+    it('should load nested azure configuration', async () => {
+      const configContent = `
+provider: azure-openai
+model: gpt-4
+providers:
+  azure-openai:
+    resourceName: my-resource
+    deploymentName: my-deployment
+    apiVersion: "2024-02-15-preview"
+`;
+      await writeFile(join(testDir, 'artemis.config.yaml'), configContent);
+
+      const config = await loadConfig();
+      expect(config).toBeDefined();
+      expect(config?.providers?.['azure-openai']?.resourceName).toBe('my-resource');
+      expect(config?.providers?.['azure-openai']?.deploymentName).toBe('my-deployment');
+    });
+
+    it('should load storage configuration', async () => {
+      const configContent = `
+provider: openai
+storage:
+  type: local
+  basePath: ./my-runs
+`;
+      await writeFile(join(testDir, 'artemis.config.yaml'), configContent);
+
+      const config = await loadConfig();
+      expect(config).toBeDefined();
+      expect(config?.storage?.type).toBe('local');
+      expect(config?.storage?.basePath).toBe('./my-runs');
+    });
+
+    it('should throw error for invalid YAML', async () => {
+      const invalidYaml = `
+provider: openai
+  model: gpt-4  # invalid indentation
+invalid: [unclosed
+`;
+      await writeFile(join(testDir, 'artemis.config.yaml'), invalidYaml);
+
+      await expect(loadConfig()).rejects.toThrow();
+    });
+
+    it('should prefer .yaml over .yml when both exist', async () => {
+      await writeFile(join(testDir, 'artemis.config.yaml'), 'provider: openai');
+      await writeFile(join(testDir, 'artemis.config.yml'), 'provider: anthropic');
+
+      const config = await loadConfig();
+      expect(config?.provider).toBe('openai');
+    });
+  });
+});