@artemiskit/cli 0.1.4 → 0.1.6

package/src/__tests__/integration/history-command.test.ts

@@ -0,0 +1,251 @@
+/**
+ * Integration tests for history command
+ */
+
+import { describe, expect, it, beforeEach, afterEach } from 'bun:test';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { createTestDir, cleanupTestDir } from '../helpers/test-utils.js';
+import { createStorage } from '../../utils/storage.js';
+
+describe('History Command', () => {
+  let testDir: string;
+  let originalCwd: string;
+
+  beforeEach(async () => {
+    testDir = await createTestDir('history-test');
+    originalCwd = process.cwd();
+    process.chdir(testDir);
+
+    // Create storage directory
+    await mkdir(join(testDir, 'artemis-runs', 'test-project'), { recursive: true });
+  });
+
+  afterEach(async () => {
+    process.chdir(originalCwd);
+    await cleanupTestDir(testDir);
+  });
+
+  describe('storage listing', () => {
+    it('should list runs from local storage', async () => {
+      // Create mock run manifests with correct structure
+      const manifest1 = {
+        run_id: 'run-001',
+        project: 'test-project',
+        config: { scenario: 'test-scenario' },
+        start_time: new Date('2026-01-15T10:00:00Z').toISOString(),
+        metrics: {
+          success_rate: 1.0,
+          passed_cases: 5,
+          failed_cases: 0,
+          total_tokens: 500,
+          median_latency_ms: 100,
+        },
+        cases: [],
+      };
+
+      const manifest2 = {
+        run_id: 'run-002',
+        project: 'test-project',
+        config: { scenario: 'another-scenario' },
+        start_time: new Date('2026-01-16T10:00:00Z').toISOString(),
+        metrics: {
+          success_rate: 0.8,
+          passed_cases: 4,
+          failed_cases: 1,
+          total_tokens: 600,
+          median_latency_ms: 150,
+        },
+        cases: [],
+      };
+
+      // Write manifest files
+      await writeFile(
+        join(testDir, 'artemis-runs', 'test-project', 'run-001.json'),
+        JSON.stringify(manifest1)
+      );
+      await writeFile(
+        join(testDir, 'artemis-runs', 'test-project', 'run-002.json'),
+        JSON.stringify(manifest2)
+      );
+
+      // Create storage and list using basePath
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      const runs = await storage.list({ limit: 10 });
+
+      expect(runs.length).toBe(2);
+      expect(runs.some((r) => r.runId === 'run-001')).toBe(true);
+      expect(runs.some((r) => r.runId === 'run-002')).toBe(true);
+    });
+
+    it('should filter by scenario', async () => {
+      const manifest1 = {
+        run_id: 'run-001',
+        project: 'test-project',
+        config: { scenario: 'scenario-a' },
+        start_time: new Date().toISOString(),
+        metrics: {
+          success_rate: 1.0,
+          passed_cases: 5,
+          failed_cases: 0,
+          total_tokens: 500,
+          median_latency_ms: 100,
+        },
+        cases: [],
+      };
+
+      const manifest2 = {
+        run_id: 'run-002',
+        project: 'test-project',
+        config: { scenario: 'scenario-b' },
+        start_time: new Date().toISOString(),
+        metrics: {
+          success_rate: 0.8,
+          passed_cases: 4,
+          failed_cases: 1,
+          total_tokens: 600,
+          median_latency_ms: 150,
+        },
+        cases: [],
+      };
+
+      await writeFile(
+        join(testDir, 'artemis-runs', 'test-project', 'run-001.json'),
+        JSON.stringify(manifest1)
+      );
+      await writeFile(
+        join(testDir, 'artemis-runs', 'test-project', 'run-002.json'),
+        JSON.stringify(manifest2)
+      );
+
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      const runs = await storage.list({ scenario: 'scenario-a', limit: 10 });
+
+      expect(runs.length).toBe(1);
+      expect(runs[0].scenario).toBe('scenario-a');
+    });
+
+    it('should respect limit parameter', async () => {
+      // Create 5 manifests
+      for (let i = 1; i <= 5; i++) {
+        const manifest = {
+          run_id: `run-00${i}`,
+          project: 'test-project',
+          config: { scenario: 'test-scenario' },
+          start_time: new Date(Date.now() - i * 1000).toISOString(),
+          metrics: {
+            success_rate: 1.0,
+            passed_cases: 5,
+            failed_cases: 0,
+            total_tokens: 500,
+            median_latency_ms: 100,
+          },
+          cases: [],
+        };
+        await writeFile(
+          join(testDir, 'artemis-runs', 'test-project', `run-00${i}.json`),
+          JSON.stringify(manifest)
+        );
+      }
+
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      const runs = await storage.list({ limit: 3 });
+
+      expect(runs.length).toBe(3);
+    });
+
+    it('should return empty array when no runs exist', async () => {
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      const runs = await storage.list({ limit: 10 });
+
+      expect(runs).toEqual([]);
+    });
+  });
+
+  describe('run data', () => {
+    it('should include success rate in listing', async () => {
+      const manifest = {
+        run_id: 'run-001',
+        project: 'test-project',
+        config: { scenario: 'test-scenario' },
+        start_time: new Date().toISOString(),
+        metrics: {
+          success_rate: 0.75,
+          passed_cases: 3,
+          failed_cases: 1,
+          total_tokens: 400,
+          median_latency_ms: 120,
+        },
+        cases: [],
+      };
+
+      await writeFile(
+        join(testDir, 'artemis-runs', 'test-project', 'run-001.json'),
+        JSON.stringify(manifest)
+      );
+
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      const runs = await storage.list({ limit: 10 });
+
+      expect(runs[0].successRate).toBe(0.75);
+    });
+
+    it('should include creation date in listing', async () => {
+      const startTime = '2026-01-17T12:00:00.000Z';
+      const manifest = {
+        run_id: 'run-001',
+        project: 'test-project',
+        config: { scenario: 'test-scenario' },
+        start_time: startTime,
+        metrics: {
+          success_rate: 1.0,
+          passed_cases: 5,
+          failed_cases: 0,
+          total_tokens: 500,
+          median_latency_ms: 100,
+        },
+        cases: [],
+      };
+
+      await writeFile(
+        join(testDir, 'artemis-runs', 'test-project', 'run-001.json'),
+        JSON.stringify(manifest)
+      );
+
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      const runs = await storage.list({ limit: 10 });
+
+      expect(runs[0].createdAt).toBe(startTime);
+    });
+  });
+});

package/src/__tests__/integration/init-command.test.ts

@@ -0,0 +1,177 @@
+/**
+ * Integration tests for init command
+ */
+
+import { describe, expect, it, beforeEach, afterEach } from 'bun:test';
+import { existsSync } from 'node:fs';
+import { readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { createTestDir, cleanupTestDir } from '../helpers/test-utils.js';
+
+// Import init command internals for testing
+// We'll test the file creation logic directly
+
+describe('Init Command', () => {
+  let testDir: string;
+  let originalCwd: string;
+
+  beforeEach(async () => {
+    testDir = await createTestDir('init-test');
+    originalCwd = process.cwd();
+    process.chdir(testDir);
+  });
+
+  afterEach(async () => {
+    process.chdir(originalCwd);
+    await cleanupTestDir(testDir);
+  });
+
+  describe('file creation', () => {
+    it('should create artemis.config.yaml with correct structure', async () => {
+      // Simulate what init command creates
+      const configContent = `# ArtemisKit Configuration
+# See: https://artemiskit.vercel.app/docs/configuration
+
+# Default provider (openai, azure-openai, anthropic)
+provider: openai
+
+# Default model
+model: gpt-4o-mini
+
+# Project name for organizing runs
+project: my-project
+
+# Storage configuration
+storage:
+  type: local
+  path: ./artemis-runs
+`;
+      const configPath = join(testDir, 'artemis.config.yaml');
+      await writeFile(configPath, configContent);
+
+      expect(existsSync(configPath)).toBe(true);
+
+      const content = await readFile(configPath, 'utf-8');
+      expect(content).toContain('provider: openai');
+      expect(content).toContain('model: gpt-4o-mini');
+      expect(content).toContain('storage:');
+    });
+
+    it('should create example scenario file', async () => {
+      const { mkdir } = await import('node:fs/promises');
+      const scenarioContent = `# Example Scenario
+name: example
+description: An example test scenario
+
+cases:
+  - id: greeting-test
+    prompt: "Say hello"
+    expected:
+      type: contains
+      values:
+        - "hello"
+      mode: any
+`;
+      const scenarioDir = join(testDir, 'scenarios');
+      await mkdir(scenarioDir, { recursive: true });
+      const scenarioPath = join(scenarioDir, 'example.yaml');
+
+      await writeFile(scenarioPath, scenarioContent);
+
+      expect(existsSync(scenarioPath)).toBe(true);
+
+      const content = await readFile(scenarioPath, 'utf-8');
+      expect(content).toContain('name: example');
+      expect(content).toContain('cases:');
+    });
+  });
+
+  describe('.env handling', () => {
+    it('should create .env with all provider keys when file does not exist', async () => {
+      const envContent = `# ArtemisKit Environment Variables
+OPENAI_API_KEY=
+AZURE_OPENAI_API_KEY=
+AZURE_OPENAI_RESOURCE=
+AZURE_OPENAI_DEPLOYMENT=
+AZURE_OPENAI_API_VERSION=
+ANTHROPIC_API_KEY=
+`;
+      const envPath = join(testDir, '.env');
+      await writeFile(envPath, envContent);
+
+      expect(existsSync(envPath)).toBe(true);
+
+      const content = await readFile(envPath, 'utf-8');
+      expect(content).toContain('OPENAI_API_KEY=');
+      expect(content).toContain('ANTHROPIC_API_KEY=');
+      expect(content).toContain('AZURE_OPENAI_API_KEY=');
+    });
+
+    it('should append missing keys to existing .env', async () => {
+      // Create existing .env with some keys
+      const existingEnv = `# Existing config
+OPENAI_API_KEY=sk-existing-key
+SOME_OTHER_VAR=value
+`;
+      const envPath = join(testDir, '.env');
+      await writeFile(envPath, existingEnv);
+
+      // Simulate appending missing keys
+      const envContent = await readFile(envPath, 'utf-8');
+      const keysToAdd = ['ANTHROPIC_API_KEY=', 'AZURE_OPENAI_API_KEY='];
+
+      const missingKeys = keysToAdd.filter((key) => {
+        const keyName = key.split('=')[0];
+        return !envContent.includes(keyName);
+      });
+
+      if (missingKeys.length > 0) {
+        const newContent = envContent + '\n# Added by ArtemisKit\n' + missingKeys.join('\n') + '\n';
+        await writeFile(envPath, newContent);
+      }
+
+      const finalContent = await readFile(envPath, 'utf-8');
+      expect(finalContent).toContain('OPENAI_API_KEY=sk-existing-key'); // preserved
+      expect(finalContent).toContain('SOME_OTHER_VAR=value'); // preserved
+      expect(finalContent).toContain('ANTHROPIC_API_KEY='); // added
+      expect(finalContent).toContain('AZURE_OPENAI_API_KEY='); // added
+    });
+
+    it('should not duplicate existing keys', async () => {
+      const existingEnv = `OPENAI_API_KEY=sk-test
+ANTHROPIC_API_KEY=sk-ant-test
+`;
+      const envPath = join(testDir, '.env');
+      await writeFile(envPath, existingEnv);
+
+      const envContent = await readFile(envPath, 'utf-8');
+      const keysToCheck = ['OPENAI_API_KEY=', 'ANTHROPIC_API_KEY='];
+
+      const missingKeys = keysToCheck.filter((key) => {
+        const keyName = key.split('=')[0];
+        return !envContent.includes(keyName);
+      });
+
+      // No keys should be missing
+      expect(missingKeys.length).toBe(0);
+    });
+  });
+
+  describe('directory structure', () => {
+    it('should create scenarios directory', async () => {
+      const { mkdir } = await import('node:fs/promises');
+      const scenariosDir = join(testDir, 'scenarios');
+      await mkdir(scenariosDir, { recursive: true });
+
+      expect(existsSync(scenariosDir)).toBe(true);
+    });
+
+    it('should create artemis-runs directory for local storage', async () => {
+      const { mkdir } = await import('node:fs/promises');
+      const runsDir = join(testDir, 'artemis-runs');
+      await mkdir(runsDir, { recursive: true });
+
+      expect(existsSync(runsDir)).toBe(true);
+    });
+  });
+});

package/src/__tests__/integration/report-command.test.ts

@@ -0,0 +1,245 @@
+/**
+ * Integration tests for report command
+ */
+
+import { describe, expect, it, beforeEach, afterEach } from 'bun:test';
+import { existsSync } from 'node:fs';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { generateHTMLReport, generateJSONReport } from '@artemiskit/reports';
+import { createTestDir, cleanupTestDir } from '../helpers/test-utils.js';
+import { createStorage } from '../../utils/storage.js';
+
+describe('Report Command', () => {
+  let testDir: string;
+  let originalCwd: string;
+
+  const sampleManifest = {
+    run_id: 'test-run-001',
+    project: 'test-project',
+    config: {
+      scenario: 'test-scenario',
+      provider: 'openai',
+      model: 'gpt-4o-mini',
+    },
+    start_time: new Date().toISOString(),
+    duration_ms: 5000,
+    metrics: {
+      success_rate: 0.8,
+      passed_cases: 4,
+      failed_cases: 1,
+      total_tokens: 500,
+      median_latency_ms: 150,
+      avg_latency_ms: 160,
+      min_latency_ms: 100,
+      max_latency_ms: 250,
+      p95_latency_ms: 240,
+      p99_latency_ms: 248,
+    },
+    cases: [
+      {
+        id: 'case-1',
+        prompt: 'Test prompt 1',
+        response: 'Test response 1',
+        expected: { type: 'contains', values: ['test'], mode: 'any' },
+        ok: true,
+        score: 1.0,
+        reason: 'Passed',
+        latencyMs: 100,
+        tokens: { input: 10, output: 20 },
+      },
+      {
+        id: 'case-2',
+        prompt: 'Test prompt 2',
+        response: 'Test response 2',
+        expected: { type: 'contains', values: ['expected'], mode: 'any' },
+        ok: false,
+        score: 0,
+        reason: 'Did not contain expected value',
+        latencyMs: 200,
+        tokens: { input: 15, output: 25 },
+      },
+    ],
+  };
+
+  beforeEach(async () => {
+    testDir = await createTestDir('report-test');
+    originalCwd = process.cwd();
+    process.chdir(testDir);
+
+    // Create storage directory and save manifest
+    await mkdir(join(testDir, 'artemis-runs', 'test-project'), { recursive: true });
+    await writeFile(
+      join(testDir, 'artemis-runs', 'test-project', 'test-run-001.json'),
+      JSON.stringify(sampleManifest)
+    );
+  });
+
+  afterEach(async () => {
+    process.chdir(originalCwd);
+    await cleanupTestDir(testDir);
+  });
+
+  describe('report generation', () => {
+    it('should load manifest from storage', async () => {
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      const manifest = await storage.load('test-run-001');
+
+      expect(manifest.run_id).toBe('test-run-001');
+      expect(manifest.config.scenario).toBe('test-scenario');
+      expect(manifest.metrics.success_rate).toBe(0.8);
+    });
+
+    it('should generate HTML report from manifest', async () => {
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      const manifest = await storage.load('test-run-001');
+      const html = generateHTMLReport(manifest);
+
+      expect(html).toContain('<!DOCTYPE html>');
+      expect(html).toContain('test-run-001');
+    });
+
+    it('should generate JSON report from manifest', async () => {
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      const manifest = await storage.load('test-run-001');
+      const json = generateJSONReport(manifest, { pretty: true });
+      const parsed = JSON.parse(json);
+
+      expect(parsed.run_id).toBe('test-run-001');
+      expect(parsed.metrics.success_rate).toBe(0.8);
+    });
+
+    it('should write HTML report to output directory', async () => {
+      const outputDir = join(testDir, 'output');
+      await mkdir(outputDir, { recursive: true });
+
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      const manifest = await storage.load('test-run-001');
+      const html = generateHTMLReport(manifest);
+      const outputPath = join(outputDir, 'test-run-001.html');
+      await writeFile(outputPath, html);
+
+      expect(existsSync(outputPath)).toBe(true);
+
+      const content = await readFile(outputPath, 'utf-8');
+      expect(content).toContain('<!DOCTYPE html>');
+    });
+
+    it('should write JSON report to output directory', async () => {
+      const outputDir = join(testDir, 'output');
+      await mkdir(outputDir, { recursive: true });
+
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      const manifest = await storage.load('test-run-001');
+      const json = generateJSONReport(manifest, { pretty: true });
+      const outputPath = join(outputDir, 'test-run-001.json');
+      await writeFile(outputPath, json);
+
+      expect(existsSync(outputPath)).toBe(true);
+
+      const content = await readFile(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      expect(parsed.run_id).toBe('test-run-001');
+    });
+
+    it('should generate both formats when format is "both"', async () => {
+      const outputDir = join(testDir, 'output');
+      await mkdir(outputDir, { recursive: true });
+
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      const manifest = await storage.load('test-run-001');
+
+      // Generate both formats
+      const html = generateHTMLReport(manifest);
+      const json = generateJSONReport(manifest, { pretty: true });
+
+      await writeFile(join(outputDir, 'test-run-001.html'), html);
+      await writeFile(join(outputDir, 'test-run-001.json'), json);
+
+      expect(existsSync(join(outputDir, 'test-run-001.html'))).toBe(true);
+      expect(existsSync(join(outputDir, 'test-run-001.json'))).toBe(true);
+    });
+
+    it('should throw error for non-existent run ID', async () => {
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      await expect(storage.load('non-existent-run')).rejects.toThrow();
+    });
+  });
+
+  describe('HTML report content', () => {
+    it('should include test case details', async () => {
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      const manifest = await storage.load('test-run-001');
+      const html = generateHTMLReport(manifest);
+
+      expect(html).toContain('case-1');
+      expect(html).toContain('case-2');
+    });
+
+    it('should include metrics summary', async () => {
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      const manifest = await storage.load('test-run-001');
+      const html = generateHTMLReport(manifest);
+
+      expect(html).toContain('80'); // 80% success rate
+    });
+
+    it('should include run ID in report', async () => {
+      const storage = createStorage({
+        fileConfig: {
+          storage: { type: 'local', basePath: join(testDir, 'artemis-runs') },
+        },
+      });
+
+      const manifest = await storage.load('test-run-001');
+      const html = generateHTMLReport(manifest);
+
+      expect(html).toContain('test-run-001');
+    });
+  });
+});