@artemiskit/sdk 0.3.0

package/src/__tests__/artemiskit.test.ts

@@ -0,0 +1,425 @@
+/**
+ * @artemiskit/sdk
+ * Tests for ArtemisKit class
+ */
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+// Mock modules before importing ArtemisKit
+vi.mock('@artemiskit/core', () => ({
+  createAdapter: vi.fn().mockResolvedValue({
+    provider: 'mock',
+    generate: vi.fn().mockResolvedValue({
+      id: 'test-id',
+      model: 'mock-model',
+      text: 'Hello, world!',
+      tokens: { prompt: 10, completion: 5, total: 15 },
+      latencyMs: 100,
+      finishReason: 'stop',
+    }),
+    capabilities: vi.fn().mockResolvedValue({
+      streaming: false,
+      functionCalling: false,
+      toolUse: false,
+      maxContext: 4096,
+    }),
+    close: vi.fn(),
+  }),
+  parseScenarioFile: vi.fn().mockResolvedValue({
+    name: 'Test Scenario',
+    version: '1.0',
+    model: 'mock-model',
+    cases: [
+      {
+        id: 'case-1',
+        name: 'Test Case 1',
+        prompt: 'Say hello',
+        expected: { type: 'contains', values: ['Hello'], mode: 'any' },
+        tags: ['greeting'],
+        metadata: {},
+        retries: 0,
+      },
+      {
+        id: 'case-2',
+        name: 'Test Case 2',
+        prompt: 'Say goodbye',
+        expected: { type: 'contains', values: ['Goodbye'], mode: 'any' },
+        tags: ['farewell'],
+        metadata: {},
+        retries: 0,
+      },
+    ],
+    tags: [],
+  }),
+  // biome-ignore lint/suspicious/noExplicitAny: Mock helper
+  runScenario: vi.fn().mockImplementation(async (options: any) => {
+    const cases = options.scenario.cases;
+    // biome-ignore lint/suspicious/noExplicitAny: Mock helper
+    const results = cases.map((c: any, i: number) => {
+      const result = {
+        id: c.id,
+        name: c.name,
+        ok: true,
+        score: 1,
+        matcherType: c.expected.type,
+        reason: 'Passed',
+        latencyMs: 100 + i * 10,
+        tokens: { prompt: 10, completion: 5, total: 15 },
+        prompt: c.prompt,
+        response: 'Hello, world!',
+        expected: c.expected,
+        tags: c.tags,
+      };
+      options.onCaseComplete?.(result, i, cases.length);
+      return result;
+    });
+
+    return {
+      manifest: {
+        version: '1.0',
+        run_id: 'test-run-id',
+        project: options.project || 'default',
+        start_time: new Date().toISOString(),
+        end_time: new Date().toISOString(),
+        duration_ms: 200,
+        config: {
+          scenario: options.scenario.name,
+          provider: 'mock',
+          model: 'mock-model',
+        },
+        metrics: {
+          success_rate: 1,
+          total_cases: cases.length,
+          passed_cases: cases.length,
+          failed_cases: 0,
+          median_latency_ms: 105,
+          p95_latency_ms: 110,
+          total_tokens: 30,
+          total_prompt_tokens: 20,
+          total_completion_tokens: 10,
+        },
+        git: {
+          commit: 'abc123',
+          branch: 'main',
+          dirty: false,
+        },
+        provenance: {
+          run_by: 'test',
+        },
+        cases: results,
+        environment: {
+          node_version: process.version,
+          platform: process.platform,
+          arch: process.arch,
+        },
+      },
+      cases: results,
+      success: true,
+    };
+  }),
+  getGitInfo: vi.fn().mockResolvedValue({
+    commit: 'abc123',
+    branch: 'main',
+    dirty: false,
+  }),
+  estimateCost: vi.fn(),
+  formatCost: vi.fn(),
+  getModelPricing: vi.fn(),
+}));
+
+vi.mock('@artemiskit/redteam', () => {
+  return {
+    TypoMutation: class {
+      name = 'typo';
+      description = 'Typo mutation';
+      severity = 'low' as const;
+      mutate(prompt: string) {
+        return `typo: ${prompt}`;
+      }
+    },
+    RoleSpoofMutation: class {
+      name = 'role-spoof';
+      description = 'Role spoof mutation';
+      severity = 'medium' as const;
+      mutate(prompt: string) {
+        return `role: ${prompt}`;
+      }
+    },
+    InstructionFlipMutation: class {
+      name = 'instruction-flip';
+      description = 'Instruction flip mutation';
+      severity = 'medium' as const;
+      mutate(prompt: string) {
+        return `flip: ${prompt}`;
+      }
+    },
+    CotInjectionMutation: class {
+      name = 'cot-injection';
+      description = 'COT injection mutation';
+      severity = 'high' as const;
+      mutate(prompt: string) {
+        return `cot: ${prompt}`;
+      }
+    },
+    EncodingMutation: class {
+      name = 'encoding';
+      description = 'Encoding mutation';
+      severity = 'medium' as const;
+      mutate(prompt: string) {
+        return `enc: ${prompt}`;
+      }
+    },
+    MultiTurnMutation: class {
+      name = 'multi-turn';
+      description = 'Multi-turn mutation';
+      severity = 'high' as const;
+      mutate(prompt: string) {
+        return `multi: ${prompt}`;
+      }
+    },
+    RedTeamGenerator: class {
+      // biome-ignore lint/suspicious/noExplicitAny: Mock helper
+      mutations: any[];
+      // biome-ignore lint/suspicious/noExplicitAny: Mock helper
+      constructor(mutations?: any[]) {
+        this.mutations = mutations ?? [];
+      }
+      generate(prompt: string, count: number) {
+        return Array.from({ length: count }, (_, i) => ({
+          original: prompt,
+          mutated: `mutated-${i}: ${prompt}`,
+          mutations: ['typo'],
+          severity: 'low' as const,
+        }));
+      }
+      listMutations() {
+        // biome-ignore lint/suspicious/noExplicitAny: Mock helper
+        return this.mutations.map((m: any) => ({
+          name: m.name,
+          description: m.description,
+          severity: m.severity,
+        }));
+      }
+    },
+    UnsafeResponseDetector: class {
+      detect() {
+        return { unsafe: false, blocked: false, reasons: [], severity: 'none' };
+      }
+    },
+    SeverityMapper: class {
+      mapMutationToSeverity() {
+        return 'none';
+      }
+    },
+  };
+});
+
+vi.mock('nanoid', () => ({
+  nanoid: () => 'test-id-12345',
+}));
+
+// Import after mocks are set up
+import { ArtemisKit } from '../artemiskit';
+
+describe('ArtemisKit', () => {
+  let kit: ArtemisKit;
+
+  beforeEach(() => {
+    kit = new ArtemisKit({
+      project: 'test-project',
+      provider: 'openai',
+      model: 'gpt-4',
+    });
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  describe('constructor', () => {
+    it('should create instance with default config', () => {
+      const defaultKit = new ArtemisKit();
+      expect(defaultKit).toBeInstanceOf(ArtemisKit);
+    });
+
+    it('should create instance with custom config', () => {
+      const customKit = new ArtemisKit({
+        project: 'my-project',
+        provider: 'azure-openai',
+        model: 'gpt-4-turbo',
+        timeout: 30000,
+        retries: 3,
+        concurrency: 5,
+      });
+      expect(customKit).toBeInstanceOf(ArtemisKit);
+    });
+  });
+
+  describe('event handling', () => {
+    it('should register and emit caseComplete events', async () => {
+      const handler = vi.fn();
+      kit.onCaseComplete(handler);
+
+      await kit.run({ scenario: './test.yaml' });
+
+      expect(handler).toHaveBeenCalledTimes(2);
+      expect(handler).toHaveBeenCalledWith(
+        expect.objectContaining({
+          result: expect.objectContaining({ id: 'case-1' }),
+          index: 0,
+          total: 2,
+        })
+      );
+    });
+
+    it('should register and emit progress events', async () => {
+      const handler = vi.fn();
+      kit.onProgress(handler);
+
+      await kit.run({ scenario: './test.yaml' });
+
+      expect(handler).toHaveBeenCalled();
+      expect(handler).toHaveBeenCalledWith(
+        expect.objectContaining({
+          phase: expect.stringMatching(/setup|running|teardown/),
+        })
+      );
+    });
+
+    it('should support once() for one-time handlers', async () => {
+      const handler = vi.fn();
+      kit.once('progress', handler);
+
+      await kit.run({ scenario: './test.yaml' });
+
+      // Handler should have been called only once despite multiple progress events
+      expect(handler).toHaveBeenCalledTimes(1);
+    });
+
+    it('should support off() to remove handlers', async () => {
+      const handler = vi.fn();
+      kit.on('progress', handler);
+      kit.off('progress', handler);
+
+      await kit.run({ scenario: './test.yaml' });
+
+      expect(handler).not.toHaveBeenCalled();
+    });
+
+    it('should chain event registration methods', () => {
+      const result = kit
+        .onCaseStart(() => {})
+        .onCaseComplete(() => {})
+        .onProgress(() => {});
+
+      expect(result).toBe(kit);
+    });
+  });
+
+  describe('run()', () => {
+    it('should run a scenario from file path', async () => {
+      const result = await kit.run({ scenario: './test.yaml' });
+
+      expect(result.success).toBe(true);
+      expect(result.manifest.metrics.total_cases).toBe(2);
+      expect(result.manifest.metrics.passed_cases).toBe(2);
+    });
+
+    it('should run a scenario from inline object', async () => {
+      const scenario = {
+        name: 'Inline Scenario',
+        version: '1.0',
+        model: 'gpt-4',
+        cases: [
+          {
+            id: 'inline-1',
+            name: 'Inline Test',
+            prompt: 'Hello',
+            expected: {
+              type: 'contains' as const,
+              values: ['Hello'],
+              mode: 'any' as const,
+            },
+            tags: [],
+            metadata: {},
+            retries: 0,
+          },
+        ],
+        tags: [],
+      };
+
+      const result = await kit.run({ scenario });
+
+      expect(result.success).toBe(true);
+    });
+
+    it('should pass tags filter to runner', async () => {
+      const result = await kit.run({
+        scenario: './test.yaml',
+        tags: ['greeting'],
+      });
+
+      expect(result.success).toBe(true);
+    });
+
+    it('should use custom concurrency', async () => {
+      const result = await kit.run({
+        scenario: './test.yaml',
+        concurrency: 5,
+      });
+
+      expect(result.success).toBe(true);
+    });
+
+    it('should use custom timeout', async () => {
+      const result = await kit.run({
+        scenario: './test.yaml',
+        timeout: 60000,
+      });
+
+      expect(result.success).toBe(true);
+    });
+  });
+
+  describe('getAvailableMutations()', () => {
+    it('should return list of available mutations', () => {
+      const mutations = kit.getAvailableMutations();
+
+      expect(Array.isArray(mutations)).toBe(true);
+      expect(mutations).toContain('typo');
+      expect(mutations).toContain('role-spoof');
+      expect(mutations).toContain('encoding');
+    });
+  });
+});
+
+describe('ArtemisKit with custom client', () => {
+  it('should use provided client instead of creating one', async () => {
+    const mockClient = {
+      provider: 'custom',
+      generate: vi.fn().mockResolvedValue({
+        id: 'custom-id',
+        model: 'custom-model',
+        text: 'Custom response',
+        tokens: { prompt: 5, completion: 3, total: 8 },
+        latencyMs: 50,
+        finishReason: 'stop',
+      }),
+      capabilities: vi.fn().mockResolvedValue({
+        streaming: false,
+        functionCalling: false,
+        toolUse: false,
+        maxContext: 4096,
+      }),
+    };
+
+    const kit = new ArtemisKit();
+    const result = await kit.run({
+      scenario: './test.yaml',
+      // @ts-expect-error - Testing with mock client
+      client: mockClient,
+    });
+
+    expect(result.success).toBe(true);
+  });
+});