@artemiskit/core 0.1.2

package/src/adapters/types.ts

@@ -0,0 +1,184 @@
+/**
+ * Core types and interfaces for model adapters
+ */
+
+/**
+ * Chat message format compatible with OpenAI/Anthropic
+ */
+export interface ChatMessage {
+  role: 'system' | 'user' | 'assistant' | 'function' | 'tool';
+  content: string;
+  name?: string;
+  function_call?: {
+    name: string;
+    arguments: string;
+  };
+  tool_calls?: ToolCall[];
+}
+
+export interface ToolCall {
+  id: string;
+  type: 'function';
+  function: {
+    name: string;
+    arguments: string;
+  };
+}
+
+/**
+ * Options for generating a completion
+ */
+export interface GenerateOptions {
+  prompt: string | ChatMessage[];
+  model?: string;
+  maxTokens?: number;
+  temperature?: number;
+  topP?: number;
+  seed?: number;
+  stop?: string[];
+  functions?: FunctionDefinition[];
+  tools?: ToolDefinition[];
+  responseFormat?: { type: 'text' | 'json_object' };
+  metadata?: Record<string, unknown>;
+}
+
+/**
+ * Function/Tool definitions for function calling
+ */
+export interface FunctionDefinition {
+  name: string;
+  description?: string;
+  parameters: Record<string, unknown>;
+}
+
+export interface ToolDefinition {
+  type: 'function';
+  function: FunctionDefinition;
+}
+
+/**
+ * Token usage information
+ */
+export interface TokenUsage {
+  prompt: number;
+  completion: number;
+  total: number;
+}
+
+/**
+ * Result from a generation request
+ */
+export interface GenerateResult {
+  id: string;
+  model: string;
+  text: string;
+  tokens: TokenUsage;
+  latencyMs: number;
+  finishReason?: 'stop' | 'length' | 'function_call' | 'tool_calls' | 'content_filter';
+  functionCall?: {
+    name: string;
+    arguments: string;
+  };
+  toolCalls?: ToolCall[];
+  raw?: unknown;
+}
+
+/**
+ * Model capabilities
+ */
+export interface ModelCapabilities {
+  streaming: boolean;
+  functionCalling: boolean;
+  toolUse: boolean;
+  maxContext: number;
+  vision?: boolean;
+  jsonMode?: boolean;
+}
+
+/**
+ * ModelClient interface - All adapters must implement this
+ */
+export interface ModelClient {
+  readonly provider: string;
+
+  generate(options: GenerateOptions): Promise<GenerateResult>;
+
+  stream?(options: GenerateOptions, onChunk: (chunk: string) => void): AsyncIterable<string>;
+
+  embed?(text: string): Promise<number[]>;
+
+  capabilities(): Promise<ModelCapabilities>;
+
+  close?(): Promise<void>;
+}
+
+/**
+ * Provider types - all supported providers
+ */
+export type ProviderType =
+  | 'openai'
+  | 'azure-openai'
+  | 'vercel-ai'
+  | 'anthropic'
+  | 'google'
+  | 'mistral'
+  | 'cohere'
+  | 'huggingface'
+  | 'ollama'
+  | 'custom';
+
+/**
+ * Base adapter configuration
+ */
+export interface BaseAdapterConfig {
+  provider: ProviderType;
+  apiKey?: string;
+  baseUrl?: string;
+  defaultModel?: string;
+  timeout?: number;
+  maxRetries?: number;
+}
+
+/**
+ * OpenAI-specific configuration
+ */
+export interface OpenAIAdapterConfig extends BaseAdapterConfig {
+  provider: 'openai';
+  organization?: string;
+}
+
+/**
+ * Azure OpenAI-specific configuration
+ */
+export interface AzureOpenAIAdapterConfig extends BaseAdapterConfig {
+  provider: 'azure-openai';
+  resourceName: string;
+  deploymentName: string;
+  apiVersion: string;
+}
+
+/**
+ * Vercel AI SDK configuration
+ */
+export interface VercelAIAdapterConfig extends BaseAdapterConfig {
+  provider: 'vercel-ai';
+  underlyingProvider: 'openai' | 'azure' | 'anthropic' | 'google' | 'mistral';
+  providerConfig?: Record<string, unknown>;
+}
+
+/**
+ * Anthropic-specific configuration (Post-MVP)
+ */
+export interface AnthropicAdapterConfig extends BaseAdapterConfig {
+  provider: 'anthropic';
+}
+
+/**
+ * Union type for all adapter configs
+ */
+export type AdapterConfig =
+  | OpenAIAdapterConfig
+  | AzureOpenAIAdapterConfig
+  | VercelAIAdapterConfig
+  | AnthropicAdapterConfig
+  | BaseAdapterConfig;

package/src/artifacts/index.ts

@@ -0,0 +1,6 @@
+/**
+ * Artifacts module exports
+ */
+
+export * from './types';
+export { createRunManifest } from './manifest';

package/src/artifacts/manifest.test.ts

@@ -0,0 +1,206 @@
+/**
+ * Tests for manifest generation
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { createRunManifest } from './manifest';
+import type { CaseResult } from './types';
+
+describe('createRunManifest', () => {
+  const mockCases: CaseResult[] = [
+    {
+      id: 'case-1',
+      name: 'Test Case 1',
+      ok: true,
+      latencyMs: 100,
+      prompt: 'Hello',
+      response: 'Hi there!',
+      tokens: { prompt: 10, completion: 5 },
+      evaluations: [
+        {
+          type: 'contains',
+          passed: true,
+          score: 1,
+          reason: 'Contains expected value',
+        },
+      ],
+    },
+    {
+      id: 'case-2',
+      name: 'Test Case 2',
+      ok: false,
+      latencyMs: 200,
+      prompt: 'Goodbye',
+      response: 'See you!',
+      tokens: { prompt: 8, completion: 4 },
+      evaluations: [
+        {
+          type: 'contains',
+          passed: false,
+          score: 0,
+          reason: 'Missing expected value',
+        },
+      ],
+    },
+  ];
+
+  test('creates manifest with correct structure', () => {
+    const startTime = new Date('2024-01-01T00:00:00Z');
+    const endTime = new Date('2024-01-01T00:01:00Z');
+
+    const manifest = createRunManifest({
+      project: 'test-project',
+      config: {
+        scenario: 'test-scenario',
+        provider: 'openai',
+        model: 'gpt-4',
+      },
+      cases: mockCases,
+      startTime,
+      endTime,
+    });
+
+    expect(manifest.version).toBe('1.0');
+    expect(manifest.project).toBe('test-project');
+    expect(manifest.run_id).toBeTruthy();
+    expect(manifest.run_id.length).toBe(12);
+    expect(manifest.config.scenario).toBe('test-scenario');
+    expect(manifest.config.provider).toBe('openai');
+    expect(manifest.config.model).toBe('gpt-4');
+  });
+
+  test('calculates metrics correctly', () => {
+    const manifest = createRunManifest({
+      project: 'test-project',
+      config: {
+        scenario: 'test-scenario',
+        provider: 'openai',
+        model: 'gpt-4',
+      },
+      cases: mockCases,
+      startTime: new Date(),
+      endTime: new Date(),
+    });
+
+    expect(manifest.metrics.total_cases).toBe(2);
+    expect(manifest.metrics.passed_cases).toBe(1);
+    expect(manifest.metrics.failed_cases).toBe(1);
+    expect(manifest.metrics.success_rate).toBe(0.5);
+    expect(manifest.metrics.total_tokens).toBe(27); // (10+5) + (8+4)
+    expect(manifest.metrics.total_prompt_tokens).toBe(18); // 10 + 8
+    expect(manifest.metrics.total_completion_tokens).toBe(9); // 5 + 4
+  });
+
+  test('calculates duration correctly', () => {
+    const startTime = new Date('2024-01-01T00:00:00Z');
+    const endTime = new Date('2024-01-01T00:01:00Z');
+
+    const manifest = createRunManifest({
+      project: 'test-project',
+      config: {
+        scenario: 'test-scenario',
+        provider: 'openai',
+        model: 'gpt-4',
+      },
+      cases: mockCases,
+      startTime,
+      endTime,
+    });
+
+    expect(manifest.duration_ms).toBe(60000); // 1 minute
+    expect(manifest.start_time).toBe('2024-01-01T00:00:00.000Z');
+    expect(manifest.end_time).toBe('2024-01-01T00:01:00.000Z');
+  });
+
+  test('handles empty cases array', () => {
+    const manifest = createRunManifest({
+      project: 'test-project',
+      config: {
+        scenario: 'test-scenario',
+        provider: 'openai',
+        model: 'gpt-4',
+      },
+      cases: [],
+      startTime: new Date(),
+      endTime: new Date(),
+    });
+
+    expect(manifest.metrics.total_cases).toBe(0);
+    expect(manifest.metrics.passed_cases).toBe(0);
+    expect(manifest.metrics.failed_cases).toBe(0);
+    expect(manifest.metrics.success_rate).toBe(0);
+    expect(manifest.metrics.median_latency_ms).toBe(0);
+    expect(manifest.metrics.p95_latency_ms).toBe(0);
+  });
+
+  test('includes resolved_config when provided', () => {
+    const manifest = createRunManifest({
+      project: 'test-project',
+      config: {
+        scenario: 'test-scenario',
+        provider: 'openai',
+        model: 'gpt-4',
+      },
+      resolvedConfig: {
+        provider: 'openai',
+        model: 'gpt-4',
+        source: {
+          provider: 'cli',
+          model: 'config',
+        },
+      },
+      cases: mockCases,
+      startTime: new Date(),
+      endTime: new Date(),
+    });
+
+    expect(manifest.resolved_config).toBeDefined();
+    expect(manifest.resolved_config?.provider).toBe('openai');
+    expect(manifest.resolved_config?.source.provider).toBe('cli');
+    expect(manifest.resolved_config?.source.model).toBe('config');
+  });
+
+  test('includes provenance information', () => {
+    const manifest = createRunManifest({
+      project: 'test-project',
+      config: {
+        scenario: 'test-scenario',
+        provider: 'openai',
+        model: 'gpt-4',
+      },
+      cases: mockCases,
+      startTime: new Date(),
+      endTime: new Date(),
+      runBy: 'test-user',
+      runReason: 'unit-test',
+    });
+
+    expect(manifest.provenance.run_by).toBe('test-user');
+    expect(manifest.provenance.run_reason).toBe('unit-test');
+  });
+
+  test('calculates latency percentiles correctly', () => {
+    const casesWithLatencies: CaseResult[] = [
+      { ...mockCases[0], latencyMs: 100 },
+      { ...mockCases[0], id: 'case-2', latencyMs: 200 },
+      { ...mockCases[0], id: 'case-3', latencyMs: 300 },
+      { ...mockCases[0], id: 'case-4', latencyMs: 400 },
+      { ...mockCases[0], id: 'case-5', latencyMs: 500 },
+    ];
+
+    const manifest = createRunManifest({
+      project: 'test-project',
+      config: {
+        scenario: 'test-scenario',
+        provider: 'openai',
+        model: 'gpt-4',
+      },
+      cases: casesWithLatencies,
+      startTime: new Date(),
+      endTime: new Date(),
+    });
+
+    expect(manifest.metrics.median_latency_ms).toBe(300);
+    expect(manifest.metrics.p95_latency_ms).toBe(500);
+  });
+});

package/src/artifacts/manifest.ts

@@ -0,0 +1,136 @@
+/**
+ * Run manifest generation utilities
+ */
+
+import { nanoid } from 'nanoid';
+import { getEnvironmentInfo } from '../provenance/environment';
+import { getGitInfo } from '../provenance/git';
+import type {
+  CaseResult,
+  ManifestRedactionInfo,
+  ResolvedConfig,
+  RunConfig,
+  RunManifest,
+  RunMetrics,
+} from './types';
+
+/**
+ * Create a new run manifest
+ */
+export function createRunManifest(options: {
+  project: string;
+  config: RunConfig;
+  resolvedConfig?: ResolvedConfig;
+  cases: CaseResult[];
+  startTime: Date;
+  endTime: Date;
+  runBy?: string;
+  runReason?: string;
+  redaction?: ManifestRedactionInfo;
+}): RunManifest {
+  const {
+    project,
+    config,
+    resolvedConfig,
+    cases,
+    startTime,
+    endTime,
+    runBy,
+    runReason,
+    redaction,
+  } = options;
+
+  const metrics = calculateMetrics(cases);
+  const git = getGitInfo();
+  const environment = getEnvironmentInfo();
+
+  return {
+    version: '1.0',
+    run_id: nanoid(12),
+    project,
+    start_time: startTime.toISOString(),
+    end_time: endTime.toISOString(),
+    duration_ms: endTime.getTime() - startTime.getTime(),
+    config,
+    resolved_config: resolvedConfig,
+    metrics,
+    git,
+    provenance: {
+      run_by: runBy || process.env.USER || 'unknown',
+      run_reason: runReason,
+      ci: detectCIEnvironment(),
+    },
+    cases,
+    environment,
+    redaction,
+  };
+}
+
+/**
+ * Calculate metrics from case results
+ */
+function calculateMetrics(cases: CaseResult[]): RunMetrics {
+  const passedCases = cases.filter((c) => c.ok);
+  const latencies = cases.map((c) => c.latencyMs).sort((a, b) => a - b);
+
+  const medianLatency = latencies.length > 0 ? latencies[Math.floor(latencies.length / 2)] : 0;
+
+  const p95Index = Math.floor(latencies.length * 0.95);
+  const p95Latency = latencies.length > 0 ? latencies[p95Index] : 0;
+
+  const totalPromptTokens = cases.reduce((sum, c) => sum + c.tokens.prompt, 0);
+  const totalCompletionTokens = cases.reduce((sum, c) => sum + c.tokens.completion, 0);
+
+  return {
+    success_rate: cases.length > 0 ? passedCases.length / cases.length : 0,
+    total_cases: cases.length,
+    passed_cases: passedCases.length,
+    failed_cases: cases.length - passedCases.length,
+    median_latency_ms: medianLatency,
+    p95_latency_ms: p95Latency,
+    total_tokens: totalPromptTokens + totalCompletionTokens,
+    total_prompt_tokens: totalPromptTokens,
+    total_completion_tokens: totalCompletionTokens,
+  };
+}
+
+/**
+ * Detect CI environment
+ */
+function detectCIEnvironment():
+  | { provider: string; build_id: string; build_url?: string }
+  | undefined {
+  if (process.env.GITHUB_ACTIONS) {
+    return {
+      provider: 'github-actions',
+      build_id: process.env.GITHUB_RUN_ID || '',
+      build_url: `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`,
+    };
+  }
+
+  if (process.env.GITLAB_CI) {
+    return {
+      provider: 'gitlab-ci',
+      build_id: process.env.CI_JOB_ID || '',
+      build_url: process.env.CI_JOB_URL,
+    };
+  }
+
+  if (process.env.CIRCLECI) {
+    return {
+      provider: 'circleci',
+      build_id: process.env.CIRCLE_BUILD_NUM || '',
+      build_url: process.env.CIRCLE_BUILD_URL,
+    };
+  }
+
+  if (process.env.JENKINS_URL) {
+    return {
+      provider: 'jenkins',
+      build_id: process.env.BUILD_ID || '',
+      build_url: process.env.BUILD_URL,
+    };
+  }
+
+  return undefined;
+}