@artemiskit/core 0.1.2

package/CHANGELOG.md

@@ -0,0 +1,48 @@
+# @artemiskit/core
+
+## 0.1.2
+
+### Patch Changes
+
+- 6350e5d: Initial release of ArtemisKit - LLM Evaluation Toolkit
+
+  Features:
+
+  - Scenario-based evaluation with YAML test definitions
+  - Multiple expectation types: contains, exact, regex, fuzzy, llm_grader, json_schema
+  - Red team security testing (injection, jailbreak, extraction, hallucination, PII)
+  - Stress testing with configurable concurrency and latency metrics
+  - PII/sensitive data redaction with built-in and custom patterns
+  - Interactive HTML reports with run comparison
+  - Multi-provider support: OpenAI, Azure OpenAI, Anthropic
+  - Local and Supabase storage backends
+
+## 0.1.1
+
+### Patch Changes
+
+- 1200625: Initial release of ArtemisKit - LLM Evaluation Toolkit
+
+  ArtemisKit is a comprehensive toolkit for testing, validating, and auditing LLM-powered applications.
+
+  Features:
+
+  - Scenario-based evaluation with YAML test definitions
+  - Multiple expectation types: contains, exact, regex, fuzzy, llm_grader, json_schema
+  - Red team security testing (injection, jailbreak, extraction, hallucination, PII disclosure)
+  - Stress testing with configurable concurrency and detailed latency metrics
+  - PII/sensitive data redaction with built-in and custom patterns
+  - Interactive HTML reports with run comparison
+  - Multi-provider support: OpenAI, Azure OpenAI, Anthropic
+  - Local and Supabase storage backends
+  - Run history tracking and comparison
+
+  CLI Commands:
+
+  - artemiskit run - Execute scenario-based evaluations
+  - artemiskit redteam - Run adversarial security tests
+  - artemiskit stress - Perform load testing
+  - artemiskit report - Regenerate reports from manifests
+  - artemiskit history - View past runs
+  - artemiskit compare - Compare two runs
+  - artemiskit init - Initialize configuration

package/dist/adapters/factory.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Adapter factory for creating model clients
+ */
+import type { AdapterConfig, ModelClient, ProviderType } from './types';
+/**
+ * Create a model adapter from configuration
+ */
+export declare function createAdapter(config: AdapterConfig): Promise<ModelClient>;
+/**
+ * Register a custom adapter
+ */
+export declare function registerAdapter(provider: ProviderType | string, factory: (config: AdapterConfig) => Promise<ModelClient>): void;
+/**
+ * List available adapters
+ */
+export declare function listAdapters(): {
+    available: string[];
+    unavailable: {
+        provider: string;
+        reason: string;
+    }[];
+};
+//# sourceMappingURL=factory.d.ts.map

package/dist/adapters/factory.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../src/adapters/factory.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAIxE;;GAEG;AACH,wBAAsB,aAAa,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,WAAW,CAAC,CAsC/E;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,YAAY,GAAG,MAAM,EAC/B,OAAO,EAAE,CAAC,MAAM,EAAE,aAAa,KAAK,OAAO,CAAC,WAAW,CAAC,GACvD,IAAI,CAEN;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI;IAC9B,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,WAAW,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACrD,CAKA"}

package/dist/adapters/index.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Adapter module exports
+ */
+export * from './types';
+export * from './factory';
+export * from './registry';
+//# sourceMappingURL=index.d.ts.map

package/dist/adapters/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/adapters/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,SAAS,CAAC;AACxB,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC"}

package/dist/adapters/registry.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Adapter registry for dynamic adapter management
+ */
+import type { AdapterConfig, ModelClient, ProviderType } from './types';
+type AdapterFactory = (config: AdapterConfig) => Promise<ModelClient>;
+interface UnavailableInfo {
+    provider: ProviderType;
+    reason: string;
+}
+/**
+ * Registry for adapter factories
+ */
+declare class AdapterRegistry {
+    private adapters;
+    private unavailableProviders;
+    /**
+     * Register an adapter factory for a provider type
+     */
+    register(provider: ProviderType, factory: AdapterFactory): void;
+    /**
+     * Mark a provider as unavailable (post-MVP)
+     */
+    markUnavailable(provider: ProviderType, reason: string): void;
+    /**
+     * Get an adapter factory by provider type
+     */
+    get(provider: ProviderType): AdapterFactory | undefined;
+    /**
+     * Check if a provider is registered
+     */
+    has(provider: ProviderType): boolean;
+    /**
+     * Check if a provider is marked as unavailable
+     */
+    isUnavailable(provider: ProviderType): boolean;
+    /**
+     * Get the reason why a provider is unavailable
+     */
+    getUnavailableReason(provider: ProviderType): string | undefined;
+    /**
+     * List all registered providers
+     */
+    list(): ProviderType[];
+    /**
+     * List unavailable providers with reasons
+     */
+    listUnavailable(): UnavailableInfo[];
+}
+export declare const adapterRegistry: AdapterRegistry;
+/**
+ * Register built-in adapters
+ * Adapters are loaded lazily when first requested
+ */
+export declare function registerBuiltInAdapters(): Promise<void>;
+export {};
+//# sourceMappingURL=registry.d.ts.map

package/dist/adapters/registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../src/adapters/registry.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAExE,KAAK,cAAc,GAAG,CAAC,MAAM,EAAE,aAAa,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC;AAEtE,UAAU,eAAe;IACvB,QAAQ,EAAE,YAAY,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,cAAM,eAAe;IACnB,OAAO,CAAC,QAAQ,CAAgD;IAChE,OAAO,CAAC,oBAAoB,CAAwC;IAEpE;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,cAAc,GAAG,IAAI;IAK/D;;OAEG;IACH,eAAe,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAI7D;;OAEG;IACH,GAAG,CAAC,QAAQ,EAAE,YAAY,GAAG,cAAc,GAAG,SAAS;IAIvD;;OAEG;IACH,GAAG,CAAC,QAAQ,EAAE,YAAY,GAAG,OAAO;IAIpC;;OAEG;IACH,aAAa,CAAC,QAAQ,EAAE,YAAY,GAAG,OAAO;IAI9C;;OAEG;IACH,oBAAoB,CAAC,QAAQ,EAAE,YAAY,GAAG,MAAM,GAAG,SAAS;IAIhE;;OAEG;IACH,IAAI,IAAI,YAAY,EAAE;IAItB;;OAEG;IACH,eAAe,IAAI,eAAe,EAAE;CAMrC;AAED,eAAO,MAAM,eAAe,iBAAwB,CAAC;AAgBrD;;;GAGG;AACH,wBAAsB,uBAAuB,IAAI,OAAO,CAAC,IAAI,CAAC,CAyC7D"}

package/dist/adapters/types.d.ts

@@ -0,0 +1,151 @@
+/**
+ * Core types and interfaces for model adapters
+ */
+/**
+ * Chat message format compatible with OpenAI/Anthropic
+ */
+export interface ChatMessage {
+    role: 'system' | 'user' | 'assistant' | 'function' | 'tool';
+    content: string;
+    name?: string;
+    function_call?: {
+        name: string;
+        arguments: string;
+    };
+    tool_calls?: ToolCall[];
+}
+export interface ToolCall {
+    id: string;
+    type: 'function';
+    function: {
+        name: string;
+        arguments: string;
+    };
+}
+/**
+ * Options for generating a completion
+ */
+export interface GenerateOptions {
+    prompt: string | ChatMessage[];
+    model?: string;
+    maxTokens?: number;
+    temperature?: number;
+    topP?: number;
+    seed?: number;
+    stop?: string[];
+    functions?: FunctionDefinition[];
+    tools?: ToolDefinition[];
+    responseFormat?: {
+        type: 'text' | 'json_object';
+    };
+    metadata?: Record<string, unknown>;
+}
+/**
+ * Function/Tool definitions for function calling
+ */
+export interface FunctionDefinition {
+    name: string;
+    description?: string;
+    parameters: Record<string, unknown>;
+}
+export interface ToolDefinition {
+    type: 'function';
+    function: FunctionDefinition;
+}
+/**
+ * Token usage information
+ */
+export interface TokenUsage {
+    prompt: number;
+    completion: number;
+    total: number;
+}
+/**
+ * Result from a generation request
+ */
+export interface GenerateResult {
+    id: string;
+    model: string;
+    text: string;
+    tokens: TokenUsage;
+    latencyMs: number;
+    finishReason?: 'stop' | 'length' | 'function_call' | 'tool_calls' | 'content_filter';
+    functionCall?: {
+        name: string;
+        arguments: string;
+    };
+    toolCalls?: ToolCall[];
+    raw?: unknown;
+}
+/**
+ * Model capabilities
+ */
+export interface ModelCapabilities {
+    streaming: boolean;
+    functionCalling: boolean;
+    toolUse: boolean;
+    maxContext: number;
+    vision?: boolean;
+    jsonMode?: boolean;
+}
+/**
+ * ModelClient interface - All adapters must implement this
+ */
+export interface ModelClient {
+    readonly provider: string;
+    generate(options: GenerateOptions): Promise<GenerateResult>;
+    stream?(options: GenerateOptions, onChunk: (chunk: string) => void): AsyncIterable<string>;
+    embed?(text: string): Promise<number[]>;
+    capabilities(): Promise<ModelCapabilities>;
+    close?(): Promise<void>;
+}
+/**
+ * Provider types - all supported providers
+ */
+export type ProviderType = 'openai' | 'azure-openai' | 'vercel-ai' | 'anthropic' | 'google' | 'mistral' | 'cohere' | 'huggingface' | 'ollama' | 'custom';
+/**
+ * Base adapter configuration
+ */
+export interface BaseAdapterConfig {
+    provider: ProviderType;
+    apiKey?: string;
+    baseUrl?: string;
+    defaultModel?: string;
+    timeout?: number;
+    maxRetries?: number;
+}
+/**
+ * OpenAI-specific configuration
+ */
+export interface OpenAIAdapterConfig extends BaseAdapterConfig {
+    provider: 'openai';
+    organization?: string;
+}
+/**
+ * Azure OpenAI-specific configuration
+ */
+export interface AzureOpenAIAdapterConfig extends BaseAdapterConfig {
+    provider: 'azure-openai';
+    resourceName: string;
+    deploymentName: string;
+    apiVersion: string;
+}
+/**
+ * Vercel AI SDK configuration
+ */
+export interface VercelAIAdapterConfig extends BaseAdapterConfig {
+    provider: 'vercel-ai';
+    underlyingProvider: 'openai' | 'azure' | 'anthropic' | 'google' | 'mistral';
+    providerConfig?: Record<string, unknown>;
+}
+/**
+ * Anthropic-specific configuration (Post-MVP)
+ */
+export interface AnthropicAdapterConfig extends BaseAdapterConfig {
+    provider: 'anthropic';
+}
+/**
+ * Union type for all adapter configs
+ */
+export type AdapterConfig = OpenAIAdapterConfig | AzureOpenAIAdapterConfig | VercelAIAdapterConfig | AnthropicAdapterConfig | BaseAdapterConfig;
+//# sourceMappingURL=types.d.ts.map

package/dist/adapters/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/adapters/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;IAC5D,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE;QACd,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,GAAG,WAAW,EAAE,CAAC;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,kBAAkB,EAAE,CAAC;IACjC,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,cAAc,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,aAAa,CAAA;KAAE,CAAC;IAClD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,kBAAkB,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,eAAe,GAAG,YAAY,GAAG,gBAAgB,CAAC;IACrF,YAAY,CAAC,EAAE;QACb,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvB,GAAG,CAAC,EAAE,OAAO,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,OAAO,CAAC;IACnB,eAAe,EAAE,OAAO,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAE5D,MAAM,CAAC,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAE3F,KAAK,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAExC,YAAY,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAE3C,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GACpB,QAAQ,GACR,cAAc,GACd,WAAW,GACX,WAAW,GACX,QAAQ,GACR,SAAS,GACT,QAAQ,GACR,aAAa,GACb,QAAQ,GACR,QAAQ,CAAC;AAEb;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,YAAY,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,iBAAiB;IAC5D,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAyB,SAAQ,iBAAiB;IACjE,QAAQ,EAAE,cAAc,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAsB,SAAQ,iBAAiB;IAC9D,QAAQ,EAAE,WAAW,CAAC;IACtB,kBAAkB,EAAE,QAAQ,GAAG,OAAO,GAAG,WAAW,GAAG,QAAQ,GAAG,SAAS,CAAC;IAC5E,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC1C;AAED;;GAEG;AACH,MAAM,WAAW,sBAAuB,SAAQ,iBAAiB;IAC/D,QAAQ,EAAE,WAAW,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GACrB,mBAAmB,GACnB,wBAAwB,GACxB,qBAAqB,GACrB,sBAAsB,GACtB,iBAAiB,CAAC"}

package/dist/artifacts/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Artifacts module exports
+ */
+export * from './types';
+export { createRunManifest } from './manifest';
+//# sourceMappingURL=index.d.ts.map

package/dist/artifacts/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/artifacts/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,SAAS,CAAC;AACxB,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC"}

package/dist/artifacts/manifest.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Run manifest generation utilities
+ */
+import type { CaseResult, ManifestRedactionInfo, ResolvedConfig, RunConfig, RunManifest } from './types';
+/**
+ * Create a new run manifest
+ */
+export declare function createRunManifest(options: {
+    project: string;
+    config: RunConfig;
+    resolvedConfig?: ResolvedConfig;
+    cases: CaseResult[];
+    startTime: Date;
+    endTime: Date;
+    runBy?: string;
+    runReason?: string;
+    redaction?: ManifestRedactionInfo;
+}): RunManifest;
+//# sourceMappingURL=manifest.d.ts.map

package/dist/artifacts/manifest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../src/artifacts/manifest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,KAAK,EACV,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,SAAS,EACT,WAAW,EAEZ,MAAM,SAAS,CAAC;AAEjB;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE;IACzC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,SAAS,CAAC;IAClB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,OAAO,EAAE,IAAI,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC,GAAG,WAAW,CAqCd"}