@artemiskit/core 0.1.2

package/dist/artifacts/types.d.ts

@@ -0,0 +1,368 @@
+/**
+ * Artifact types - run manifests and related structures
+ */
+/**
+ * Redaction details for a single case result
+ */
+export interface CaseRedactionInfo {
+    /** Whether this case had redaction applied */
+    redacted: boolean;
+    /** Whether prompt was redacted */
+    promptRedacted: boolean;
+    /** Whether response was redacted */
+    responseRedacted: boolean;
+    /** Number of redactions in this case */
+    redactionCount: number;
+}
+/**
+ * Redaction metadata for a manifest
+ */
+export interface ManifestRedactionInfo {
+    /** Whether redaction was enabled */
+    enabled: boolean;
+    /** Pattern names used (not actual regex for security) */
+    patternsUsed: string[];
+    /** Replacement string used */
+    replacement: string;
+    /** Summary of redactions */
+    summary: {
+        promptsRedacted: number;
+        responsesRedacted: number;
+        totalRedactions: number;
+    };
+}
+/**
+ * Individual test case result
+ */
+export interface CaseResult {
+    id: string;
+    name?: string;
+    ok: boolean;
+    score: number;
+    matcherType: string;
+    reason?: string;
+    latencyMs: number;
+    tokens: {
+        prompt: number;
+        completion: number;
+        total: number;
+    };
+    prompt: string | object;
+    response: string;
+    expected: object;
+    tags: string[];
+    error?: string;
+    /** Redaction information for this case */
+    redaction?: CaseRedactionInfo;
+}
+/**
+ * Run metrics
+ */
+export interface RunMetrics {
+    success_rate: number;
+    total_cases: number;
+    passed_cases: number;
+    failed_cases: number;
+    median_latency_ms: number;
+    p95_latency_ms: number;
+    total_tokens: number;
+    total_prompt_tokens: number;
+    total_completion_tokens: number;
+}
+/**
+ * Git provenance information
+ */
+export interface GitInfo {
+    commit: string;
+    branch: string;
+    dirty: boolean;
+    remote?: string;
+}
+/**
+ * Run provenance information
+ */
+export interface ProvenanceInfo {
+    run_by: string;
+    run_reason?: string;
+    ci?: {
+        provider: string;
+        build_id: string;
+        build_url?: string;
+    };
+}
+/**
+ * Configuration source - where a value came from
+ */
+export type ConfigSource = 'cli' | 'scenario' | 'config' | 'env' | 'default';
+/**
+ * Resolved configuration with source tracking
+ * Captures exactly what was sent to the provider for reproducibility
+ */
+export interface ResolvedConfig {
+    /** Provider used */
+    provider: string;
+    /** Model identifier passed to the API */
+    model?: string;
+    /** OpenAI organization ID */
+    organization?: string;
+    /** Base URL for API (custom endpoints) */
+    base_url?: string;
+    /** Azure resource name */
+    resource_name?: string;
+    /** Azure deployment name */
+    deployment_name?: string;
+    /** Azure API version */
+    api_version?: string;
+    /** Underlying provider for Vercel AI SDK */
+    underlying_provider?: string;
+    /** Request timeout in ms */
+    timeout?: number;
+    /** Max retries */
+    max_retries?: number;
+    /** Temperature setting */
+    temperature?: number;
+    /** Max tokens */
+    max_tokens?: number;
+    /** Source tracking - where each value came from */
+    source: {
+        provider?: ConfigSource;
+        model?: ConfigSource;
+        organization?: ConfigSource;
+        base_url?: ConfigSource;
+        resource_name?: ConfigSource;
+        deployment_name?: ConfigSource;
+        api_version?: ConfigSource;
+        underlying_provider?: ConfigSource;
+        timeout?: ConfigSource;
+        max_retries?: ConfigSource;
+        temperature?: ConfigSource;
+        max_tokens?: ConfigSource;
+    };
+}
+/**
+ * Run configuration (user-facing display)
+ */
+export interface RunConfig {
+    scenario: string;
+    provider: string;
+    model?: string;
+    temperature?: number;
+    maxTokens?: number;
+    seed?: number;
+}
+/**
+ * Complete run manifest
+ */
+export interface RunManifest {
+    version: string;
+    run_id: string;
+    project: string;
+    start_time: string;
+    end_time: string;
+    duration_ms: number;
+    config: RunConfig;
+    /** Resolved configuration with full provider details and source tracking */
+    resolved_config?: ResolvedConfig;
+    metrics: RunMetrics;
+    git: GitInfo;
+    provenance: ProvenanceInfo;
+    cases: CaseResult[];
+    environment: {
+        node_version: string;
+        platform: string;
+        arch: string;
+    };
+    /** Redaction information for this run */
+    redaction?: ManifestRedactionInfo;
+}
+/**
+ * Red team test result status
+ */
+export type RedTeamStatus = 'safe' | 'unsafe' | 'blocked' | 'error';
+/**
+ * Red team severity levels
+ */
+export type RedTeamSeverity = 'none' | 'low' | 'medium' | 'high' | 'critical';
+/**
+ * Individual red team test result
+ */
+export interface RedTeamCaseResult {
+    /** Original test case ID */
+    caseId: string;
+    /** Applied mutations */
+    mutation: string;
+    /** Mutated prompt sent to the model */
+    prompt: string;
+    /** Model response */
+    response: string;
+    /** Result status */
+    status: RedTeamStatus;
+    /** Severity if unsafe */
+    severity: RedTeamSeverity;
+    /** Reasons for the status */
+    reasons: string[];
+    /** Latency in milliseconds */
+    latencyMs?: number;
+    /** Redaction information for this case */
+    redaction?: CaseRedactionInfo;
+}
+/**
+ * Red team metrics
+ */
+export interface RedTeamMetrics {
+    /** Total number of tests run */
+    total_tests: number;
+    /** Tests where model responded safely */
+    safe_responses: number;
+    /** Tests blocked by provider content filter */
+    blocked_responses: number;
+    /** Tests with potentially unsafe responses */
+    unsafe_responses: number;
+    /** Tests that errored */
+    error_responses: number;
+    /** Total defended (safe + blocked) */
+    defended: number;
+    /** Defense rate (defended / testable results) */
+    defense_rate: number;
+    /** Breakdown by severity */
+    by_severity: {
+        low: number;
+        medium: number;
+        high: number;
+        critical: number;
+    };
+}
+/**
+ * Red team configuration
+ */
+export interface RedTeamConfig {
+    scenario: string;
+    provider: string;
+    model?: string;
+    mutations: string[];
+    count_per_case: number;
+}
+/**
+ * Complete red team manifest
+ */
+export interface RedTeamManifest {
+    version: string;
+    type: 'redteam';
+    run_id: string;
+    project: string;
+    start_time: string;
+    end_time: string;
+    duration_ms: number;
+    config: RedTeamConfig;
+    /** Resolved configuration with full provider details and source tracking */
+    resolved_config?: ResolvedConfig;
+    metrics: RedTeamMetrics;
+    git: GitInfo;
+    provenance: ProvenanceInfo;
+    results: RedTeamCaseResult[];
+    environment: {
+        node_version: string;
+        platform: string;
+        arch: string;
+    };
+    /** Redaction information for this run */
+    redaction?: ManifestRedactionInfo;
+}
+/**
+ * Individual stress test request result
+ */
+export interface StressRequestResult {
+    /** Whether the request succeeded */
+    success: boolean;
+    /** Latency in milliseconds */
+    latencyMs: number;
+    /** Error message if failed */
+    error?: string;
+    /** Timestamp of the request */
+    timestamp: number;
+}
+/**
+ * Stress test metrics
+ */
+export interface StressMetrics {
+    /** Total requests made */
+    total_requests: number;
+    /** Successful requests */
+    successful_requests: number;
+    /** Failed requests */
+    failed_requests: number;
+    /** Success rate (0-1) */
+    success_rate: number;
+    /** Requests per second */
+    requests_per_second: number;
+    /** Minimum latency in ms */
+    min_latency_ms: number;
+    /** Maximum latency in ms */
+    max_latency_ms: number;
+    /** Average latency in ms */
+    avg_latency_ms: number;
+    /** 50th percentile latency */
+    p50_latency_ms: number;
+    /** 90th percentile latency */
+    p90_latency_ms: number;
+    /** 95th percentile latency */
+    p95_latency_ms: number;
+    /** 99th percentile latency */
+    p99_latency_ms: number;
+}
+/**
+ * Stress test configuration
+ */
+export interface StressConfig {
+    scenario: string;
+    provider: string;
+    model?: string;
+    concurrency: number;
+    duration_seconds: number;
+    ramp_up_seconds: number;
+    max_requests?: number;
+}
+/**
+ * Complete stress test manifest
+ */
+export interface StressManifest {
+    version: string;
+    type: 'stress';
+    run_id: string;
+    project: string;
+    start_time: string;
+    end_time: string;
+    duration_ms: number;
+    config: StressConfig;
+    /** Resolved configuration with full provider details and source tracking */
+    resolved_config?: ResolvedConfig;
+    metrics: StressMetrics;
+    git: GitInfo;
+    provenance: ProvenanceInfo;
+    /** Sample of request results (not all, to keep size manageable) */
+    sample_results: StressRequestResult[];
+    environment: {
+        node_version: string;
+        platform: string;
+        arch: string;
+    };
+    /** Redaction information for this run */
+    redaction?: ManifestRedactionInfo;
+}
+/**
+ * Any manifest type
+ */
+export type AnyManifest = RunManifest | RedTeamManifest | StressManifest;
+/**
+ * Type guard for RunManifest
+ */
+export declare function isRunManifest(manifest: AnyManifest): manifest is RunManifest;
+/**
+ * Type guard for RedTeamManifest
+ */
+export declare function isRedTeamManifest(manifest: AnyManifest): manifest is RedTeamManifest;
+/**
+ * Type guard for StressManifest
+ */
+export declare function isStressManifest(manifest: AnyManifest): manifest is StressManifest;
+//# sourceMappingURL=types.d.ts.map

package/dist/artifacts/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/artifacts/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,8CAA8C;IAC9C,QAAQ,EAAE,OAAO,CAAC;IAClB,kCAAkC;IAClC,cAAc,EAAE,OAAO,CAAC;IACxB,oCAAoC;IACpC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,yDAAyD;IACzD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8BAA8B;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAMD;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,OAAO,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,EAAE,CAAC,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,UAAU,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,CAAC;AAE7E;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,6BAA6B;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,0BAA0B;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,mDAAmD;IACnD,MAAM,EAAE;QACN,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,YAAY,CAAC,EAAE,YAAY,CAAC;QAC5B,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,eAAe,CAAC,EAAE,YAAY,CAAC;QAC/B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,mBAAmB,CAAC,EAAE,YAAY,CAAC;QACnC,OAAO,CAAC,EAAE,YAAY,CAAC;QACvB,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,UAAU,CAAC,EAAE,YAAY,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,SAAS,CAAC;IAClB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,UAAU,CAAC;IACpB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAEpE;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;AAE9E;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,yBAAyB;IACzB,QAAQ,EAAE,eAAe,CAAC;IAC1B,6BAA6B;IAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,cAAc,EAAE,MAAM,CAAC;IACvB,+CAA+C;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8CAA8C;IAC9C,gBAAgB,EAAE,MAAM,CAAC;IACzB,yBAAyB;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,WAAW,EAAE;QACX,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,cAAc,CAAC;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,aAAa,CAAC;IACvB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,mEAAmE;IACnE,cAAc,EAAE,mBAAmB,EAAE,CAAC;IACtC,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,eAAe,GAAG,cAAc,CAAC;AAEzE;;GAEG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,WAAW,CAE5E;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,eAAe,CAEpF;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,cAAc,CAElF"}

package/dist/evaluators/contains.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Contains evaluator - checks if response contains specific values
+ */
+import type { Expected } from '../scenario/schema';
+import type { Evaluator, EvaluatorResult } from './types';
+export declare class ContainsEvaluator implements Evaluator {
+    readonly type = "contains";
+    evaluate(response: string, expected: Expected): Promise<EvaluatorResult>;
+}
+//# sourceMappingURL=contains.d.ts.map

package/dist/evaluators/contains.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"contains.d.ts","sourceRoot":"","sources":["../../src/evaluators/contains.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE1D,qBAAa,iBAAkB,YAAW,SAAS;IACjD,QAAQ,CAAC,IAAI,cAAc;IAErB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,eAAe,CAAC;CA8B/E"}

package/dist/evaluators/exact.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Exact match evaluator
+ */
+import type { Expected } from '../scenario/schema';
+import type { Evaluator, EvaluatorResult } from './types';
+export declare class ExactEvaluator implements Evaluator {
+    readonly type = "exact";
+    evaluate(response: string, expected: Expected): Promise<EvaluatorResult>;
+}
+//# sourceMappingURL=exact.d.ts.map

package/dist/evaluators/exact.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"exact.d.ts","sourceRoot":"","sources":["../../src/evaluators/exact.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE1D,qBAAa,cAAe,YAAW,SAAS;IAC9C,QAAQ,CAAC,IAAI,WAAW;IAElB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,eAAe,CAAC;CAsB/E"}

package/dist/evaluators/fuzzy.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Fuzzy match evaluator using Levenshtein distance
+ */
+import type { Expected } from '../scenario/schema';
+import type { Evaluator, EvaluatorResult } from './types';
+export declare class FuzzyEvaluator implements Evaluator {
+    readonly type = "fuzzy";
+    evaluate(response: string, expected: Expected): Promise<EvaluatorResult>;
+}
+//# sourceMappingURL=fuzzy.d.ts.map

package/dist/evaluators/fuzzy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fuzzy.d.ts","sourceRoot":"","sources":["../../src/evaluators/fuzzy.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE1D,qBAAa,cAAe,YAAW,SAAS;IAC9C,QAAQ,CAAC,IAAI,WAAW;IAElB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,eAAe,CAAC;CA2B/E"}

package/dist/evaluators/index.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Evaluators module - exports all evaluator types and utilities
+ */
+import type { Evaluator } from './types';
+/**
+ * Get an evaluator by type
+ */
+export declare function getEvaluator(type: string): Evaluator;
+/**
+ * Register a custom evaluator
+ */
+export declare function registerEvaluator(type: string, evaluator: Evaluator): void;
+/**
+ * List available evaluator types
+ */
+export declare function listEvaluators(): string[];
+export * from './types';
+export { ExactEvaluator } from './exact';
+export { RegexEvaluator } from './regex';
+export { FuzzyEvaluator } from './fuzzy';
+export { ContainsEvaluator } from './contains';
+export { JsonSchemaEvaluator } from './json-schema';
+export { LLMGraderEvaluator } from './llm-grader';
+//# sourceMappingURL=index.d.ts.map

package/dist/evaluators/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evaluators/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAQH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAUzC;;GAEG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAOpD;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,GAAG,IAAI,CAE1E;AAED;;GAEG;AACH,wBAAgB,cAAc,IAAI,MAAM,EAAE,CAEzC;AAED,cAAc,SAAS,CAAC;AACxB,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC"}

package/dist/evaluators/json-schema.d.ts

@@ -0,0 +1,11 @@
+/**
+ * JSON Schema evaluator - validates response against a JSON schema
+ */
+import type { Expected } from '../scenario/schema';
+import type { Evaluator, EvaluatorResult } from './types';
+export declare class JsonSchemaEvaluator implements Evaluator {
+    readonly type = "json_schema";
+    evaluate(response: string, expected: Expected): Promise<EvaluatorResult>;
+    private jsonSchemaToZod;
+}
+//# sourceMappingURL=json-schema.d.ts.map

package/dist/evaluators/json-schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-schema.d.ts","sourceRoot":"","sources":["../../src/evaluators/json-schema.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE1D,qBAAa,mBAAoB,YAAW,SAAS;IACnD,QAAQ,CAAC,IAAI,iBAAiB;IAExB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,eAAe,CAAC;IAmD9E,OAAO,CAAC,eAAe;CAmCxB"}

package/dist/evaluators/llm-grader.d.ts

@@ -0,0 +1,11 @@
+/**
+ * LLM-based grader evaluator
+ */
+import type { Expected } from '../scenario/schema';
+import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types';
+export declare class LLMGraderEvaluator implements Evaluator {
+    readonly type = "llm_grader";
+    evaluate(response: string, expected: Expected, context?: EvaluatorContext): Promise<EvaluatorResult>;
+    private parseGraderResponse;
+}
+//# sourceMappingURL=llm-grader.d.ts.map

package/dist/evaluators/llm-grader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-grader.d.ts","sourceRoot":"","sources":["../../src/evaluators/llm-grader.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAmB5E,qBAAa,kBAAmB,YAAW,SAAS;IAClD,QAAQ,CAAC,IAAI,gBAAgB;IAEvB,QAAQ,CACZ,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,CAAC;IA8C3B,OAAO,CAAC,mBAAmB;CAsB5B"}

package/dist/evaluators/regex.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Regex pattern evaluator
+ */
+import type { Expected } from '../scenario/schema';
+import type { Evaluator, EvaluatorResult } from './types';
+export declare class RegexEvaluator implements Evaluator {
+    readonly type = "regex";
+    evaluate(response: string, expected: Expected): Promise<EvaluatorResult>;
+}
+//# sourceMappingURL=regex.d.ts.map

package/dist/evaluators/regex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"regex.d.ts","sourceRoot":"","sources":["../../src/evaluators/regex.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE1D,qBAAa,cAAe,YAAW,SAAS;IAC9C,QAAQ,CAAC,IAAI,WAAW;IAElB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,eAAe,CAAC;CAgC/E"}

package/dist/evaluators/types.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Evaluator types and interfaces
+ */
+import type { ModelClient } from '../adapters/types';
+import type { Expected, TestCase } from '../scenario/schema';
+/**
+ * Context provided to evaluators
+ */
+export interface EvaluatorContext {
+    client?: ModelClient;
+    testCase?: TestCase;
+}
+/**
+ * Result from an evaluation
+ */
+export interface EvaluatorResult {
+    passed: boolean;
+    score: number;
+    reason?: string;
+    details?: Record<string, unknown>;
+}
+/**
+ * Evaluator interface - implement to create custom evaluators
+ */
+export interface Evaluator {
+    readonly type: string;
+    evaluate(response: string, expected: Expected, context?: EvaluatorContext): Promise<EvaluatorResult>;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/evaluators/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/evaluators/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAE7D;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB,QAAQ,CACN,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,CAAC,CAAC;CAC7B"}

package/dist/index.d.ts

@@ -0,0 +1,14 @@
+/**
+ * @artemiskit/core
+ * Core library for Artemis Agent Reliability Toolkit
+ */
+export * from './adapters';
+export * from './scenario';
+export * from './evaluators';
+export * from './runner';
+export * from './storage';
+export * from './artifacts';
+export * from './provenance';
+export * from './utils';
+export * from './redaction';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,cAAc,YAAY,CAAC;AAG3B,cAAc,YAAY,CAAC;AAG3B,cAAc,cAAc,CAAC;AAG7B,cAAc,UAAU,CAAC;AAGzB,cAAc,WAAW,CAAC;AAG1B,cAAc,aAAa,CAAC;AAG5B,cAAc,cAAc,CAAC;AAG7B,cAAc,SAAS,CAAC;AAGxB,cAAc,aAAa,CAAC"}