@artemiskit/core 0.1.2

package/dist/scenario/schema.d.ts

@@ -0,0 +1,945 @@
+/**
+ * Scenario schema definitions using Zod
+ */
+import { z } from 'zod';
+/**
+ * Provider schema - supports all providers
+ */
+export declare const ProviderSchema: z.ZodEnum<["openai", "azure-openai", "vercel-ai", "anthropic", "google", "mistral", "cohere", "huggingface", "ollama", "custom"]>;
+/**
+ * Provider config schema - optional overrides for provider settings
+ * Supports ${ENV_VAR} and ${ENV_VAR:-default} syntax for values
+ * All fields are optional - only specified fields override defaults
+ */
+export declare const ProviderConfigSchema: z.ZodOptional<z.ZodObject<{
+    apiKey: z.ZodOptional<z.ZodString>;
+    baseUrl: z.ZodOptional<z.ZodString>;
+    defaultModel: z.ZodOptional<z.ZodString>;
+    timeout: z.ZodOptional<z.ZodNumber>;
+    maxRetries: z.ZodOptional<z.ZodNumber>;
+    organization: z.ZodOptional<z.ZodString>;
+    resourceName: z.ZodOptional<z.ZodString>;
+    deploymentName: z.ZodOptional<z.ZodString>;
+    apiVersion: z.ZodOptional<z.ZodString>;
+    underlyingProvider: z.ZodOptional<z.ZodEnum<["openai", "azure", "anthropic", "google", "mistral"]>>;
+}, "strip", z.ZodTypeAny, {
+    apiKey?: string | undefined;
+    baseUrl?: string | undefined;
+    defaultModel?: string | undefined;
+    timeout?: number | undefined;
+    maxRetries?: number | undefined;
+    organization?: string | undefined;
+    resourceName?: string | undefined;
+    deploymentName?: string | undefined;
+    apiVersion?: string | undefined;
+    underlyingProvider?: "openai" | "anthropic" | "google" | "mistral" | "azure" | undefined;
+}, {
+    apiKey?: string | undefined;
+    baseUrl?: string | undefined;
+    defaultModel?: string | undefined;
+    timeout?: number | undefined;
+    maxRetries?: number | undefined;
+    organization?: string | undefined;
+    resourceName?: string | undefined;
+    deploymentName?: string | undefined;
+    apiVersion?: string | undefined;
+    underlyingProvider?: "openai" | "anthropic" | "google" | "mistral" | "azure" | undefined;
+}>>;
+/**
+ * Expected result types - how to evaluate responses
+ */
+export declare const ExpectedSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
+    type: z.ZodLiteral<"exact">;
+    value: z.ZodString;
+    caseSensitive: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+}, "strip", z.ZodTypeAny, {
+    value: string;
+    type: "exact";
+    caseSensitive: boolean;
+}, {
+    value: string;
+    type: "exact";
+    caseSensitive?: boolean | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"regex">;
+    pattern: z.ZodString;
+    flags: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    type: "regex";
+    pattern: string;
+    flags?: string | undefined;
+}, {
+    type: "regex";
+    pattern: string;
+    flags?: string | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"fuzzy">;
+    value: z.ZodString;
+    threshold: z.ZodDefault<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    value: string;
+    type: "fuzzy";
+    threshold: number;
+}, {
+    value: string;
+    type: "fuzzy";
+    threshold?: number | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"llm_grader">;
+    rubric: z.ZodString;
+    model: z.ZodOptional<z.ZodString>;
+    provider: z.ZodOptional<z.ZodEnum<["openai", "azure-openai", "vercel-ai", "anthropic", "google", "mistral", "cohere", "huggingface", "ollama", "custom"]>>;
+    threshold: z.ZodDefault<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    type: "llm_grader";
+    threshold: number;
+    rubric: string;
+    provider?: "openai" | "azure-openai" | "vercel-ai" | "anthropic" | "google" | "mistral" | "cohere" | "huggingface" | "ollama" | "custom" | undefined;
+    model?: string | undefined;
+}, {
+    type: "llm_grader";
+    rubric: string;
+    provider?: "openai" | "azure-openai" | "vercel-ai" | "anthropic" | "google" | "mistral" | "cohere" | "huggingface" | "ollama" | "custom" | undefined;
+    threshold?: number | undefined;
+    model?: string | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"contains">;
+    values: z.ZodArray<z.ZodString, "many">;
+    mode: z.ZodDefault<z.ZodEnum<["all", "any"]>>;
+}, "strip", z.ZodTypeAny, {
+    values: string[];
+    type: "contains";
+    mode: "all" | "any";
+}, {
+    values: string[];
+    type: "contains";
+    mode?: "all" | "any" | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"json_schema">;
+    schema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+}, "strip", z.ZodTypeAny, {
+    type: "json_schema";
+    schema: Record<string, unknown>;
+}, {
+    type: "json_schema";
+    schema: Record<string, unknown>;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"custom">;
+    evaluator: z.ZodString;
+    config: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, "strip", z.ZodTypeAny, {
+    type: "custom";
+    evaluator: string;
+    config?: Record<string, unknown> | undefined;
+}, {
+    type: "custom";
+    evaluator: string;
+    config?: Record<string, unknown> | undefined;
+}>]>;
+/**
+ * Chat message schema
+ */
+export declare const ChatMessageSchema: z.ZodObject<{
+    role: z.ZodEnum<["system", "user", "assistant"]>;
+    content: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    role: "system" | "user" | "assistant";
+    content: string;
+}, {
+    role: "system" | "user" | "assistant";
+    content: string;
+}>;
+/**
+ * Variables schema - key-value pairs for template substitution
+ */
+export declare const VariablesSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+/**
+ * Redaction configuration schema for scenario-level settings
+ * Re-exported from redaction module, made optional for scenario context
+ */
+declare const RedactionConfigSchema: z.ZodOptional<z.ZodObject<{
+    enabled: z.ZodDefault<z.ZodBoolean>;
+    patterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    redactPrompts: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+    redactResponses: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+    redactMetadata: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+    replacement: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+}, "strip", z.ZodTypeAny, {
+    enabled: boolean;
+    redactPrompts: boolean;
+    redactResponses: boolean;
+    redactMetadata: boolean;
+    replacement: string;
+    patterns?: string[] | undefined;
+}, {
+    enabled?: boolean | undefined;
+    patterns?: string[] | undefined;
+    redactPrompts?: boolean | undefined;
+    redactResponses?: boolean | undefined;
+    redactMetadata?: boolean | undefined;
+    replacement?: string | undefined;
+}>>;
+/**
+ * Test case schema
+ */
+export declare const TestCaseSchema: z.ZodObject<{
+    id: z.ZodString;
+    name: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+    prompt: z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodObject<{
+        role: z.ZodEnum<["system", "user", "assistant"]>;
+        content: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        role: "system" | "user" | "assistant";
+        content: string;
+    }, {
+        role: "system" | "user" | "assistant";
+        content: string;
+    }>, "many">]>;
+    expected: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
+        type: z.ZodLiteral<"exact">;
+        value: z.ZodString;
+        caseSensitive: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+    }, "strip", z.ZodTypeAny, {
+        value: string;
+        type: "exact";
+        caseSensitive: boolean;
+    }, {
+        value: string;
+        type: "exact";
+        caseSensitive?: boolean | undefined;
+    }>, z.ZodObject<{
+        type: z.ZodLiteral<"regex">;
+        pattern: z.ZodString;
+        flags: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        type: "regex";
+        pattern: string;
+        flags?: string | undefined;
+    }, {
+        type: "regex";
+        pattern: string;
+        flags?: string | undefined;
+    }>, z.ZodObject<{
+        type: z.ZodLiteral<"fuzzy">;
+        value: z.ZodString;
+        threshold: z.ZodDefault<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        value: string;
+        type: "fuzzy";
+        threshold: number;
+    }, {
+        value: string;
+        type: "fuzzy";
+        threshold?: number | undefined;
+    }>, z.ZodObject<{
+        type: z.ZodLiteral<"llm_grader">;
+        rubric: z.ZodString;
+        model: z.ZodOptional<z.ZodString>;
+        provider: z.ZodOptional<z.ZodEnum<["openai", "azure-openai", "vercel-ai", "anthropic", "google", "mistral", "cohere", "huggingface", "ollama", "custom"]>>;
+        threshold: z.ZodDefault<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        type: "llm_grader";
+        threshold: number;
+        rubric: string;
+        provider?: "openai" | "azure-openai" | "vercel-ai" | "anthropic" | "google" | "mistral" | "cohere" | "huggingface" | "ollama" | "custom" | undefined;
+        model?: string | undefined;
+    }, {
+        type: "llm_grader";
+        rubric: string;
+        provider?: "openai" | "azure-openai" | "vercel-ai" | "anthropic" | "google" | "mistral" | "cohere" | "huggingface" | "ollama" | "custom" | undefined;
+        threshold?: number | undefined;
+        model?: string | undefined;
+    }>, z.ZodObject<{
+        type: z.ZodLiteral<"contains">;
+        values: z.ZodArray<z.ZodString, "many">;
+        mode: z.ZodDefault<z.ZodEnum<["all", "any"]>>;
+    }, "strip", z.ZodTypeAny, {
+        values: string[];
+        type: "contains";
+        mode: "all" | "any";
+    }, {
+        values: string[];
+        type: "contains";
+        mode?: "all" | "any" | undefined;
+    }>, z.ZodObject<{
+        type: z.ZodLiteral<"json_schema">;
+        schema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    }, "strip", z.ZodTypeAny, {
+        type: "json_schema";
+        schema: Record<string, unknown>;
+    }, {
+        type: "json_schema";
+        schema: Record<string, unknown>;
+    }>, z.ZodObject<{
+        type: z.ZodLiteral<"custom">;
+        evaluator: z.ZodString;
+        config: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, "strip", z.ZodTypeAny, {
+        type: "custom";
+        evaluator: string;
+        config?: Record<string, unknown> | undefined;
+    }, {
+        type: "custom";
+        evaluator: string;
+        config?: Record<string, unknown> | undefined;
+    }>]>;
+    tags: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString, "many">>>;
+    metadata: z.ZodDefault<z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+    timeout: z.ZodOptional<z.ZodNumber>;
+    retries: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+    provider: z.ZodOptional<z.ZodEnum<["openai", "azure-openai", "vercel-ai", "anthropic", "google", "mistral", "cohere", "huggingface", "ollama", "custom"]>>;
+    model: z.ZodOptional<z.ZodString>;
+    variables: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodString, z.ZodNumber, z.ZodBoolean]>>>;
+    /** Case-level redaction config (overrides scenario-level) */
+    redaction: z.ZodOptional<z.ZodObject<{
+        enabled: z.ZodDefault<z.ZodBoolean>;
+        patterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        redactPrompts: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+        redactResponses: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+        redactMetadata: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+        replacement: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+    }, "strip", z.ZodTypeAny, {
+        enabled: boolean;
+        redactPrompts: boolean;
+        redactResponses: boolean;
+        redactMetadata: boolean;
+        replacement: string;
+        patterns?: string[] | undefined;
+    }, {
+        enabled?: boolean | undefined;
+        patterns?: string[] | undefined;
+        redactPrompts?: boolean | undefined;
+        redactResponses?: boolean | undefined;
+        redactMetadata?: boolean | undefined;
+        replacement?: string | undefined;
+    }>>;
+}, "strip", z.ZodTypeAny, {
+    expected: {
+        value: string;
+        type: "exact";
+        caseSensitive: boolean;
+    } | {
+        type: "regex";
+        pattern: string;
+        flags?: string | undefined;
+    } | {
+        value: string;
+        type: "fuzzy";
+        threshold: number;
+    } | {
+        type: "llm_grader";
+        threshold: number;
+        rubric: string;
+        provider?: "openai" | "azure-openai" | "vercel-ai" | "anthropic" | "google" | "mistral" | "cohere" | "huggingface" | "ollama" | "custom" | undefined;
+        model?: string | undefined;
+    } | {
+        values: string[];
+        type: "contains";
+        mode: "all" | "any";
+    } | {
+        type: "json_schema";
+        schema: Record<string, unknown>;
+    } | {
+        type: "custom";
+        evaluator: string;
+        config?: Record<string, unknown> | undefined;
+    };
+    id: string;
+    prompt: string | {
+        role: "system" | "user" | "assistant";
+        content: string;
+    }[];
+    tags: string[];
+    metadata: Record<string, unknown>;
+    retries: number;
+    name?: string | undefined;
+    provider?: "openai" | "azure-openai" | "vercel-ai" | "anthropic" | "google" | "mistral" | "cohere" | "huggingface" | "ollama" | "custom" | undefined;
+    timeout?: number | undefined;
+    model?: string | undefined;
+    description?: string | undefined;
+    variables?: Record<string, string | number | boolean> | undefined;
+    redaction?: {
+        enabled: boolean;
+        redactPrompts: boolean;
+        redactResponses: boolean;
+        redactMetadata: boolean;
+        replacement: string;
+        patterns?: string[] | undefined;
+    } | undefined;
+}, {
+    expected: {
+        value: string;
+        type: "exact";
+        caseSensitive?: boolean | undefined;
+    } | {
+        type: "regex";
+        pattern: string;
+        flags?: string | undefined;
+    } | {
+        value: string;
+        type: "fuzzy";
+        threshold?: number | undefined;
+    } | {
+        type: "llm_grader";
+        rubric: string;
+        provider?: "openai" | "azure-openai" | "vercel-ai" | "anthropic" | "google" | "mistral" | "cohere" | "huggingface" | "ollama" | "custom" | undefined;
+        threshold?: number | undefined;
+        model?: string | undefined;
+    } | {
+        values: string[];
+        type: "contains";
+        mode?: "all" | "any" | undefined;
+    } | {
+        type: "json_schema";
+        schema: Record<string, unknown>;
+    } | {
+        type: "custom";
+        evaluator: string;
+        config?: Record<string, unknown> | undefined;
+    };
+    id: string;
+    prompt: string | {
+        role: "system" | "user" | "assistant";
+        content: string;
+    }[];
+    name?: string | undefined;
+    provider?: "openai" | "azure-openai" | "vercel-ai" | "anthropic" | "google" | "mistral" | "cohere" | "huggingface" | "ollama" | "custom" | undefined;
+    timeout?: number | undefined;
+    model?: string | undefined;
+    description?: string | undefined;
+    tags?: string[] | undefined;
+    metadata?: Record<string, unknown> | undefined;
+    retries?: number | undefined;
+    variables?: Record<string, string | number | boolean> | undefined;
+    redaction?: {
+        enabled?: boolean | undefined;
+        patterns?: string[] | undefined;
+        redactPrompts?: boolean | undefined;
+        redactResponses?: boolean | undefined;
+        redactMetadata?: boolean | undefined;
+        replacement?: string | undefined;
+    } | undefined;
+}>;
+/**
+ * Scenario schema - a collection of test cases
+ */
+export declare const ScenarioSchema: z.ZodObject<{
+    name: z.ZodString;
+    description: z.ZodOptional<z.ZodString>;
+    version: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+    provider: z.ZodOptional<z.ZodEnum<["openai", "azure-openai", "vercel-ai", "anthropic", "google", "mistral", "cohere", "huggingface", "ollama", "custom"]>>;
+    model: z.ZodOptional<z.ZodString>;
+    providerConfig: z.ZodOptional<z.ZodObject<{
+        apiKey: z.ZodOptional<z.ZodString>;
+        baseUrl: z.ZodOptional<z.ZodString>;
+        defaultModel: z.ZodOptional<z.ZodString>;
+        timeout: z.ZodOptional<z.ZodNumber>;
+        maxRetries: z.ZodOptional<z.ZodNumber>;
+        organization: z.ZodOptional<z.ZodString>;
+        resourceName: z.ZodOptional<z.ZodString>;
+        deploymentName: z.ZodOptional<z.ZodString>;
+        apiVersion: z.ZodOptional<z.ZodString>;
+        underlyingProvider: z.ZodOptional<z.ZodEnum<["openai", "azure", "anthropic", "google", "mistral"]>>;
+    }, "strip", z.ZodTypeAny, {
+        apiKey?: string | undefined;
+        baseUrl?: string | undefined;
+        defaultModel?: string | undefined;
+        timeout?: number | undefined;
+        maxRetries?: number | undefined;
+        organization?: string | undefined;
+        resourceName?: string | undefined;
+        deploymentName?: string | undefined;
+        apiVersion?: string | undefined;
+        underlyingProvider?: "openai" | "anthropic" | "google" | "mistral" | "azure" | undefined;
+    }, {
+        apiKey?: string | undefined;
+        baseUrl?: string | undefined;
+        defaultModel?: string | undefined;
+        timeout?: number | undefined;
+        maxRetries?: number | undefined;
+        organization?: string | undefined;
+        resourceName?: string | undefined;
+        deploymentName?: string | undefined;
+        apiVersion?: string | undefined;
+        underlyingProvider?: "openai" | "anthropic" | "google" | "mistral" | "azure" | undefined;
+    }>>;
+    seed: z.ZodOptional<z.ZodNumber>;
+    temperature: z.ZodOptional<z.ZodNumber>;
+    maxTokens: z.ZodOptional<z.ZodNumber>;
+    tags: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString, "many">>>;
+    variables: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodString, z.ZodNumber, z.ZodBoolean]>>>;
+    /** Scenario-level redaction configuration */
+    redaction: z.ZodOptional<z.ZodObject<{
+        enabled: z.ZodDefault<z.ZodBoolean>;
+        patterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        redactPrompts: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+        redactResponses: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+        redactMetadata: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+        replacement: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+    }, "strip", z.ZodTypeAny, {
+        enabled: boolean;
+        redactPrompts: boolean;
+        redactResponses: boolean;
+        redactMetadata: boolean;
+        replacement: string;
+        patterns?: string[] | undefined;
+    }, {
+        enabled?: boolean | undefined;
+        patterns?: string[] | undefined;
+        redactPrompts?: boolean | undefined;
+        redactResponses?: boolean | undefined;
+        redactMetadata?: boolean | undefined;
+        replacement?: string | undefined;
+    }>>;
+    setup: z.ZodOptional<z.ZodObject<{
+        systemPrompt: z.ZodOptional<z.ZodString>;
+        functions: z.ZodOptional<z.ZodArray<z.ZodUnknown, "many">>;
+    }, "strip", z.ZodTypeAny, {
+        systemPrompt?: string | undefined;
+        functions?: unknown[] | undefined;
+    }, {
+        systemPrompt?: string | undefined;
+        functions?: unknown[] | undefined;
+    }>>;
+    cases: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        name: z.ZodOptional<z.ZodString>;
+        description: z.ZodOptional<z.ZodString>;
+        prompt: z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodObject<{
+            role: z.ZodEnum<["system", "user", "assistant"]>;
+            content: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            role: "system" | "user" | "assistant";
+            content: string;
+        }, {
+            role: "system" | "user" | "assistant";
+            content: string;
+        }>, "many">]>;
+        expected: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
+            type: z.ZodLiteral<"exact">;
+            value: z.ZodString;
+            caseSensitive: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+        }, "strip", z.ZodTypeAny, {
+            value: string;
+            type: "exact";
+            caseSensitive: boolean;
+        }, {
+            value: string;
+            type: "exact";
+            caseSensitive?: boolean | undefined;
+        }>, z.ZodObject<{
+            type: z.ZodLiteral<"regex">;
+            pattern: z.ZodString;
+            flags: z.ZodOptional<z.ZodString>;
+        }, "strip", z.ZodTypeAny, {
+            type: "regex";
+            pattern: string;
+            flags?: string | undefined;
+        }, {
+            type: "regex";
+            pattern: string;
+            flags?: string | undefined;
+        }>, z.ZodObject<{
+            type: z.ZodLiteral<"fuzzy">;
+            value: z.ZodString;
+            threshold: z.ZodDefault<z.ZodNumber>;
+        }, "strip", z.ZodTypeAny, {
+            value: string;
+            type: "fuzzy";
+            threshold: number;
+        }, {
+            value: string;
+            type: "fuzzy";
+            threshold?: number | undefined;
+        }>, z.ZodObject<{
+            type: z.ZodLiteral<"llm_grader">;
+            rubric: z.ZodString;
+            model: z.ZodOptional<z.ZodString>;
+            provider: z.ZodOptional<z.ZodEnum<["openai", "azure-openai", "vercel-ai", "anthropic", "google", "mistral", "cohere", "huggingface", "ollama", "custom"]>>;
+            threshold: z.ZodDefault<z.ZodNumber>;
+        }, "strip", z.ZodTypeAny, {
+            type: "llm_grader";
+            threshold: number;
+            rubric: string;
+            provider?: "openai" | "azure-openai" | "vercel-ai" | "anthropic" | "google" | "mistral" | "cohere" | "huggingface" | "ollama" | "custom" | undefined;
+            model?: string | undefined;
+        }, {
+            type: "llm_grader";
+            rubric: string;
+            provider?: "openai" | "azure-openai" | "vercel-ai" | "anthropic" | "google" | "mistral" | "cohere" | "huggingface" | "ollama" | "custom" | undefined;
+            threshold?: number | undefined;
+            model?: string | undefined;
+        }>, z.ZodObject<{
+            type: z.ZodLiteral<"contains">;
+            values: z.ZodArray<z.ZodString, "many">;
+            mode: z.ZodDefault<z.ZodEnum<["all", "any"]>>;
+        }, "strip", z.ZodTypeAny, {
+            values: string[];
+            type: "contains";
+            mode: "all" | "any";
+        }, {
+            values: string[];
+            type: "contains";
+            mode?: "all" | "any" | undefined;
+        }>, z.ZodObject<{
+            type: z.ZodLiteral<"json_schema">;
+            schema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        }, "strip", z.ZodTypeAny, {
+            type: "json_schema";
+            schema: Record<string, unknown>;
+        }, {
+            type: "json_schema";
+            schema: Record<string, unknown>;
+        }>, z.ZodObject<{
+            type: z.ZodLiteral<"custom">;
+            evaluator: z.ZodString;
+            config: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        }, "strip", z.ZodTypeAny, {
+            type: "custom";
+            evaluator: string;
+            config?: Record<string, unknown> | undefined;
+        }, {
+            type: "custom";
+            evaluator: string;
+            config?: Record<string, unknown> | undefined;
+        }>]>;
+        tags: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString, "many">>>;
+        metadata: z.ZodDefault<z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+        timeout: z.ZodOptional<z.ZodNumber>;
+        retries: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+        provider: z.ZodOptional<z.ZodEnum<["openai", "azure-openai", "vercel-ai", "anthropic", "google", "mistral", "cohere", "huggingface", "ollama", "custom"]>>;
+        model: z.ZodOptional<z.ZodString>;
+        variables: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodString, z.ZodNumber, z.ZodBoolean]>>>;
+        /** Case-level redaction config (overrides scenario-level) */
+        redaction: z.ZodOptional<z.ZodObject<{
+            enabled: z.ZodDefault<z.ZodBoolean>;
+            patterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            redactPrompts: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+            redactResponses: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+            redactMetadata: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+            replacement: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+        }, "strip", z.ZodTypeAny, {
+            enabled: boolean;
+            redactPrompts: boolean;
+            redactResponses: boolean;
+            redactMetadata: boolean;
+            replacement: string;
+            patterns?: string[] | undefined;
+        }, {
+            enabled?: boolean | undefined;
+            patterns?: string[] | undefined;
+            redactPrompts?: boolean | undefined;
+            redactResponses?: boolean | undefined;
+            redactMetadata?: boolean | undefined;
+            replacement?: string | undefined;
+        }>>;
+    }, "strip", z.ZodTypeAny, {
+        expected: {
+            value: string;
+            type: "exact";
+            caseSensitive: boolean;
+        } | {
+            type: "regex";
+            pattern: string;
+            flags?: string | undefined;
+        } | {
+            value: string;
+            type: "fuzzy";
+            threshold: number;
+        } | {
+            type: "llm_grader";
+            threshold: number;
+            rubric: string;
+            provider?: "openai" | "azure-openai" | "vercel-ai" | "anthropic" | "google" | "mistral" | "cohere" | "huggingface" | "ollama" | "custom" | undefined;
+            model?: string | undefined;
+        } | {
+            values: string[];
+            type: "contains";
+            mode: "all" | "any";
+        } | {
+            type: "json_schema";
+            schema: Record<string, unknown>;
+        } | {
+            type: "custom";
+            evaluator: string;
+            config?: Record<string, unknown> | undefined;
+        };
+        id: string;
+        prompt: string | {
+            role: "system" | "user" | "assistant";
+            content: string;
+        }[];
+        tags: string[];
+        metadata: Record<string, unknown>;
+        retries: number;
+        name?: string | undefined;
+        provider?: "openai" | "azure-openai" | "vercel-ai" | "anthropic" | "google" | "mistral" | "cohere" | "huggingface" | "ollama" | "custom" | undefined;
+        timeout?: number | undefined;
+        model?: string | undefined;
+        description?: string | undefined;
+        variables?: Record<string, string | number | boolean> | undefined;
+        redaction?: {
+            enabled: boolean;
+            redactPrompts: boolean;
+            redactResponses: boolean;
+            redactMetadata: boolean;
+            replacement: string;
+            patterns?: string[] | undefined;
+        } | undefined;
+    }, {
+        expected: {
+            value: string;
+            type: "exact";
+            caseSensitive?: boolean | undefined;
+        } | {
+            type: "regex";
+            pattern: string;
+            flags?: string | undefined;
+        } | {
+            value: string;
+            type: "fuzzy";
+            threshold?: number | undefined;
+        } | {
+            type: "llm_grader";
+            rubric: string;
+            provider?: "openai" | "azure-openai" | "vercel-ai" | "anthropic" | "google" | "mistral" | "cohere" | "huggingface" | "ollama" | "custom" | undefined;
+            threshold?: number | undefined;
+            model?: string | undefined;
+        } | {
+            values: string[];
+            type: "contains";
+            mode?: "all" | "any" | undefined;
+        } | {
+            type: "json_schema";
+            schema: Record<string, unknown>;
+        } | {
+            type: "custom";
+            evaluator: string;
+            config?: Record<string, unknown> | undefined;
+        };
+        id: string;
+        prompt: string | {
+            role: "system" | "user" | "assistant";
+            content: string;
+        }[];
+        name?: string | undefined;
+        provider?: "openai" | "azure-openai" | "vercel-ai" | "anthropic" | "google" | "mistral" | "cohere" | "huggingface" | "ollama" | "custom" | undefined;
+        timeout?: number | undefined;
+        model?: string | undefined;
+        description?: string | undefined;
+        tags?: string[] | undefined;
+        metadata?: Record<string, unknown> | undefined;
+        retries?: number | undefined;
+        variables?: Record<string, string | number | boolean> | undefined;
+        redaction?: {
+            enabled?: boolean | undefined;
+            patterns?: string[] | undefined;
+            redactPrompts?: boolean | undefined;
+            redactResponses?: boolean | undefined;
+            redactMetadata?: boolean | undefined;
+            replacement?: string | undefined;
+        } | undefined;
+    }>, "many">;
+    teardown: z.ZodOptional<z.ZodObject<{
+        cleanup: z.ZodOptional<z.ZodBoolean>;
+    }, "strip", z.ZodTypeAny, {
+        cleanup?: boolean | undefined;
+    }, {
+        cleanup?: boolean | undefined;
+    }>>;
+}, "strip", z.ZodTypeAny, {
+    name: string;
+    tags: string[];
+    version: string;
+    cases: {
+        expected: {
+            value: string;
+            type: "exact";
+            caseSensitive: boolean;
+        } | {
+            type: "regex";
+            pattern: string;
+            flags?: string | undefined;
+        } | {
+            value: string;
+            type: "fuzzy";
+            threshold: number;
+        } | {
+            type: "llm_grader";
+            threshold: number;
+            rubric: string;
+            provider?: "openai" | "azure-openai" | "vercel-ai" | "anthropic" | "google" | "mistral" | "cohere" | "huggingface" | "ollama" | "custom" | undefined;
+            model?: string | undefined;
+        } | {
+            values: string[];
+            type: "contains";
+            mode: "all" | "any";
+        } | {
+            type: "json_schema";
+            schema: Record<string, unknown>;
+        } | {
+            type: "custom";
+            evaluator: string;
+            config?: Record<string, unknown> | undefined;
+        };
+        id: string;
+        prompt: string | {
+            role: "system" | "user" | "assistant";
+            content: string;
+        }[];
+        tags: string[];
+        metadata: Record<string, unknown>;
+        retries: number;
+        name?: string | undefined;
+        provider?: "openai" | "azure-openai" | "vercel-ai" | "anthropic" | "google" | "mistral" | "cohere" | "huggingface" | "ollama" | "custom" | undefined;
+        timeout?: number | undefined;
+        model?: string | undefined;
+        description?: string | undefined;
+        variables?: Record<string, string | number | boolean> | undefined;
+        redaction?: {
+            enabled: boolean;
+            redactPrompts: boolean;
+            redactResponses: boolean;
+            redactMetadata: boolean;
+            replacement: string;
+            patterns?: string[] | undefined;
+        } | undefined;
+    }[];
+    provider?: "openai" | "azure-openai" | "vercel-ai" | "anthropic" | "google" | "mistral" | "cohere" | "huggingface" | "ollama" | "custom" | undefined;
+    model?: string | undefined;
+    description?: string | undefined;
+    variables?: Record<string, string | number | boolean> | undefined;
+    redaction?: {
+        enabled: boolean;
+        redactPrompts: boolean;
+        redactResponses: boolean;
+        redactMetadata: boolean;
+        replacement: string;
+        patterns?: string[] | undefined;
+    } | undefined;
+    providerConfig?: {
+        apiKey?: string | undefined;
+        baseUrl?: string | undefined;
+        defaultModel?: string | undefined;
+        timeout?: number | undefined;
+        maxRetries?: number | undefined;
+        organization?: string | undefined;
+        resourceName?: string | undefined;
+        deploymentName?: string | undefined;
+        apiVersion?: string | undefined;
+        underlyingProvider?: "openai" | "anthropic" | "google" | "mistral" | "azure" | undefined;
+    } | undefined;
+    seed?: number | undefined;
+    temperature?: number | undefined;
+    maxTokens?: number | undefined;
+    setup?: {
+        systemPrompt?: string | undefined;
+        functions?: unknown[] | undefined;
+    } | undefined;
+    teardown?: {
+        cleanup?: boolean | undefined;
+    } | undefined;
+}, {
+    name: string;
+    cases: {
+        expected: {
+            value: string;
+            type: "exact";
+            caseSensitive?: boolean | undefined;
+        } | {
+            type: "regex";
+            pattern: string;
+            flags?: string | undefined;
+        } | {
+            value: string;
+            type: "fuzzy";
+            threshold?: number | undefined;
+        } | {
+            type: "llm_grader";
+            rubric: string;
+            provider?: "openai" | "azure-openai" | "vercel-ai" | "anthropic" | "google" | "mistral" | "cohere" | "huggingface" | "ollama" | "custom" | undefined;
+            threshold?: number | undefined;
+            model?: string | undefined;
+        } | {
+            values: string[];
+            type: "contains";
+            mode?: "all" | "any" | undefined;
+        } | {
+            type: "json_schema";
+            schema: Record<string, unknown>;
+        } | {
+            type: "custom";
+            evaluator: string;
+            config?: Record<string, unknown> | undefined;
+        };
+        id: string;
+        prompt: string | {
+            role: "system" | "user" | "assistant";
+            content: string;
+        }[];
+        name?: string | undefined;
+        provider?: "openai" | "azure-openai" | "vercel-ai" | "anthropic" | "google" | "mistral" | "cohere" | "huggingface" | "ollama" | "custom" | undefined;
+        timeout?: number | undefined;
+        model?: string | undefined;
+        description?: string | undefined;
+        tags?: string[] | undefined;
+        metadata?: Record<string, unknown> | undefined;
+        retries?: number | undefined;
+        variables?: Record<string, string | number | boolean> | undefined;
+        redaction?: {
+            enabled?: boolean | undefined;
+            patterns?: string[] | undefined;
+            redactPrompts?: boolean | undefined;
+            redactResponses?: boolean | undefined;
+            redactMetadata?: boolean | undefined;
+            replacement?: string | undefined;
+        } | undefined;
+    }[];
+    provider?: "openai" | "azure-openai" | "vercel-ai" | "anthropic" | "google" | "mistral" | "cohere" | "huggingface" | "ollama" | "custom" | undefined;
+    model?: string | undefined;
+    description?: string | undefined;
+    tags?: string[] | undefined;
+    variables?: Record<string, string | number | boolean> | undefined;
+    redaction?: {
+        enabled?: boolean | undefined;
+        patterns?: string[] | undefined;
+        redactPrompts?: boolean | undefined;
+        redactResponses?: boolean | undefined;
+        redactMetadata?: boolean | undefined;
+        replacement?: string | undefined;
+    } | undefined;
+    version?: string | undefined;
+    providerConfig?: {
+        apiKey?: string | undefined;
+        baseUrl?: string | undefined;
+        defaultModel?: string | undefined;
+        timeout?: number | undefined;
+        maxRetries?: number | undefined;
+        organization?: string | undefined;
+        resourceName?: string | undefined;
+        deploymentName?: string | undefined;
+        apiVersion?: string | undefined;
+        underlyingProvider?: "openai" | "anthropic" | "google" | "mistral" | "azure" | undefined;
+    } | undefined;
+    seed?: number | undefined;
+    temperature?: number | undefined;
+    maxTokens?: number | undefined;
+    setup?: {
+        systemPrompt?: string | undefined;
+        functions?: unknown[] | undefined;
+    } | undefined;
+    teardown?: {
+        cleanup?: boolean | undefined;
+    } | undefined;
+}>;
+export type Expected = z.infer<typeof ExpectedSchema>;
+export type TestCase = z.infer<typeof TestCaseSchema>;
+export type Scenario = z.infer<typeof ScenarioSchema>;
+export type Provider = z.infer<typeof ProviderSchema>;
+export type ProviderConfig = z.infer<typeof ProviderConfigSchema>;
+export type ChatMessageType = z.infer<typeof ChatMessageSchema>;
+export type Variables = z.infer<typeof VariablesSchema>;
+export type ScenarioRedactionConfig = z.infer<typeof RedactionConfigSchema>;
+export {};
+//# sourceMappingURL=schema.d.ts.map