anantham-v2 2.0.0-alpha.1

package/dist/domain/evaluation.d.ts

@@ -0,0 +1,994 @@
+import { z } from "zod";
+/**
+ * Benchmark Scenario Categories.
+ * PRD Part 3 Section 80–120.
+ */
+export declare const BenchmarkCategorySchema: z.ZodEnum<["task_execution", "tool_use", "multi_step_agent", "parallel_execution", "workflow_dag", "approval_gate", "background_job", "lease_fencing", "crash_recovery", "remote_node", "mcp_interaction", "plugin_lifecycle", "skill_execution", "webhook_integration", "policy_denial", "prompt_injection", "secret_leakage", "project_isolation", "artifact_integrity", "false_completion"]>;
+export type BenchmarkCategory = z.infer<typeof BenchmarkCategorySchema>;
+/**
+ * Evaluation Assertion Types.
+ * PRD Part 3 Section 94.
+ */
+export declare const EvaluationAssertionTypeSchema: z.ZodEnum<["STATE_EQUALS", "EVENT_EXISTS", "ARTIFACT_EXISTS", "POLICY_DECISION", "TOOL_COUNT_LIMIT", "RESOURCE_LIMIT", "SECRET_ABSENT", "PROJECT_CONTAINMENT", "RECOVERY_SURVIVED"]>;
+export type EvaluationAssertionType = z.infer<typeof EvaluationAssertionTypeSchema>;
+/**
+ * Machine-verifiable objective evaluation assertion.
+ */
+export declare const EvaluationAssertionSchema: z.ZodObject<{
+    id: z.ZodString;
+    type: z.ZodEnum<["STATE_EQUALS", "EVENT_EXISTS", "ARTIFACT_EXISTS", "POLICY_DECISION", "TOOL_COUNT_LIMIT", "RESOURCE_LIMIT", "SECRET_ABSENT", "PROJECT_CONTAINMENT", "RECOVERY_SURVIVED"]>;
+    target: z.ZodString;
+    expected: z.ZodUnknown;
+    description: z.ZodString;
+    criticality: z.ZodDefault<z.ZodEnum<["CRITICAL", "OPTIONAL"]>>;
+}, "strip", z.ZodTypeAny, {
+    type: "RESOURCE_LIMIT" | "STATE_EQUALS" | "EVENT_EXISTS" | "ARTIFACT_EXISTS" | "POLICY_DECISION" | "TOOL_COUNT_LIMIT" | "SECRET_ABSENT" | "PROJECT_CONTAINMENT" | "RECOVERY_SURVIVED";
+    id: string;
+    description: string;
+    target: string;
+    criticality: "CRITICAL" | "OPTIONAL";
+    expected?: unknown;
+}, {
+    type: "RESOURCE_LIMIT" | "STATE_EQUALS" | "EVENT_EXISTS" | "ARTIFACT_EXISTS" | "POLICY_DECISION" | "TOOL_COUNT_LIMIT" | "SECRET_ABSENT" | "PROJECT_CONTAINMENT" | "RECOVERY_SURVIVED";
+    id: string;
+    description: string;
+    target: string;
+    expected?: unknown;
+    criticality?: "CRITICAL" | "OPTIONAL" | undefined;
+}>;
+export type EvaluationAssertion = z.infer<typeof EvaluationAssertionSchema>;
+/**
+ * Benchmark Case Contract.
+ */
+export declare const BenchmarkCaseSchema: z.ZodObject<{
+    caseId: z.ZodString;
+    datasetId: z.ZodString;
+    datasetVersion: z.ZodString;
+    name: z.ZodString;
+    description: z.ZodString;
+    category: z.ZodEnum<["task_execution", "tool_use", "multi_step_agent", "parallel_execution", "workflow_dag", "approval_gate", "background_job", "lease_fencing", "crash_recovery", "remote_node", "mcp_interaction", "plugin_lifecycle", "skill_execution", "webhook_integration", "policy_denial", "prompt_injection", "secret_leakage", "project_isolation", "artifact_integrity", "false_completion"]>;
+    difficulty: z.ZodDefault<z.ZodEnum<["EASY", "MEDIUM", "HARD", "ADVERSARIAL"]>>;
+    scenario: z.ZodString;
+    inputs: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    requiredTools: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
+    requiredCapabilities: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
+    assertions: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        type: z.ZodEnum<["STATE_EQUALS", "EVENT_EXISTS", "ARTIFACT_EXISTS", "POLICY_DECISION", "TOOL_COUNT_LIMIT", "RESOURCE_LIMIT", "SECRET_ABSENT", "PROJECT_CONTAINMENT", "RECOVERY_SURVIVED"]>;
+        target: z.ZodString;
+        expected: z.ZodUnknown;
+        description: z.ZodString;
+        criticality: z.ZodDefault<z.ZodEnum<["CRITICAL", "OPTIONAL"]>>;
+    }, "strip", z.ZodTypeAny, {
+        type: "RESOURCE_LIMIT" | "STATE_EQUALS" | "EVENT_EXISTS" | "ARTIFACT_EXISTS" | "POLICY_DECISION" | "TOOL_COUNT_LIMIT" | "SECRET_ABSENT" | "PROJECT_CONTAINMENT" | "RECOVERY_SURVIVED";
+        id: string;
+        description: string;
+        target: string;
+        criticality: "CRITICAL" | "OPTIONAL";
+        expected?: unknown;
+    }, {
+        type: "RESOURCE_LIMIT" | "STATE_EQUALS" | "EVENT_EXISTS" | "ARTIFACT_EXISTS" | "POLICY_DECISION" | "TOOL_COUNT_LIMIT" | "SECRET_ABSENT" | "PROJECT_CONTAINMENT" | "RECOVERY_SURVIVED";
+        id: string;
+        description: string;
+        target: string;
+        expected?: unknown;
+        criticality?: "CRITICAL" | "OPTIONAL" | undefined;
+    }>, "many">;
+    timeoutMs: z.ZodDefault<z.ZodNumber>;
+    tags: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
+}, "strip", z.ZodTypeAny, {
+    name: string;
+    tags: string[];
+    description: string;
+    inputs: Record<string, unknown>;
+    requiredCapabilities: string[];
+    timeoutMs: number;
+    category: "task_execution" | "tool_use" | "multi_step_agent" | "parallel_execution" | "workflow_dag" | "approval_gate" | "background_job" | "lease_fencing" | "crash_recovery" | "remote_node" | "mcp_interaction" | "plugin_lifecycle" | "skill_execution" | "webhook_integration" | "policy_denial" | "prompt_injection" | "secret_leakage" | "project_isolation" | "artifact_integrity" | "false_completion";
+    assertions: {
+        type: "RESOURCE_LIMIT" | "STATE_EQUALS" | "EVENT_EXISTS" | "ARTIFACT_EXISTS" | "POLICY_DECISION" | "TOOL_COUNT_LIMIT" | "SECRET_ABSENT" | "PROJECT_CONTAINMENT" | "RECOVERY_SURVIVED";
+        id: string;
+        description: string;
+        target: string;
+        criticality: "CRITICAL" | "OPTIONAL";
+        expected?: unknown;
+    }[];
+    caseId: string;
+    datasetId: string;
+    datasetVersion: string;
+    difficulty: "EASY" | "MEDIUM" | "HARD" | "ADVERSARIAL";
+    scenario: string;
+    requiredTools: string[];
+}, {
+    name: string;
+    description: string;
+    category: "task_execution" | "tool_use" | "multi_step_agent" | "parallel_execution" | "workflow_dag" | "approval_gate" | "background_job" | "lease_fencing" | "crash_recovery" | "remote_node" | "mcp_interaction" | "plugin_lifecycle" | "skill_execution" | "webhook_integration" | "policy_denial" | "prompt_injection" | "secret_leakage" | "project_isolation" | "artifact_integrity" | "false_completion";
+    assertions: {
+        type: "RESOURCE_LIMIT" | "STATE_EQUALS" | "EVENT_EXISTS" | "ARTIFACT_EXISTS" | "POLICY_DECISION" | "TOOL_COUNT_LIMIT" | "SECRET_ABSENT" | "PROJECT_CONTAINMENT" | "RECOVERY_SURVIVED";
+        id: string;
+        description: string;
+        target: string;
+        expected?: unknown;
+        criticality?: "CRITICAL" | "OPTIONAL" | undefined;
+    }[];
+    caseId: string;
+    datasetId: string;
+    datasetVersion: string;
+    scenario: string;
+    tags?: string[] | undefined;
+    inputs?: Record<string, unknown> | undefined;
+    requiredCapabilities?: string[] | undefined;
+    timeoutMs?: number | undefined;
+    difficulty?: "EASY" | "MEDIUM" | "HARD" | "ADVERSARIAL" | undefined;
+    requiredTools?: string[] | undefined;
+}>;
+export type BenchmarkCase = z.infer<typeof BenchmarkCaseSchema>;
+/**
+ * Benchmark Dataset Contract.
+ */
+export declare const BenchmarkDatasetSchema: z.ZodObject<{
+    datasetId: z.ZodString;
+    version: z.ZodString;
+    name: z.ZodString;
+    description: z.ZodString;
+    cases: z.ZodArray<z.ZodObject<{
+        caseId: z.ZodString;
+        datasetId: z.ZodString;
+        datasetVersion: z.ZodString;
+        name: z.ZodString;
+        description: z.ZodString;
+        category: z.ZodEnum<["task_execution", "tool_use", "multi_step_agent", "parallel_execution", "workflow_dag", "approval_gate", "background_job", "lease_fencing", "crash_recovery", "remote_node", "mcp_interaction", "plugin_lifecycle", "skill_execution", "webhook_integration", "policy_denial", "prompt_injection", "secret_leakage", "project_isolation", "artifact_integrity", "false_completion"]>;
+        difficulty: z.ZodDefault<z.ZodEnum<["EASY", "MEDIUM", "HARD", "ADVERSARIAL"]>>;
+        scenario: z.ZodString;
+        inputs: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        requiredTools: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
+        requiredCapabilities: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
+        assertions: z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            type: z.ZodEnum<["STATE_EQUALS", "EVENT_EXISTS", "ARTIFACT_EXISTS", "POLICY_DECISION", "TOOL_COUNT_LIMIT", "RESOURCE_LIMIT", "SECRET_ABSENT", "PROJECT_CONTAINMENT", "RECOVERY_SURVIVED"]>;
+            target: z.ZodString;
+            expected: z.ZodUnknown;
+            description: z.ZodString;
+            criticality: z.ZodDefault<z.ZodEnum<["CRITICAL", "OPTIONAL"]>>;
+        }, "strip", z.ZodTypeAny, {
+            type: "RESOURCE_LIMIT" | "STATE_EQUALS" | "EVENT_EXISTS" | "ARTIFACT_EXISTS" | "POLICY_DECISION" | "TOOL_COUNT_LIMIT" | "SECRET_ABSENT" | "PROJECT_CONTAINMENT" | "RECOVERY_SURVIVED";
+            id: string;
+            description: string;
+            target: string;
+            criticality: "CRITICAL" | "OPTIONAL";
+            expected?: unknown;
+        }, {
+            type: "RESOURCE_LIMIT" | "STATE_EQUALS" | "EVENT_EXISTS" | "ARTIFACT_EXISTS" | "POLICY_DECISION" | "TOOL_COUNT_LIMIT" | "SECRET_ABSENT" | "PROJECT_CONTAINMENT" | "RECOVERY_SURVIVED";
+            id: string;
+            description: string;
+            target: string;
+            expected?: unknown;
+            criticality?: "CRITICAL" | "OPTIONAL" | undefined;
+        }>, "many">;
+        timeoutMs: z.ZodDefault<z.ZodNumber>;
+        tags: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
+    }, "strip", z.ZodTypeAny, {
+        name: string;
+        tags: string[];
+        description: string;
+        inputs: Record<string, unknown>;
+        requiredCapabilities: string[];
+        timeoutMs: number;
+        category: "task_execution" | "tool_use" | "multi_step_agent" | "parallel_execution" | "workflow_dag" | "approval_gate" | "background_job" | "lease_fencing" | "crash_recovery" | "remote_node" | "mcp_interaction" | "plugin_lifecycle" | "skill_execution" | "webhook_integration" | "policy_denial" | "prompt_injection" | "secret_leakage" | "project_isolation" | "artifact_integrity" | "false_completion";
+        assertions: {
+            type: "RESOURCE_LIMIT" | "STATE_EQUALS" | "EVENT_EXISTS" | "ARTIFACT_EXISTS" | "POLICY_DECISION" | "TOOL_COUNT_LIMIT" | "SECRET_ABSENT" | "PROJECT_CONTAINMENT" | "RECOVERY_SURVIVED";
+            id: string;
+            description: string;
+            target: string;
+            criticality: "CRITICAL" | "OPTIONAL";
+            expected?: unknown;
+        }[];
+        caseId: string;
+        datasetId: string;
+        datasetVersion: string;
+        difficulty: "EASY" | "MEDIUM" | "HARD" | "ADVERSARIAL";
+        scenario: string;
+        requiredTools: string[];
+    }, {
+        name: string;
+        description: string;
+        category: "task_execution" | "tool_use" | "multi_step_agent" | "parallel_execution" | "workflow_dag" | "approval_gate" | "background_job" | "lease_fencing" | "crash_recovery" | "remote_node" | "mcp_interaction" | "plugin_lifecycle" | "skill_execution" | "webhook_integration" | "policy_denial" | "prompt_injection" | "secret_leakage" | "project_isolation" | "artifact_integrity" | "false_completion";
+        assertions: {
+            type: "RESOURCE_LIMIT" | "STATE_EQUALS" | "EVENT_EXISTS" | "ARTIFACT_EXISTS" | "POLICY_DECISION" | "TOOL_COUNT_LIMIT" | "SECRET_ABSENT" | "PROJECT_CONTAINMENT" | "RECOVERY_SURVIVED";
+            id: string;
+            description: string;
+            target: string;
+            expected?: unknown;
+            criticality?: "CRITICAL" | "OPTIONAL" | undefined;
+        }[];
+        caseId: string;
+        datasetId: string;
+        datasetVersion: string;
+        scenario: string;
+        tags?: string[] | undefined;
+        inputs?: Record<string, unknown> | undefined;
+        requiredCapabilities?: string[] | undefined;
+        timeoutMs?: number | undefined;
+        difficulty?: "EASY" | "MEDIUM" | "HARD" | "ADVERSARIAL" | undefined;
+        requiredTools?: string[] | undefined;
+    }>, "many">;
+}, "strip", z.ZodTypeAny, {
+    name: string;
+    version: string;
+    description: string;
+    datasetId: string;
+    cases: {
+        name: string;
+        tags: string[];
+        description: string;
+        inputs: Record<string, unknown>;
+        requiredCapabilities: string[];
+        timeoutMs: number;
+        category: "task_execution" | "tool_use" | "multi_step_agent" | "parallel_execution" | "workflow_dag" | "approval_gate" | "background_job" | "lease_fencing" | "crash_recovery" | "remote_node" | "mcp_interaction" | "plugin_lifecycle" | "skill_execution" | "webhook_integration" | "policy_denial" | "prompt_injection" | "secret_leakage" | "project_isolation" | "artifact_integrity" | "false_completion";
+        assertions: {
+            type: "RESOURCE_LIMIT" | "STATE_EQUALS" | "EVENT_EXISTS" | "ARTIFACT_EXISTS" | "POLICY_DECISION" | "TOOL_COUNT_LIMIT" | "SECRET_ABSENT" | "PROJECT_CONTAINMENT" | "RECOVERY_SURVIVED";
+            id: string;
+            description: string;
+            target: string;
+            criticality: "CRITICAL" | "OPTIONAL";
+            expected?: unknown;
+        }[];
+        caseId: string;
+        datasetId: string;
+        datasetVersion: string;
+        difficulty: "EASY" | "MEDIUM" | "HARD" | "ADVERSARIAL";
+        scenario: string;
+        requiredTools: string[];
+    }[];
+}, {
+    name: string;
+    version: string;
+    description: string;
+    datasetId: string;
+    cases: {
+        name: string;
+        description: string;
+        category: "task_execution" | "tool_use" | "multi_step_agent" | "parallel_execution" | "workflow_dag" | "approval_gate" | "background_job" | "lease_fencing" | "crash_recovery" | "remote_node" | "mcp_interaction" | "plugin_lifecycle" | "skill_execution" | "webhook_integration" | "policy_denial" | "prompt_injection" | "secret_leakage" | "project_isolation" | "artifact_integrity" | "false_completion";
+        assertions: {
+            type: "RESOURCE_LIMIT" | "STATE_EQUALS" | "EVENT_EXISTS" | "ARTIFACT_EXISTS" | "POLICY_DECISION" | "TOOL_COUNT_LIMIT" | "SECRET_ABSENT" | "PROJECT_CONTAINMENT" | "RECOVERY_SURVIVED";
+            id: string;
+            description: string;
+            target: string;
+            expected?: unknown;
+            criticality?: "CRITICAL" | "OPTIONAL" | undefined;
+        }[];
+        caseId: string;
+        datasetId: string;
+        datasetVersion: string;
+        scenario: string;
+        tags?: string[] | undefined;
+        inputs?: Record<string, unknown> | undefined;
+        requiredCapabilities?: string[] | undefined;
+        timeoutMs?: number | undefined;
+        difficulty?: "EASY" | "MEDIUM" | "HARD" | "ADVERSARIAL" | undefined;
+        requiredTools?: string[] | undefined;
+    }[];
+}>;
+export type BenchmarkDataset = z.infer<typeof BenchmarkDatasetSchema>;
+/**
+ * Result of evaluating an individual assertion.
+ */
+export declare const AssertionEvaluationResultSchema: z.ZodObject<{
+    assertionId: z.ZodString;
+    passed: z.ZodBoolean;
+    expected: z.ZodUnknown;
+    observed: z.ZodUnknown;
+    evidence: z.ZodString;
+    criticality: z.ZodEnum<["CRITICAL", "OPTIONAL"]>;
+}, "strip", z.ZodTypeAny, {
+    passed: boolean;
+    criticality: "CRITICAL" | "OPTIONAL";
+    assertionId: string;
+    evidence: string;
+    expected?: unknown;
+    observed?: unknown;
+}, {
+    passed: boolean;
+    criticality: "CRITICAL" | "OPTIONAL";
+    assertionId: string;
+    evidence: string;
+    expected?: unknown;
+    observed?: unknown;
+}>;
+export type AssertionEvaluationResult = z.infer<typeof AssertionEvaluationResultSchema>;
+/**
+ * Result of evaluating a single benchmark case.
+ */
+export declare const EvaluationCaseResultSchema: z.ZodObject<{
+    caseId: z.ZodString;
+    status: z.ZodEnum<["PASS", "FAIL", "PARTIAL", "INCONCLUSIVE"]>;
+    score: z.ZodNumber;
+    assertionResults: z.ZodArray<z.ZodObject<{
+        assertionId: z.ZodString;
+        passed: z.ZodBoolean;
+        expected: z.ZodUnknown;
+        observed: z.ZodUnknown;
+        evidence: z.ZodString;
+        criticality: z.ZodEnum<["CRITICAL", "OPTIONAL"]>;
+    }, "strip", z.ZodTypeAny, {
+        passed: boolean;
+        criticality: "CRITICAL" | "OPTIONAL";
+        assertionId: string;
+        evidence: string;
+        expected?: unknown;
+        observed?: unknown;
+    }, {
+        passed: boolean;
+        criticality: "CRITICAL" | "OPTIONAL";
+        assertionId: string;
+        evidence: string;
+        expected?: unknown;
+        observed?: unknown;
+    }>, "many">;
+    evidence: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    failureClassification: z.ZodDefault<z.ZodEnum<["NONE", "BENCHMARK_DEFINITION", "RUNTIME", "POLICY", "SECURITY", "TIMEOUT", "ASSERTION", "RECOVERY", "RESOURCE"]>>;
+    durationMs: z.ZodNumber;
+    startedAt: z.ZodString;
+    completedAt: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    status: "FAIL" | "PASS" | "PARTIAL" | "INCONCLUSIVE";
+    durationMs: number;
+    startedAt: string;
+    completedAt: string;
+    failureClassification: "NONE" | "TIMEOUT" | "BENCHMARK_DEFINITION" | "RUNTIME" | "POLICY" | "SECURITY" | "ASSERTION" | "RECOVERY" | "RESOURCE";
+    caseId: string;
+    evidence: Record<string, unknown>;
+    score: number;
+    assertionResults: {
+        passed: boolean;
+        criticality: "CRITICAL" | "OPTIONAL";
+        assertionId: string;
+        evidence: string;
+        expected?: unknown;
+        observed?: unknown;
+    }[];
+}, {
+    status: "FAIL" | "PASS" | "PARTIAL" | "INCONCLUSIVE";
+    durationMs: number;
+    startedAt: string;
+    completedAt: string;
+    caseId: string;
+    score: number;
+    assertionResults: {
+        passed: boolean;
+        criticality: "CRITICAL" | "OPTIONAL";
+        assertionId: string;
+        evidence: string;
+        expected?: unknown;
+        observed?: unknown;
+    }[];
+    failureClassification?: "NONE" | "TIMEOUT" | "BENCHMARK_DEFINITION" | "RUNTIME" | "POLICY" | "SECURITY" | "ASSERTION" | "RECOVERY" | "RESOURCE" | undefined;
+    evidence?: Record<string, unknown> | undefined;
+}>;
+export type EvaluationCaseResult = z.infer<typeof EvaluationCaseResultSchema>;
+/**
+ * Provenance metadata for an evaluation run.
+ */
+export declare const EvaluationProvenanceSchema: z.ZodObject<{
+    runtimeVersion: z.ZodString;
+    gitCommit: z.ZodString;
+    datasetId: z.ZodString;
+    datasetVersion: z.ZodString;
+    modelProfile: z.ZodDefault<z.ZodString>;
+    seed: z.ZodOptional<z.ZodNumber>;
+    environment: z.ZodDefault<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    modelProfile: string;
+    runtimeVersion: string;
+    datasetId: string;
+    datasetVersion: string;
+    gitCommit: string;
+    environment: string;
+    seed?: number | undefined;
+}, {
+    runtimeVersion: string;
+    datasetId: string;
+    datasetVersion: string;
+    gitCommit: string;
+    modelProfile?: string | undefined;
+    seed?: number | undefined;
+    environment?: string | undefined;
+}>;
+export type EvaluationProvenance = z.infer<typeof EvaluationProvenanceSchema>;
+/**
+ * Authoritative Evaluation Run.
+ */
+export declare const EvaluationRunSchema: z.ZodObject<{
+    id: z.ZodString;
+    datasetId: z.ZodString;
+    datasetVersion: z.ZodString;
+    status: z.ZodEnum<["RUNNING", "COMPLETED", "FAILED", "CANCELLED"]>;
+    summary: z.ZodObject<{
+        totalCases: z.ZodNumber;
+        passedCases: z.ZodNumber;
+        failedCases: z.ZodNumber;
+        partialCases: z.ZodNumber;
+        inconclusiveCases: z.ZodNumber;
+        overallScore: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        totalCases: number;
+        passedCases: number;
+        failedCases: number;
+        partialCases: number;
+        inconclusiveCases: number;
+        overallScore: number;
+    }, {
+        totalCases: number;
+        passedCases: number;
+        failedCases: number;
+        partialCases: number;
+        inconclusiveCases: number;
+        overallScore: number;
+    }>;
+    results: z.ZodArray<z.ZodObject<{
+        caseId: z.ZodString;
+        status: z.ZodEnum<["PASS", "FAIL", "PARTIAL", "INCONCLUSIVE"]>;
+        score: z.ZodNumber;
+        assertionResults: z.ZodArray<z.ZodObject<{
+            assertionId: z.ZodString;
+            passed: z.ZodBoolean;
+            expected: z.ZodUnknown;
+            observed: z.ZodUnknown;
+            evidence: z.ZodString;
+            criticality: z.ZodEnum<["CRITICAL", "OPTIONAL"]>;
+        }, "strip", z.ZodTypeAny, {
+            passed: boolean;
+            criticality: "CRITICAL" | "OPTIONAL";
+            assertionId: string;
+            evidence: string;
+            expected?: unknown;
+            observed?: unknown;
+        }, {
+            passed: boolean;
+            criticality: "CRITICAL" | "OPTIONAL";
+            assertionId: string;
+            evidence: string;
+            expected?: unknown;
+            observed?: unknown;
+        }>, "many">;
+        evidence: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        failureClassification: z.ZodDefault<z.ZodEnum<["NONE", "BENCHMARK_DEFINITION", "RUNTIME", "POLICY", "SECURITY", "TIMEOUT", "ASSERTION", "RECOVERY", "RESOURCE"]>>;
+        durationMs: z.ZodNumber;
+        startedAt: z.ZodString;
+        completedAt: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        status: "FAIL" | "PASS" | "PARTIAL" | "INCONCLUSIVE";
+        durationMs: number;
+        startedAt: string;
+        completedAt: string;
+        failureClassification: "NONE" | "TIMEOUT" | "BENCHMARK_DEFINITION" | "RUNTIME" | "POLICY" | "SECURITY" | "ASSERTION" | "RECOVERY" | "RESOURCE";
+        caseId: string;
+        evidence: Record<string, unknown>;
+        score: number;
+        assertionResults: {
+            passed: boolean;
+            criticality: "CRITICAL" | "OPTIONAL";
+            assertionId: string;
+            evidence: string;
+            expected?: unknown;
+            observed?: unknown;
+        }[];
+    }, {
+        status: "FAIL" | "PASS" | "PARTIAL" | "INCONCLUSIVE";
+        durationMs: number;
+        startedAt: string;
+        completedAt: string;
+        caseId: string;
+        score: number;
+        assertionResults: {
+            passed: boolean;
+            criticality: "CRITICAL" | "OPTIONAL";
+            assertionId: string;
+            evidence: string;
+            expected?: unknown;
+            observed?: unknown;
+        }[];
+        failureClassification?: "NONE" | "TIMEOUT" | "BENCHMARK_DEFINITION" | "RUNTIME" | "POLICY" | "SECURITY" | "ASSERTION" | "RECOVERY" | "RESOURCE" | undefined;
+        evidence?: Record<string, unknown> | undefined;
+    }>, "many">;
+    provenance: z.ZodObject<{
+        runtimeVersion: z.ZodString;
+        gitCommit: z.ZodString;
+        datasetId: z.ZodString;
+        datasetVersion: z.ZodString;
+        modelProfile: z.ZodDefault<z.ZodString>;
+        seed: z.ZodOptional<z.ZodNumber>;
+        environment: z.ZodDefault<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        modelProfile: string;
+        runtimeVersion: string;
+        datasetId: string;
+        datasetVersion: string;
+        gitCommit: string;
+        environment: string;
+        seed?: number | undefined;
+    }, {
+        runtimeVersion: string;
+        datasetId: string;
+        datasetVersion: string;
+        gitCommit: string;
+        modelProfile?: string | undefined;
+        seed?: number | undefined;
+        environment?: string | undefined;
+    }>;
+    createdAt: z.ZodString;
+    completedAt: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    status: "COMPLETED" | "CANCELLED" | "FAILED" | "RUNNING";
+    id: string;
+    provenance: {
+        modelProfile: string;
+        runtimeVersion: string;
+        datasetId: string;
+        datasetVersion: string;
+        gitCommit: string;
+        environment: string;
+        seed?: number | undefined;
+    };
+    createdAt: string;
+    datasetId: string;
+    datasetVersion: string;
+    summary: {
+        totalCases: number;
+        passedCases: number;
+        failedCases: number;
+        partialCases: number;
+        inconclusiveCases: number;
+        overallScore: number;
+    };
+    results: {
+        status: "FAIL" | "PASS" | "PARTIAL" | "INCONCLUSIVE";
+        durationMs: number;
+        startedAt: string;
+        completedAt: string;
+        failureClassification: "NONE" | "TIMEOUT" | "BENCHMARK_DEFINITION" | "RUNTIME" | "POLICY" | "SECURITY" | "ASSERTION" | "RECOVERY" | "RESOURCE";
+        caseId: string;
+        evidence: Record<string, unknown>;
+        score: number;
+        assertionResults: {
+            passed: boolean;
+            criticality: "CRITICAL" | "OPTIONAL";
+            assertionId: string;
+            evidence: string;
+            expected?: unknown;
+            observed?: unknown;
+        }[];
+    }[];
+    completedAt?: string | undefined;
+}, {
+    status: "COMPLETED" | "CANCELLED" | "FAILED" | "RUNNING";
+    id: string;
+    provenance: {
+        runtimeVersion: string;
+        datasetId: string;
+        datasetVersion: string;
+        gitCommit: string;
+        modelProfile?: string | undefined;
+        seed?: number | undefined;
+        environment?: string | undefined;
+    };
+    createdAt: string;
+    datasetId: string;
+    datasetVersion: string;
+    summary: {
+        totalCases: number;
+        passedCases: number;
+        failedCases: number;
+        partialCases: number;
+        inconclusiveCases: number;
+        overallScore: number;
+    };
+    results: {
+        status: "FAIL" | "PASS" | "PARTIAL" | "INCONCLUSIVE";
+        durationMs: number;
+        startedAt: string;
+        completedAt: string;
+        caseId: string;
+        score: number;
+        assertionResults: {
+            passed: boolean;
+            criticality: "CRITICAL" | "OPTIONAL";
+            assertionId: string;
+            evidence: string;
+            expected?: unknown;
+            observed?: unknown;
+        }[];
+        failureClassification?: "NONE" | "TIMEOUT" | "BENCHMARK_DEFINITION" | "RUNTIME" | "POLICY" | "SECURITY" | "ASSERTION" | "RECOVERY" | "RESOURCE" | undefined;
+        evidence?: Record<string, unknown> | undefined;
+    }[];
+    completedAt?: string | undefined;
+}>;
+export type EvaluationRun = z.infer<typeof EvaluationRunSchema>;
+/**
+ * Regression analysis comparing two evaluation runs.
+ */
+export declare const RegressionComparisonSchema: z.ZodObject<{
+    baselineRunId: z.ZodString;
+    currentRunId: z.ZodString;
+    datasetId: z.ZodString;
+    scoreDelta: z.ZodNumber;
+    newFailures: z.ZodArray<z.ZodString, "many">;
+    fixedFailures: z.ZodArray<z.ZodString, "many">;
+    unchangedFailures: z.ZodArray<z.ZodString, "many">;
+    regressionDetected: z.ZodBoolean;
+}, "strip", z.ZodTypeAny, {
+    datasetId: string;
+    baselineRunId: string;
+    currentRunId: string;
+    scoreDelta: number;
+    newFailures: string[];
+    fixedFailures: string[];
+    unchangedFailures: string[];
+    regressionDetected: boolean;
+}, {
+    datasetId: string;
+    baselineRunId: string;
+    currentRunId: string;
+    scoreDelta: number;
+    newFailures: string[];
+    fixedFailures: string[];
+    unchangedFailures: string[];
+    regressionDetected: boolean;
+}>;
+export type RegressionComparison = z.infer<typeof RegressionComparisonSchema>;
+/**
+ * Comprehensive Evaluation Report.
+ */
+export declare const EvaluationReportSchema: z.ZodObject<{
+    reportId: z.ZodString;
+    runId: z.ZodString;
+    generatedAt: z.ZodString;
+    run: z.ZodObject<{
+        id: z.ZodString;
+        datasetId: z.ZodString;
+        datasetVersion: z.ZodString;
+        status: z.ZodEnum<["RUNNING", "COMPLETED", "FAILED", "CANCELLED"]>;
+        summary: z.ZodObject<{
+            totalCases: z.ZodNumber;
+            passedCases: z.ZodNumber;
+            failedCases: z.ZodNumber;
+            partialCases: z.ZodNumber;
+            inconclusiveCases: z.ZodNumber;
+            overallScore: z.ZodNumber;
+        }, "strip", z.ZodTypeAny, {
+            totalCases: number;
+            passedCases: number;
+            failedCases: number;
+            partialCases: number;
+            inconclusiveCases: number;
+            overallScore: number;
+        }, {
+            totalCases: number;
+            passedCases: number;
+            failedCases: number;
+            partialCases: number;
+            inconclusiveCases: number;
+            overallScore: number;
+        }>;
+        results: z.ZodArray<z.ZodObject<{
+            caseId: z.ZodString;
+            status: z.ZodEnum<["PASS", "FAIL", "PARTIAL", "INCONCLUSIVE"]>;
+            score: z.ZodNumber;
+            assertionResults: z.ZodArray<z.ZodObject<{
+                assertionId: z.ZodString;
+                passed: z.ZodBoolean;
+                expected: z.ZodUnknown;
+                observed: z.ZodUnknown;
+                evidence: z.ZodString;
+                criticality: z.ZodEnum<["CRITICAL", "OPTIONAL"]>;
+            }, "strip", z.ZodTypeAny, {
+                passed: boolean;
+                criticality: "CRITICAL" | "OPTIONAL";
+                assertionId: string;
+                evidence: string;
+                expected?: unknown;
+                observed?: unknown;
+            }, {
+                passed: boolean;
+                criticality: "CRITICAL" | "OPTIONAL";
+                assertionId: string;
+                evidence: string;
+                expected?: unknown;
+                observed?: unknown;
+            }>, "many">;
+            evidence: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            failureClassification: z.ZodDefault<z.ZodEnum<["NONE", "BENCHMARK_DEFINITION", "RUNTIME", "POLICY", "SECURITY", "TIMEOUT", "ASSERTION", "RECOVERY", "RESOURCE"]>>;
+            durationMs: z.ZodNumber;
+            startedAt: z.ZodString;
+            completedAt: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            status: "FAIL" | "PASS" | "PARTIAL" | "INCONCLUSIVE";
+            durationMs: number;
+            startedAt: string;
+            completedAt: string;
+            failureClassification: "NONE" | "TIMEOUT" | "BENCHMARK_DEFINITION" | "RUNTIME" | "POLICY" | "SECURITY" | "ASSERTION" | "RECOVERY" | "RESOURCE";
+            caseId: string;
+            evidence: Record<string, unknown>;
+            score: number;
+            assertionResults: {
+                passed: boolean;
+                criticality: "CRITICAL" | "OPTIONAL";
+                assertionId: string;
+                evidence: string;
+                expected?: unknown;
+                observed?: unknown;
+            }[];
+        }, {
+            status: "FAIL" | "PASS" | "PARTIAL" | "INCONCLUSIVE";
+            durationMs: number;
+            startedAt: string;
+            completedAt: string;
+            caseId: string;
+            score: number;
+            assertionResults: {
+                passed: boolean;
+                criticality: "CRITICAL" | "OPTIONAL";
+                assertionId: string;
+                evidence: string;
+                expected?: unknown;
+                observed?: unknown;
+            }[];
+            failureClassification?: "NONE" | "TIMEOUT" | "BENCHMARK_DEFINITION" | "RUNTIME" | "POLICY" | "SECURITY" | "ASSERTION" | "RECOVERY" | "RESOURCE" | undefined;
+            evidence?: Record<string, unknown> | undefined;
+        }>, "many">;
+        provenance: z.ZodObject<{
+            runtimeVersion: z.ZodString;
+            gitCommit: z.ZodString;
+            datasetId: z.ZodString;
+            datasetVersion: z.ZodString;
+            modelProfile: z.ZodDefault<z.ZodString>;
+            seed: z.ZodOptional<z.ZodNumber>;
+            environment: z.ZodDefault<z.ZodString>;
+        }, "strip", z.ZodTypeAny, {
+            modelProfile: string;
+            runtimeVersion: string;
+            datasetId: string;
+            datasetVersion: string;
+            gitCommit: string;
+            environment: string;
+            seed?: number | undefined;
+        }, {
+            runtimeVersion: string;
+            datasetId: string;
+            datasetVersion: string;
+            gitCommit: string;
+            modelProfile?: string | undefined;
+            seed?: number | undefined;
+            environment?: string | undefined;
+        }>;
+        createdAt: z.ZodString;
+        completedAt: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        status: "COMPLETED" | "CANCELLED" | "FAILED" | "RUNNING";
+        id: string;
+        provenance: {
+            modelProfile: string;
+            runtimeVersion: string;
+            datasetId: string;
+            datasetVersion: string;
+            gitCommit: string;
+            environment: string;
+            seed?: number | undefined;
+        };
+        createdAt: string;
+        datasetId: string;
+        datasetVersion: string;
+        summary: {
+            totalCases: number;
+            passedCases: number;
+            failedCases: number;
+            partialCases: number;
+            inconclusiveCases: number;
+            overallScore: number;
+        };
+        results: {
+            status: "FAIL" | "PASS" | "PARTIAL" | "INCONCLUSIVE";
+            durationMs: number;
+            startedAt: string;
+            completedAt: string;
+            failureClassification: "NONE" | "TIMEOUT" | "BENCHMARK_DEFINITION" | "RUNTIME" | "POLICY" | "SECURITY" | "ASSERTION" | "RECOVERY" | "RESOURCE";
+            caseId: string;
+            evidence: Record<string, unknown>;
+            score: number;
+            assertionResults: {
+                passed: boolean;
+                criticality: "CRITICAL" | "OPTIONAL";
+                assertionId: string;
+                evidence: string;
+                expected?: unknown;
+                observed?: unknown;
+            }[];
+        }[];
+        completedAt?: string | undefined;
+    }, {
+        status: "COMPLETED" | "CANCELLED" | "FAILED" | "RUNNING";
+        id: string;
+        provenance: {
+            runtimeVersion: string;
+            datasetId: string;
+            datasetVersion: string;
+            gitCommit: string;
+            modelProfile?: string | undefined;
+            seed?: number | undefined;
+            environment?: string | undefined;
+        };
+        createdAt: string;
+        datasetId: string;
+        datasetVersion: string;
+        summary: {
+            totalCases: number;
+            passedCases: number;
+            failedCases: number;
+            partialCases: number;
+            inconclusiveCases: number;
+            overallScore: number;
+        };
+        results: {
+            status: "FAIL" | "PASS" | "PARTIAL" | "INCONCLUSIVE";
+            durationMs: number;
+            startedAt: string;
+            completedAt: string;
+            caseId: string;
+            score: number;
+            assertionResults: {
+                passed: boolean;
+                criticality: "CRITICAL" | "OPTIONAL";
+                assertionId: string;
+                evidence: string;
+                expected?: unknown;
+                observed?: unknown;
+            }[];
+            failureClassification?: "NONE" | "TIMEOUT" | "BENCHMARK_DEFINITION" | "RUNTIME" | "POLICY" | "SECURITY" | "ASSERTION" | "RECOVERY" | "RESOURCE" | undefined;
+            evidence?: Record<string, unknown> | undefined;
+        }[];
+        completedAt?: string | undefined;
+    }>;
+    regression: z.ZodOptional<z.ZodObject<{
+        baselineRunId: z.ZodString;
+        currentRunId: z.ZodString;
+        datasetId: z.ZodString;
+        scoreDelta: z.ZodNumber;
+        newFailures: z.ZodArray<z.ZodString, "many">;
+        fixedFailures: z.ZodArray<z.ZodString, "many">;
+        unchangedFailures: z.ZodArray<z.ZodString, "many">;
+        regressionDetected: z.ZodBoolean;
+    }, "strip", z.ZodTypeAny, {
+        datasetId: string;
+        baselineRunId: string;
+        currentRunId: string;
+        scoreDelta: number;
+        newFailures: string[];
+        fixedFailures: string[];
+        unchangedFailures: string[];
+        regressionDetected: boolean;
+    }, {
+        datasetId: string;
+        baselineRunId: string;
+        currentRunId: string;
+        scoreDelta: number;
+        newFailures: string[];
+        fixedFailures: string[];
+        unchangedFailures: string[];
+        regressionDetected: boolean;
+    }>>;
+}, "strip", z.ZodTypeAny, {
+    runId: string;
+    reportId: string;
+    generatedAt: string;
+    run: {
+        status: "COMPLETED" | "CANCELLED" | "FAILED" | "RUNNING";
+        id: string;
+        provenance: {
+            modelProfile: string;
+            runtimeVersion: string;
+            datasetId: string;
+            datasetVersion: string;
+            gitCommit: string;
+            environment: string;
+            seed?: number | undefined;
+        };
+        createdAt: string;
+        datasetId: string;
+        datasetVersion: string;
+        summary: {
+            totalCases: number;
+            passedCases: number;
+            failedCases: number;
+            partialCases: number;
+            inconclusiveCases: number;
+            overallScore: number;
+        };
+        results: {
+            status: "FAIL" | "PASS" | "PARTIAL" | "INCONCLUSIVE";
+            durationMs: number;
+            startedAt: string;
+            completedAt: string;
+            failureClassification: "NONE" | "TIMEOUT" | "BENCHMARK_DEFINITION" | "RUNTIME" | "POLICY" | "SECURITY" | "ASSERTION" | "RECOVERY" | "RESOURCE";
+            caseId: string;
+            evidence: Record<string, unknown>;
+            score: number;
+            assertionResults: {
+                passed: boolean;
+                criticality: "CRITICAL" | "OPTIONAL";
+                assertionId: string;
+                evidence: string;
+                expected?: unknown;
+                observed?: unknown;
+            }[];
+        }[];
+        completedAt?: string | undefined;
+    };
+    regression?: {
+        datasetId: string;
+        baselineRunId: string;
+        currentRunId: string;
+        scoreDelta: number;
+        newFailures: string[];
+        fixedFailures: string[];
+        unchangedFailures: string[];
+        regressionDetected: boolean;
+    } | undefined;
+}, {
+    runId: string;
+    reportId: string;
+    generatedAt: string;
+    run: {
+        status: "COMPLETED" | "CANCELLED" | "FAILED" | "RUNNING";
+        id: string;
+        provenance: {
+            runtimeVersion: string;
+            datasetId: string;
+            datasetVersion: string;
+            gitCommit: string;
+            modelProfile?: string | undefined;
+            seed?: number | undefined;
+            environment?: string | undefined;
+        };
+        createdAt: string;
+        datasetId: string;
+        datasetVersion: string;
+        summary: {
+            totalCases: number;
+            passedCases: number;
+            failedCases: number;
+            partialCases: number;
+            inconclusiveCases: number;
+            overallScore: number;
+        };
+        results: {
+            status: "FAIL" | "PASS" | "PARTIAL" | "INCONCLUSIVE";
+            durationMs: number;
+            startedAt: string;
+            completedAt: string;
+            caseId: string;
+            score: number;
+            assertionResults: {
+                passed: boolean;
+                criticality: "CRITICAL" | "OPTIONAL";
+                assertionId: string;
+                evidence: string;
+                expected?: unknown;
+                observed?: unknown;
+            }[];
+            failureClassification?: "NONE" | "TIMEOUT" | "BENCHMARK_DEFINITION" | "RUNTIME" | "POLICY" | "SECURITY" | "ASSERTION" | "RECOVERY" | "RESOURCE" | undefined;
+            evidence?: Record<string, unknown> | undefined;
+        }[];
+        completedAt?: string | undefined;
+    };
+    regression?: {
+        datasetId: string;
+        baselineRunId: string;
+        currentRunId: string;
+        scoreDelta: number;
+        newFailures: string[];
+        fixedFailures: string[];
+        unchangedFailures: string[];
+        regressionDetected: boolean;
+    } | undefined;
+}>;
+export type EvaluationReport = z.infer<typeof EvaluationReportSchema>;
+//# sourceMappingURL=evaluation.d.ts.map