@kat-ai/eval 0.1.0

package/dist/agent/index.d.cts

@@ -0,0 +1,170 @@
+import { B as BaseEvalConfig, E as EvalCriterion, b as EvalResult, a as EvalEvidence } from '../types-BJjlqNhg.cjs';
+
+/**
+ * Agent Eval Types
+ *
+ * Types for evaluating end-to-end agent behavior.
+ */
+
+/**
+ * Configuration for agent evaluation.
+ */
+interface AgentEvalConfig extends BaseEvalConfig {
+    /** API endpoint URL or agent function */
+    agentEndpoint: string | AgentFunction;
+    /** Test scenarios to run */
+    scenarios: AgentTestScenario[];
+    /** Maximum conversation turns per scenario (default: 5) */
+    maxTurns?: number;
+    /** Timeout per scenario in milliseconds (default: 60000) */
+    timeout?: number;
+    /** LLM grading configuration */
+    graderConfig?: {
+        model?: string;
+        temperature?: number;
+    };
+}
+/**
+ * Function type for local agent testing.
+ */
+type AgentFunction = (message: string, context?: AgentContext) => Promise<AgentResponse>;
+/**
+ * Context passed to agent function.
+ */
+interface AgentContext {
+    sessionId?: string;
+    previousContext?: unknown;
+    previousIntent?: unknown;
+    conversationHistory?: Array<{
+        role: 'user' | 'assistant';
+        content: string;
+    }>;
+}
+/**
+ * A test scenario for agent evaluation.
+ */
+interface AgentTestScenario {
+    /** Unique name for the scenario */
+    name: string;
+    /** Description of what the scenario tests */
+    description?: string;
+    /** Initial user query */
+    initialQuery: string;
+    /** Expected outcome type */
+    expectedOutcome?: 'answer' | 'follow_up' | 'blocked' | 'out_of_scope';
+    /** Responses to follow-up questions (pattern -> response) */
+    followUpResponses?: Record<string, string>;
+    /** Expected KBs to be called */
+    expectedKBs?: string[];
+    /** Maximum turns for this scenario */
+    maxTurns?: number;
+    /** Evaluation criteria */
+    evaluation: ScenarioEvaluation;
+}
+/**
+ * Evaluation criteria for a scenario.
+ */
+interface ScenarioEvaluation {
+    /** Strings that MUST appear in the answer */
+    mustContain?: string[];
+    /** Strings that must NOT appear in the answer */
+    mustNotContain?: string[];
+    /** LLM rubric for grading the answer */
+    rubric?: string;
+    /** Custom evaluation criteria */
+    criteria?: EvalCriterion[];
+}
+/**
+ * Result from agent evaluation.
+ */
+interface AgentEvalResult extends EvalResult {
+    /** Individual metric scores */
+    scores: {
+        /** Percentage of scenarios with correct outcome type (0-100) */
+        accuracy: number;
+        /** Average answer relevance to queries (0-100) */
+        relevance: number;
+        /** Average answer completeness (0-100) */
+        completeness: number;
+        /** Average helpfulness/actionability (0-100) */
+        helpfulness: number;
+    };
+    /** Results for each scenario */
+    scenarioResults: ScenarioResult[];
+}
+/**
+ * Result from running a single scenario.
+ */
+interface ScenarioResult {
+    /** The scenario that was run */
+    scenario: AgentTestScenario;
+    /** Whether the scenario passed */
+    passed: boolean;
+    /** Number of conversation turns */
+    turns: number;
+    /** Final outcome type */
+    finalOutcome: string;
+    /** Final answer (null if not 'answer' outcome) */
+    finalAnswer: string | null;
+    /** Detailed evaluation result */
+    evaluation: {
+        passed: boolean;
+        score: number;
+        evidence: EvalEvidence[];
+    };
+    /** Full conversation history */
+    conversation: ConversationTurn[];
+    /** Duration in milliseconds */
+    duration: number;
+    /** Error message if scenario failed */
+    error?: string;
+}
+/**
+ * A single turn in the conversation.
+ */
+interface ConversationTurn {
+    /** Turn number (1-indexed) */
+    turn: number;
+    /** User message */
+    userMessage: string;
+    /** Agent response */
+    agentResponse: AgentResponse;
+}
+/**
+ * Response from the agent (matches @kat/core ChatResponse structure).
+ */
+interface AgentResponse {
+    /** Outcome type */
+    outcome: 'answer' | 'follow_up' | 'blocked' | 'out_of_scope';
+    /** Answer text (for 'answer' outcome) */
+    answer?: string | null;
+    /** Follow-up question (for 'follow_up' outcome) */
+    followUpQuestion?: string | null;
+    /** Multi-choice options */
+    options?: Array<{
+        id: string;
+        label: string;
+    }>;
+    /** Context for multi-turn */
+    context?: unknown;
+    /** Intent for multi-turn */
+    intent?: unknown;
+    /** Execution trace */
+    trace?: unknown;
+    /** Session ID */
+    sessionId?: string;
+}
+
+/**
+ * Agent Eval - Layer 3
+ *
+ * Evaluates end-to-end agent behavior by running multi-turn
+ * conversation scenarios and grading the responses.
+ */
+
+/**
+ * Evaluate agent behavior by running test scenarios.
+ */
+declare function evaluateAgent(config: AgentEvalConfig): Promise<AgentEvalResult>;
+
+export { type AgentEvalConfig, type AgentEvalResult, type AgentResponse, type AgentTestScenario, type ConversationTurn, type ScenarioEvaluation, type ScenarioResult, evaluateAgent };

package/dist/agent/index.d.ts

@@ -0,0 +1,170 @@
+import { B as BaseEvalConfig, E as EvalCriterion, b as EvalResult, a as EvalEvidence } from '../types-BJjlqNhg.js';
+
+/**
+ * Agent Eval Types
+ *
+ * Types for evaluating end-to-end agent behavior.
+ */
+
+/**
+ * Configuration for agent evaluation.
+ */
+interface AgentEvalConfig extends BaseEvalConfig {
+    /** API endpoint URL or agent function */
+    agentEndpoint: string | AgentFunction;
+    /** Test scenarios to run */
+    scenarios: AgentTestScenario[];
+    /** Maximum conversation turns per scenario (default: 5) */
+    maxTurns?: number;
+    /** Timeout per scenario in milliseconds (default: 60000) */
+    timeout?: number;
+    /** LLM grading configuration */
+    graderConfig?: {
+        model?: string;
+        temperature?: number;
+    };
+}
+/**
+ * Function type for local agent testing.
+ */
+type AgentFunction = (message: string, context?: AgentContext) => Promise<AgentResponse>;
+/**
+ * Context passed to agent function.
+ */
+interface AgentContext {
+    sessionId?: string;
+    previousContext?: unknown;
+    previousIntent?: unknown;
+    conversationHistory?: Array<{
+        role: 'user' | 'assistant';
+        content: string;
+    }>;
+}
+/**
+ * A test scenario for agent evaluation.
+ */
+interface AgentTestScenario {
+    /** Unique name for the scenario */
+    name: string;
+    /** Description of what the scenario tests */
+    description?: string;
+    /** Initial user query */
+    initialQuery: string;
+    /** Expected outcome type */
+    expectedOutcome?: 'answer' | 'follow_up' | 'blocked' | 'out_of_scope';
+    /** Responses to follow-up questions (pattern -> response) */
+    followUpResponses?: Record<string, string>;
+    /** Expected KBs to be called */
+    expectedKBs?: string[];
+    /** Maximum turns for this scenario */
+    maxTurns?: number;
+    /** Evaluation criteria */
+    evaluation: ScenarioEvaluation;
+}
+/**
+ * Evaluation criteria for a scenario.
+ */
+interface ScenarioEvaluation {
+    /** Strings that MUST appear in the answer */
+    mustContain?: string[];
+    /** Strings that must NOT appear in the answer */
+    mustNotContain?: string[];
+    /** LLM rubric for grading the answer */
+    rubric?: string;
+    /** Custom evaluation criteria */
+    criteria?: EvalCriterion[];
+}
+/**
+ * Result from agent evaluation.
+ */
+interface AgentEvalResult extends EvalResult {
+    /** Individual metric scores */
+    scores: {
+        /** Percentage of scenarios with correct outcome type (0-100) */
+        accuracy: number;
+        /** Average answer relevance to queries (0-100) */
+        relevance: number;
+        /** Average answer completeness (0-100) */
+        completeness: number;
+        /** Average helpfulness/actionability (0-100) */
+        helpfulness: number;
+    };
+    /** Results for each scenario */
+    scenarioResults: ScenarioResult[];
+}
+/**
+ * Result from running a single scenario.
+ */
+interface ScenarioResult {
+    /** The scenario that was run */
+    scenario: AgentTestScenario;
+    /** Whether the scenario passed */
+    passed: boolean;
+    /** Number of conversation turns */
+    turns: number;
+    /** Final outcome type */
+    finalOutcome: string;
+    /** Final answer (null if not 'answer' outcome) */
+    finalAnswer: string | null;
+    /** Detailed evaluation result */
+    evaluation: {
+        passed: boolean;
+        score: number;
+        evidence: EvalEvidence[];
+    };
+    /** Full conversation history */
+    conversation: ConversationTurn[];
+    /** Duration in milliseconds */
+    duration: number;
+    /** Error message if scenario failed */
+    error?: string;
+}
+/**
+ * A single turn in the conversation.
+ */
+interface ConversationTurn {
+    /** Turn number (1-indexed) */
+    turn: number;
+    /** User message */
+    userMessage: string;
+    /** Agent response */
+    agentResponse: AgentResponse;
+}
+/**
+ * Response from the agent (matches @kat/core ChatResponse structure).
+ */
+interface AgentResponse {
+    /** Outcome type */
+    outcome: 'answer' | 'follow_up' | 'blocked' | 'out_of_scope';
+    /** Answer text (for 'answer' outcome) */
+    answer?: string | null;
+    /** Follow-up question (for 'follow_up' outcome) */
+    followUpQuestion?: string | null;
+    /** Multi-choice options */
+    options?: Array<{
+        id: string;
+        label: string;
+    }>;
+    /** Context for multi-turn */
+    context?: unknown;
+    /** Intent for multi-turn */
+    intent?: unknown;
+    /** Execution trace */
+    trace?: unknown;
+    /** Session ID */
+    sessionId?: string;
+}
+
+/**
+ * Agent Eval - Layer 3
+ *
+ * Evaluates end-to-end agent behavior by running multi-turn
+ * conversation scenarios and grading the responses.
+ */
+
+/**
+ * Evaluate agent behavior by running test scenarios.
+ */
+declare function evaluateAgent(config: AgentEvalConfig): Promise<AgentEvalResult>;
+
+export { type AgentEvalConfig, type AgentEvalResult, type AgentResponse, type AgentTestScenario, type ConversationTurn, type ScenarioEvaluation, type ScenarioResult, evaluateAgent };