@assay-ai/core 0.2.0-beta → 0.3.0-beta

package/dist/index.d.cts

@@ -19,6 +19,7 @@ interface LLMTestCase {
     completionTime?: number;
     name?: string;
     tags?: string[];
+    conversation?: ConversationalTestCase;
 }
 interface ConversationalTestCase {
     turns: Array<{
@@ -238,6 +239,49 @@ declare class JsonCorrectnessMetric extends BaseMetric {
     measure(testCase: LLMTestCase): Promise<MetricResult>;
 }
 
+interface ToolCorrectnessConfig extends MetricConfig {
+    /** Whether to also compare tool input parameters (default: false) */
+    matchParameters?: boolean;
+}
+declare class ToolCorrectnessMetric extends BaseMetric {
+    readonly name = "Tool Correctness";
+    readonly requiredFields: (keyof LLMTestCase)[];
+    readonly requiresProvider = false;
+    private readonly matchParameters;
+    constructor(config?: ToolCorrectnessConfig);
+    measure(testCase: LLMTestCase): Promise<MetricResult>;
+}
+
+declare class TaskCompletionMetric extends BaseMetric {
+    readonly name = "Task Completion";
+    readonly requiredFields: (keyof LLMTestCase)[];
+    measure(testCase: LLMTestCase): Promise<MetricResult>;
+}
+
+declare class GoalAccuracyMetric extends BaseMetric {
+    readonly name = "Goal Accuracy";
+    readonly requiredFields: (keyof LLMTestCase)[];
+    measure(testCase: LLMTestCase): Promise<MetricResult>;
+}
+
+declare class ConversationCompletenessMetric extends BaseMetric {
+    readonly name = "Conversation Completeness";
+    readonly requiredFields: (keyof LLMTestCase)[];
+    measure(testCase: LLMTestCase): Promise<MetricResult>;
+}
+
+declare class KnowledgeRetentionMetric extends BaseMetric {
+    readonly name = "Knowledge Retention";
+    readonly requiredFields: (keyof LLMTestCase)[];
+    measure(testCase: LLMTestCase): Promise<MetricResult>;
+}
+
+declare class RoleAdherenceMetric extends BaseMetric {
+    readonly name = "Role Adherence";
+    readonly requiredFields: (keyof LLMTestCase)[];
+    measure(testCase: LLMTestCase): Promise<MetricResult>;
+}
+
 declare class OpenAIProvider extends BaseLLMProvider {
     private client;
     constructor(config?: ProviderConfig);
@@ -259,6 +303,30 @@ declare class OllamaProvider extends BaseLLMProvider {
     generate(prompt: string): Promise<string>;
 }
 
+declare class GeminiProvider extends BaseLLMProvider {
+    private readonly apiKey;
+    constructor(config?: ProviderConfig);
+    get providerName(): string;
+    generate(prompt: string): Promise<string>;
+}
+
+interface AzureOpenAIConfig extends ProviderConfig {
+    endpoint?: string;
+    deploymentName?: string;
+    apiVersion?: string;
+}
+declare class AzureOpenAIProvider extends BaseLLMProvider {
+    private readonly azureConfig;
+    private client;
+    private readonly endpoint;
+    private readonly apiKey;
+    private readonly deploymentName;
+    private readonly apiVersion;
+    constructor(azureConfig?: AzureOpenAIConfig);
+    get providerName(): string;
+    generate(prompt: string): Promise<string>;
+}
+
 /**
  * Resolve a provider from a string name, provider instance, or auto-detect from env vars.
  * Returns a noop provider if undefined (for non-LLM metrics).
@@ -457,4 +525,4 @@ declare function weightedAverage(values: number[], weights: number[]): number;
  */
 declare function meanAveragePrecision(relevances: boolean[]): number;
 
-export { AnswerRelevancyMetric, AnthropicProvider, type AssayConfig, type AssertEvalOptions, type AssertEvalResult, BaseLLMProvider, BaseMetric, BiasMetric, ConsoleReporter, ContextualPrecisionMetric, ContextualRecallMetric, ContextualRelevancyMetric, type ConversationalTestCase, type EvaluateConfig, type EvaluateResult, type EvaluationDataset, type EvaluationSummary, type ExactMatchConfig, ExactMatchMetric, FaithfulnessMetric, GEval, type GEvalConfig, type Golden, HallucinationMetric, type JsonCorrectnessConfig, JsonCorrectnessMetric, type LLMTestCase, type Limiter, type MetricConfig, type MetricResult, OllamaProvider, OpenAIProvider, type ParseJsonOptions, type ProviderConfig, SummarizationMetric, type TestCaseResult, type ToolCall, ToxicityMetric, assertEval, createLimiter, evaluate, meanAveragePrecision, parseJson, ratio, resetConfigCache, resolveConfig, resolveProvider, tryParseJson, weightedAverage };
+export { AnswerRelevancyMetric, AnthropicProvider, type AssayConfig, type AssertEvalOptions, type AssertEvalResult, type AzureOpenAIConfig, AzureOpenAIProvider, BaseLLMProvider, BaseMetric, BiasMetric, ConsoleReporter, ContextualPrecisionMetric, ContextualRecallMetric, ContextualRelevancyMetric, ConversationCompletenessMetric, type ConversationalTestCase, type EvaluateConfig, type EvaluateResult, type EvaluationDataset, type EvaluationSummary, type ExactMatchConfig, ExactMatchMetric, FaithfulnessMetric, GEval, type GEvalConfig, GeminiProvider, GoalAccuracyMetric, type Golden, HallucinationMetric, type JsonCorrectnessConfig, JsonCorrectnessMetric, KnowledgeRetentionMetric, type LLMTestCase, type Limiter, type MetricConfig, type MetricResult, OllamaProvider, OpenAIProvider, type ParseJsonOptions, type ProviderConfig, RoleAdherenceMetric, SummarizationMetric, TaskCompletionMetric, type TestCaseResult, type ToolCall, type ToolCorrectnessConfig, ToolCorrectnessMetric, ToxicityMetric, assertEval, createLimiter, evaluate, meanAveragePrecision, parseJson, ratio, resetConfigCache, resolveConfig, resolveProvider, tryParseJson, weightedAverage };

package/dist/index.d.ts

@@ -19,6 +19,7 @@ interface LLMTestCase {
     completionTime?: number;
     name?: string;
     tags?: string[];
+    conversation?: ConversationalTestCase;
 }
 interface ConversationalTestCase {
     turns: Array<{
@@ -238,6 +239,49 @@ declare class JsonCorrectnessMetric extends BaseMetric {
     measure(testCase: LLMTestCase): Promise<MetricResult>;
 }
 
+interface ToolCorrectnessConfig extends MetricConfig {
+    /** Whether to also compare tool input parameters (default: false) */
+    matchParameters?: boolean;
+}
+declare class ToolCorrectnessMetric extends BaseMetric {
+    readonly name = "Tool Correctness";
+    readonly requiredFields: (keyof LLMTestCase)[];
+    readonly requiresProvider = false;
+    private readonly matchParameters;
+    constructor(config?: ToolCorrectnessConfig);
+    measure(testCase: LLMTestCase): Promise<MetricResult>;
+}
+
+declare class TaskCompletionMetric extends BaseMetric {
+    readonly name = "Task Completion";
+    readonly requiredFields: (keyof LLMTestCase)[];
+    measure(testCase: LLMTestCase): Promise<MetricResult>;
+}
+
+declare class GoalAccuracyMetric extends BaseMetric {
+    readonly name = "Goal Accuracy";
+    readonly requiredFields: (keyof LLMTestCase)[];
+    measure(testCase: LLMTestCase): Promise<MetricResult>;
+}
+
+declare class ConversationCompletenessMetric extends BaseMetric {
+    readonly name = "Conversation Completeness";
+    readonly requiredFields: (keyof LLMTestCase)[];
+    measure(testCase: LLMTestCase): Promise<MetricResult>;
+}
+
+declare class KnowledgeRetentionMetric extends BaseMetric {
+    readonly name = "Knowledge Retention";
+    readonly requiredFields: (keyof LLMTestCase)[];
+    measure(testCase: LLMTestCase): Promise<MetricResult>;
+}
+
+declare class RoleAdherenceMetric extends BaseMetric {
+    readonly name = "Role Adherence";
+    readonly requiredFields: (keyof LLMTestCase)[];
+    measure(testCase: LLMTestCase): Promise<MetricResult>;
+}
+
 declare class OpenAIProvider extends BaseLLMProvider {
     private client;
     constructor(config?: ProviderConfig);
@@ -259,6 +303,30 @@ declare class OllamaProvider extends BaseLLMProvider {
     generate(prompt: string): Promise<string>;
 }
 
+declare class GeminiProvider extends BaseLLMProvider {
+    private readonly apiKey;
+    constructor(config?: ProviderConfig);
+    get providerName(): string;
+    generate(prompt: string): Promise<string>;
+}
+
+interface AzureOpenAIConfig extends ProviderConfig {
+    endpoint?: string;
+    deploymentName?: string;
+    apiVersion?: string;
+}
+declare class AzureOpenAIProvider extends BaseLLMProvider {
+    private readonly azureConfig;
+    private client;
+    private readonly endpoint;
+    private readonly apiKey;
+    private readonly deploymentName;
+    private readonly apiVersion;
+    constructor(azureConfig?: AzureOpenAIConfig);
+    get providerName(): string;
+    generate(prompt: string): Promise<string>;
+}
+
 /**
  * Resolve a provider from a string name, provider instance, or auto-detect from env vars.
  * Returns a noop provider if undefined (for non-LLM metrics).
@@ -457,4 +525,4 @@ declare function weightedAverage(values: number[], weights: number[]): number;
  */
 declare function meanAveragePrecision(relevances: boolean[]): number;
 
-export { AnswerRelevancyMetric, AnthropicProvider, type AssayConfig, type AssertEvalOptions, type AssertEvalResult, BaseLLMProvider, BaseMetric, BiasMetric, ConsoleReporter, ContextualPrecisionMetric, ContextualRecallMetric, ContextualRelevancyMetric, type ConversationalTestCase, type EvaluateConfig, type EvaluateResult, type EvaluationDataset, type EvaluationSummary, type ExactMatchConfig, ExactMatchMetric, FaithfulnessMetric, GEval, type GEvalConfig, type Golden, HallucinationMetric, type JsonCorrectnessConfig, JsonCorrectnessMetric, type LLMTestCase, type Limiter, type MetricConfig, type MetricResult, OllamaProvider, OpenAIProvider, type ParseJsonOptions, type ProviderConfig, SummarizationMetric, type TestCaseResult, type ToolCall, ToxicityMetric, assertEval, createLimiter, evaluate, meanAveragePrecision, parseJson, ratio, resetConfigCache, resolveConfig, resolveProvider, tryParseJson, weightedAverage };
+export { AnswerRelevancyMetric, AnthropicProvider, type AssayConfig, type AssertEvalOptions, type AssertEvalResult, type AzureOpenAIConfig, AzureOpenAIProvider, BaseLLMProvider, BaseMetric, BiasMetric, ConsoleReporter, ContextualPrecisionMetric, ContextualRecallMetric, ContextualRelevancyMetric, ConversationCompletenessMetric, type ConversationalTestCase, type EvaluateConfig, type EvaluateResult, type EvaluationDataset, type EvaluationSummary, type ExactMatchConfig, ExactMatchMetric, FaithfulnessMetric, GEval, type GEvalConfig, GeminiProvider, GoalAccuracyMetric, type Golden, HallucinationMetric, type JsonCorrectnessConfig, JsonCorrectnessMetric, KnowledgeRetentionMetric, type LLMTestCase, type Limiter, type MetricConfig, type MetricResult, OllamaProvider, OpenAIProvider, type ParseJsonOptions, type ProviderConfig, RoleAdherenceMetric, SummarizationMetric, TaskCompletionMetric, type TestCaseResult, type ToolCall, type ToolCorrectnessConfig, ToolCorrectnessMetric, ToxicityMetric, assertEval, createLimiter, evaluate, meanAveragePrecision, parseJson, ratio, resetConfigCache, resolveConfig, resolveProvider, tryParseJson, weightedAverage };