@arizeai/phoenix-evals 0.8.0 → 0.10.0

package/src/llm/ClassificationEvaluator.ts

@@ -1,19 +1,18 @@
+import type { LanguageModel } from "ai";
+
 import { getTemplateVariables } from "../template";
-import {
+import type {
   ClassificationChoicesMap,
   CreateClassificationEvaluatorArgs,
   EvaluatorFn,
   PromptTemplate,
   WithPromptTemplate,
 } from "../types";
-import { ObjectMapping } from "../types/data";
+import type { ObjectMapping } from "../types/data";
 import { remapObject } from "../utils/objectMappingUtils";
-
 import { createClassifierFn } from "./createClassifierFn";
 import { LLMEvaluator } from "./LLMEvaluator";
 
-import { LanguageModel } from "ai";
-
 /**
  * An LLM evaluator that performs evaluation via classification
  */

package/src/llm/LLMEvaluator.ts

@@ -1,5 +1,5 @@
 import { EvaluatorBase } from "../core/EvaluatorBase";
-import { CreateLLMEvaluatorArgs } from "../types";
+import type { CreateLLMEvaluatorArgs } from "../types";
 
 /**
  * Base class for llm evaluation metrics / scores

package/src/llm/createClassificationEvaluator.ts

@@ -1,5 +1,4 @@
-import { CreateClassificationEvaluatorArgs } from "../types/evals";
-
+import type { CreateClassificationEvaluatorArgs } from "../types/evals";
 import { ClassificationEvaluator } from "./ClassificationEvaluator";
 
 export function createClassificationEvaluator<

package/src/llm/createClassifierFn.ts

@@ -1,11 +1,10 @@
 import { formatTemplate } from "../template";
-import {
+import type {
   ClassificationChoicesMap,
   CreateClassifierArgs,
   EvaluationResult,
   EvaluatorFn,
 } from "../types/evals";
-
 import { generateClassification } from "./generateClassification";
 
 /**

package/src/llm/createConcisenessEvaluator.ts

@@ -0,0 +1,71 @@
+import { CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
+import type { CreateClassificationEvaluatorArgs } from "../types/evals";
+import type { ClassificationEvaluator } from "./ClassificationEvaluator";
+import { createClassificationEvaluator } from "./createClassificationEvaluator";
+
+export interface ConcisenessEvaluatorArgs<
+  RecordType extends Record<string, unknown> = ConcisenessEvaluationRecord,
+> extends Omit<
+  CreateClassificationEvaluatorArgs<RecordType>,
+  "promptTemplate" | "choices" | "optimizationDirection" | "name"
+> {
+  optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
+  name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
+  choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
+  promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
+}
+
+/**
+ * A record to be evaluated by the conciseness evaluator.
+ */
+export type ConcisenessEvaluationRecord = {
+  input: string;
+  output: string;
+};
+
+/**
+ * Creates a conciseness evaluator function.
+ *
+ * This function returns an evaluator that determines whether a given output
+ * is concise and free of unnecessary content such as pleasantries, hedging,
+ * meta-commentary, or redundant information.
+ *
+ * @param args - The arguments for creating the conciseness evaluator.
+ * @param args.model - The model to use for classification.
+ * @param args.choices - The possible classification choices (defaults to CONCISENESS_CHOICES).
+ * @param args.promptTemplate - The prompt template to use (defaults to CONCISENESS_TEMPLATE).
+ * @param args.telemetry - The telemetry to use for the evaluator.
+ *
+ * @returns An evaluator function that takes a {@link ConcisenessEvaluationRecord} and returns a classification result
+ * indicating whether the output is concise or verbose.
+ *
+ * @example
+ * ```ts
+ * const evaluator = createConcisenessEvaluator({ model: openai("gpt-4o-mini") });
+ * const result = await evaluator.evaluate({
+ *   input: "What is the capital of France?",
+ *   output: "Paris.",
+ * });
+ * console.log(result.label); // "concise" or "verbose"
+ * ```
+ */
+export function createConcisenessEvaluator<
+  RecordType extends Record<string, unknown> = ConcisenessEvaluationRecord,
+>(
+  args: ConcisenessEvaluatorArgs<RecordType>
+): ClassificationEvaluator<RecordType> {
+  const {
+    choices = CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG.choices,
+    promptTemplate = CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG.template,
+    optimizationDirection = CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection,
+    name = CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG.name,
+    ...rest
+  } = args;
+  return createClassificationEvaluator<RecordType>({
+    ...rest,
+    promptTemplate,
+    choices,
+    optimizationDirection,
+    name,
+  });
+}

package/src/llm/createCorrectnessEvaluator.ts

@@ -1,7 +1,6 @@
 import { CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
-import { CreateClassificationEvaluatorArgs } from "../types/evals";
-
-import { ClassificationEvaluator } from "./ClassificationEvaluator";
+import type { CreateClassificationEvaluatorArgs } from "../types/evals";
+import type { ClassificationEvaluator } from "./ClassificationEvaluator";
 import { createClassificationEvaluator } from "./createClassificationEvaluator";
 
 export interface CorrectnessEvaluatorArgs<

package/src/llm/createDocumentRelevanceEvaluator.ts

@@ -1,7 +1,6 @@
 import { DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
-import { CreateClassificationEvaluatorArgs } from "../types/evals";
-
-import { ClassificationEvaluator } from "./ClassificationEvaluator";
+import type { CreateClassificationEvaluatorArgs } from "../types/evals";
+import type { ClassificationEvaluator } from "./ClassificationEvaluator";
 import { createClassificationEvaluator } from "./createClassificationEvaluator";
 
 export interface DocumentRelevanceEvaluatorArgs<

package/src/llm/createFaithfulnessEvaluator.ts

@@ -1,7 +1,6 @@
 import { FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
-import { CreateClassificationEvaluatorArgs } from "../types/evals";
-
-import { ClassificationEvaluator } from "./ClassificationEvaluator";
+import type { CreateClassificationEvaluatorArgs } from "../types/evals";
+import type { ClassificationEvaluator } from "./ClassificationEvaluator";
 import { createClassificationEvaluator } from "./createClassificationEvaluator";
 
 export interface FaithfulnessEvaluatorArgs<

package/src/llm/createHallucinationEvaluator.ts

@@ -6,9 +6,8 @@
  */
 
 import { HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
-import { CreateClassificationEvaluatorArgs } from "../types/evals";
-
-import { ClassificationEvaluator } from "./ClassificationEvaluator";
+import type { CreateClassificationEvaluatorArgs } from "../types/evals";
+import type { ClassificationEvaluator } from "./ClassificationEvaluator";
 import { createClassificationEvaluator } from "./createClassificationEvaluator";
 
 export interface HallucinationEvaluatorArgs<

package/src/llm/createToolInvocationEvaluator.ts

@@ -1,7 +1,6 @@
 import { TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
-import { CreateClassificationEvaluatorArgs } from "../types/evals";
-
-import { ClassificationEvaluator } from "./ClassificationEvaluator";
+import type { CreateClassificationEvaluatorArgs } from "../types/evals";
+import type { ClassificationEvaluator } from "./ClassificationEvaluator";
 import { createClassificationEvaluator } from "./createClassificationEvaluator";
 
 export interface ToolInvocationEvaluatorArgs<

package/src/llm/createToolResponseHandlingEvaluator.ts

@@ -0,0 +1,108 @@
+import { TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
+import type { CreateClassificationEvaluatorArgs } from "../types/evals";
+import type { ClassificationEvaluator } from "./ClassificationEvaluator";
+import { createClassificationEvaluator } from "./createClassificationEvaluator";
+
+export interface ToolResponseHandlingEvaluatorArgs<
+  RecordType extends Record<string, unknown> =
+    ToolResponseHandlingEvaluationRecord,
+> extends Omit<
+  CreateClassificationEvaluatorArgs<RecordType>,
+  "promptTemplate" | "choices" | "optimizationDirection" | "name"
+> {
+  optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
+  name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
+  choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
+  promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
+}
+
+/**
+ * A record to be evaluated by the tool response handling evaluator.
+ */
+export type ToolResponseHandlingEvaluationRecord = {
+  /**
+   * The user query or conversation context.
+   */
+  input: string;
+  /**
+   * The tool invocation(s) made by the agent, including arguments.
+   */
+  toolCall: string;
+  /**
+   * The tool's response (data, errors, or partial results).
+   */
+  toolResult: string;
+  /**
+   * The agent's handling after receiving the tool result
+   * (may include retries, follow-ups, or final response).
+   */
+  output: string;
+};
+
+/**
+ * Creates a tool response handling evaluator function.
+ *
+ * This function returns an evaluator that determines whether an AI agent properly
+ * handled a tool's response, including error handling, data extraction,
+ * transformation, and safe information disclosure.
+ *
+ * @param args - The arguments for creating the tool response handling evaluator.
+ * @param args.model - The model to use for classification.
+ * @param args.choices - The possible classification choices (defaults to correct/incorrect).
+ * @param args.promptTemplate - The prompt template to use.
+ * @param args.telemetry - The telemetry to use for the evaluator.
+ *
+ * @returns An evaluator function that takes a {@link ToolResponseHandlingEvaluationRecord}
+ * and returns a classification result indicating whether the tool response handling
+ * is correct or incorrect.
+ *
+ * @example
+ * ```ts
+ * const evaluator = createToolResponseHandlingEvaluator({ model: openai("gpt-4o-mini") });
+ *
+ * // Example: Correct extraction from tool result
+ * const result = await evaluator.evaluate({
+ *   input: "What's the weather in Seattle?",
+ *   toolCall: 'get_weather(location="Seattle")',
+ *   toolResult: JSON.stringify({
+ *     temperature: 58,
+ *     unit: "fahrenheit",
+ *     conditions: "partly cloudy"
+ *   }),
+ *   output: "The weather in Seattle is 58°F and partly cloudy."
+ * });
+ * console.log(result.label); // "correct"
+ *
+ * // Example: Hallucinated data (incorrect)
+ * const resultHallucinated = await evaluator.evaluate({
+ *   input: "What restaurants are nearby?",
+ *   toolCall: 'search_restaurants(location="downtown")',
+ *   toolResult: JSON.stringify({
+ *     results: [{ name: "Cafe Luna", rating: 4.2 }]
+ *   }),
+ *   output: "I found Cafe Luna (4.2 stars) and Mario's Italian (4.8 stars) nearby."
+ * });
+ * console.log(resultHallucinated.label); // "incorrect" - Mario's was hallucinated
+ * ```
+ */
+export function createToolResponseHandlingEvaluator<
+  RecordType extends Record<string, unknown> =
+    ToolResponseHandlingEvaluationRecord,
+>(
+  args: ToolResponseHandlingEvaluatorArgs<RecordType>
+): ClassificationEvaluator<RecordType> {
+  const {
+    choices = TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.choices,
+    promptTemplate = TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.template,
+    optimizationDirection = TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection,
+    name = TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.name,
+    ...rest
+  } = args;
+  return createClassificationEvaluator<RecordType>({
+    ...rest,
+    promptTemplate,
+    choices,
+    optimizationDirection,
+    name,
+  });
+}

package/src/llm/createToolSelectionEvaluator.ts

@@ -0,0 +1,92 @@
+import { TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
+import type { CreateClassificationEvaluatorArgs } from "../types/evals";
+import type { ClassificationEvaluator } from "./ClassificationEvaluator";
+import { createClassificationEvaluator } from "./createClassificationEvaluator";
+
+export interface ToolSelectionEvaluatorArgs<
+  RecordType extends Record<string, unknown> = ToolSelectionEvaluationRecord,
+> extends Omit<
+  CreateClassificationEvaluatorArgs<RecordType>,
+  "promptTemplate" | "choices" | "optimizationDirection" | "name"
+> {
+  optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
+  name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
+  choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
+  promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
+}
+
+/**
+ * A record to be evaluated by the tool selection evaluator.
+ */
+export type ToolSelectionEvaluationRecord = {
+  /**
+   * The input query or conversation context.
+   */
+  input: string;
+  /**
+   * The available tools that the LLM could use.
+   */
+  availableTools: string;
+  /**
+   * The tool or tools selected by the LLM.
+   */
+  toolSelection: string;
+};
+
+/**
+ * Creates a tool selection evaluator function.
+ *
+ * This function returns an evaluator that determines whether the correct tool
+ * was selected for a given context. Unlike the tool invocation evaluator which
+ * checks if the tool was called correctly with proper arguments, this evaluator
+ * focuses on whether the right tool was chosen in the first place.
+ *
+ * The evaluator checks for:
+ * - Whether the LLM chose the best available tool for the user query
+ * - Whether the tool name exists in the available tools list
+ * - Whether the correct number of tools were selected for the task
+ * - Whether the tool selection is safe and appropriate
+ *
+ * @param args - The arguments for creating the tool selection evaluator.
+ * @param args.model - The model to use for classification.
+ * @param args.choices - The possible classification choices (defaults to correct/incorrect).
+ * @param args.promptTemplate - The prompt template to use (defaults to TOOL_SELECTION_TEMPLATE).
+ * @param args.telemetry - The telemetry to use for the evaluator.
+ *
+ * @returns An evaluator function that takes a {@link ToolSelectionEvaluationRecord} and returns
+ * a classification result indicating whether the tool selection is correct or incorrect.
+ *
+ * @example
+ * ```ts
+ * const evaluator = createToolSelectionEvaluator({ model: openai("gpt-4o-mini") });
+ *
+ * const result = await evaluator.evaluate({
+ *   input: "User: What is the weather in San Francisco?",
+ *   availableTools: `WeatherTool: Get the current weather for a location.
+ * NewsTool: Stay connected to global events with our up-to-date news around the world.
+ * MusicTool: Create playlists, search for music, and check the latest music trends.`,
+ *   toolSelection: "WeatherTool"
+ * });
+ * console.log(result.label); // "correct" or "incorrect"
+ * ```
+ */
+export function createToolSelectionEvaluator<
+  RecordType extends Record<string, unknown> = ToolSelectionEvaluationRecord,
+>(
+  args: ToolSelectionEvaluatorArgs<RecordType>
+): ClassificationEvaluator<RecordType> {
+  const {
+    choices = TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG.choices,
+    promptTemplate = TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG.template,
+    optimizationDirection = TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection,
+    name = TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG.name,
+    ...rest
+  } = args;
+  return createClassificationEvaluator<RecordType>({
+    ...rest,
+    promptTemplate,
+    choices,
+    optimizationDirection,
+    name,
+  });
+}

package/src/llm/generateClassification.ts

@@ -1,10 +1,10 @@
-import { tracer } from "../telemetry";
-import { ClassificationResult, WithLLM } from "../types/evals";
-import { WithTelemetry } from "../types/otel";
-import type { WithPrompt } from "../types/prompts";
-
 import { generateObject } from "ai";
 import { z } from "zod";
+
+import { tracer } from "../telemetry";
+import type { ClassificationResult, WithLLM } from "../types/evals";
+import type { WithTelemetry } from "../types/otel";
+import type { WithPrompt } from "../types/prompts";
 export type ClassifyArgs = WithLLM &
   WithTelemetry &
   WithPrompt & {

package/src/llm/index.ts

@@ -1,10 +1,13 @@
 export * from "./ClassificationEvaluator";
 export * from "./createClassificationEvaluator";
 export * from "./createClassifierFn";
+export * from "./createConcisenessEvaluator";
 export * from "./createCorrectnessEvaluator";
 export * from "./createDocumentRelevanceEvaluator";
 export * from "./createFaithfulnessEvaluator";
 export * from "./createHallucinationEvaluator"; // Deprecated: use createFaithfulnessEvaluator
 export * from "./createToolInvocationEvaluator";
+export * from "./createToolResponseHandlingEvaluator";
+export * from "./createToolSelectionEvaluator";
 export * from "./generateClassification";
 export * from "./LLMEvaluator";

package/src/template/applyTemplate.ts

@@ -1,9 +1,8 @@
-import { PromptTemplate, RenderedPrompt } from "../types/templating";
+import Mustache from "mustache";
 
+import type { PromptTemplate, RenderedPrompt } from "../types/templating";
 import { createTemplateVariablesProxy } from "./createTemplateVariablesProxy";
 
-import Mustache from "mustache";
-
 /**
  * A function that applies a set of variables to a template (e.g. a prompt)
  * Uses the Mustache library to apply the variables to the template

package/src/template/getTemplateVariables.ts

@@ -1,7 +1,7 @@
-import { PromptTemplate } from "../types/templating";
-
 import Mustache from "mustache";
 
+import type { PromptTemplate } from "../types/templating";
+
 type GetTemplateVariableArgs = {
   template: PromptTemplate;
 };

package/src/types/evals.ts

@@ -1,8 +1,8 @@
-import { ObjectMapping } from "./data";
-import { WithTelemetry } from "./otel";
-import { PromptTemplate } from "./templating";
+import type { LanguageModel } from "ai";
 
-import { LanguageModel } from "ai";
+import type { ObjectMapping } from "./data";
+import type { WithTelemetry } from "./otel";
+import type { PromptTemplate } from "./templating";
 
 /**
  * A specific AI example that is under evaluation

package/src/types/otel.ts

@@ -1,4 +1,4 @@
-import { Tracer } from "@opentelemetry/api";
+import type { Tracer } from "@opentelemetry/api";
 
 export type TelemetryConfig = {
   /**

package/src/utils/bindEvaluator.ts

@@ -1,5 +1,5 @@
-import { EvaluatorBase } from "../core/EvaluatorBase";
-import { ObjectMapping } from "../types/data";
+import type { EvaluatorBase } from "../core/EvaluatorBase";
+import type { ObjectMapping } from "../types/data";
 
 /**
  * Context for binding an evaluator with input mapping configuration.

package/src/utils/objectMappingUtils.ts

@@ -1,7 +1,7 @@
-import { ObjectMapping, ValueGetter } from "../types/data";
-
 import { JSONPath } from "jsonpath-plus";
 
+import type { ObjectMapping, ValueGetter } from "../types/data";
+
 /**
  * Remaps an object by applying field mappings while preserving original data.
  *