@arizeai/phoenix-evals 0.7.0 → 0.8.0

package/src/__generated__/default_templates/FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.ts

@@ -0,0 +1,35 @@
+// This file is generated. Do not edit by hand.
+
+import type { ClassificationEvaluatorConfig } from "../types";
+
+export const FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG: ClassificationEvaluatorConfig = {
+  name: "faithfulness",
+  description: "A specialized evaluator for detecting faithfulness in grounded LLM responses.",
+  optimizationDirection: "MAXIMIZE",
+  template: [
+    {
+      role: "user",
+      content: `
+In this task, you will be presented with a query, some context and a response. The response is generated to the question based on the context. The response may contain false information. You must use the context to determine if the response to the question contains false information, if the response is unfaithful to the facts. Your objective is to determine whether the response text contains factual information and is faithful to the context. An 'unfaithful' response refers to a response that is not based on the context or assumes information that is not available in the context. Your response should be a single word: either 'faithful' or 'unfaithful', and it should not include any other text or characters. 'unfaithful' indicates that the response provides factually inaccurate information to the query based on the context. 'faithful' indicates that the response to the question is correct relative to the context, and does not contain made up information. Please read the query and context carefully before determining your response.
+
+<data>
+<query>
+{{input}}
+</query>
+<context>
+{{context}}
+</context>
+<response>
+{{output}}
+</response>
+</data>
+
+Is the response above faithful or unfaithful based on the query and context?
+`,
+    },
+  ],
+  choices: {
+  "faithful": 1,
+  "unfaithful": 0
+},
+};

package/src/__generated__/default_templates/TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.ts

@@ -0,0 +1,32 @@
+// This file is generated. Do not edit by hand.
+
+import type { ClassificationEvaluatorConfig } from "../types";
+
+export const TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG: ClassificationEvaluatorConfig = {
+  name: "tool_invocation",
+  description: "For determining if a tool was invoked correctly with proper arguments, formatting, and safe content. Requires conversation context, available tool schemas, and the LLM's tool invocation(s).",
+  optimizationDirection: "MAXIMIZE",
+  template: [
+    {
+      role: "user",
+      content: `
+You are an impartial judge evaluating an LLM's tool-calling behavior, specifically whether the LLM invoked a tool (or tools) correctly with valid arguments and proper formatting.
+Your task: Determine whether the LLM's tool invocation(s) were correct or incorrect based on: - The full conversation context (including all previous turns, not just the most recent message) - The available tool schemas - The LLM's tool invocation(s) with arguments
+IMPORTANT - Tool Invocation vs. Tool Selection: - You are ONLY evaluating the tool invocation, not the tool selection. - If the tool selection is incorrect or not relevant to the user's query, but the tool invocation is correct, return "correct". - If the tool selection is correct but the tool invocation is incorrect, return "incorrect".
+IMPORTANT - Multi-Tool Invocations: - The LLM may invoke MULTIPLE tools in a single response. This is valid and expected for complex requests. - When multiple tools are invoked, evaluate EACH tool invocation independently. - Return "correct" only if ALL tool invocations are correct. - Return "incorrect" if ANY tool invocation has an error.
+IMPORTANT - Conversation Context: - Read the entire conversation history carefully, not just the final user message. - Argument values may need to be extracted from EARLIER turns in the conversation (e.g., user mentions a location, date, or quantity in a previous message). - The LLM should use context from the full conversation to populate argument values correctly.
+Criteria Return "correct" only when ALL of the following are true for EVERY tool invocation: - JSON is properly structured (if applicable). - All required fields/parameters are present. - No hallucinated or nonexistent fields (all fields exist in the tool schema). - Argument values match the user's intent from the conversation context (correct types, realistic values). - No unsafe content (e.g., PII like SSNs, credit card numbers, passwords) in arguments.
+Return "incorrect" if ANY of the following are true for ANY tool invocation: - The invocation contains hallucinated or nonexistent fields not in the schema. - Required fields/parameters are missing. - JSON is improperly formatted or malformed. - Argument values are incorrect, hallucinated, or do not match user intent from the conversation. - Arguments contain unsafe content (e.g., PII, sensitive data that should not be passed).
+Before providing your final judgment, explain your reasoning and consider: - How many tools were invoked? Evaluate each one. - Does each tool invocation match the schema for that tool? - Are all required parameters provided with appropriate values for each invocation? - Are there any extra fields that don't exist in the schema? - Looking at the FULL conversation: do the argument values accurately reflect what the user requested across all messages? - Is there any unsafe or sensitive content in any of the arguments? - Check that you are not evaluating the tool selection, only the tool invocation.
+<data> <context> {{input}} </context>
+<available_tools> {{availableTools}} </available_tools>
+<tool_invocation> {{toolSelection}} </tool_invocation> </data>
+Given the above data, is the tool invocation correct or incorrect?
+`,
+    },
+  ],
+  choices: {
+  "correct": 1,
+  "incorrect": 0
+},
+};

package/src/__generated__/default_templates/index.ts

@@ -2,5 +2,7 @@
 
 export { CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG } from "./CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG";
 export { DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG } from "./DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG";
+export { FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG } from "./FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG";
 export { HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG } from "./HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG";
+export { TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG";
 export { TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG";

package/src/core/EvaluatorBase.ts

@@ -11,9 +11,9 @@ import { ObjectMapping } from "../types/data";
 /**
  * Base class for all evaluators
  */
-export abstract class EvaluatorBase<RecordType extends Record<string, unknown>>
-  implements EvaluatorInterface<RecordType>
-{
+export abstract class EvaluatorBase<
+  RecordType extends Record<string, unknown>,
+> implements EvaluatorInterface<RecordType> {
   readonly name: string;
   readonly kind: EvaluationKind;
   readonly optimizationDirection?: OptimizationDirection;

package/src/llm/createCorrectnessEvaluator.ts

@@ -7,9 +7,9 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator";
 export interface CorrectnessEvaluatorArgs<
   RecordType extends Record<string, unknown> = CorrectnessEvaluationRecord,
 > extends Omit<
-    CreateClassificationEvaluatorArgs<RecordType>,
-    "promptTemplate" | "choices" | "optimizationDirection" | "name"
-  > {
+  CreateClassificationEvaluatorArgs<RecordType>,
+  "promptTemplate" | "choices" | "optimizationDirection" | "name"
+> {
   optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
   name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
   choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];

package/src/llm/createDocumentRelevanceEvaluator.ts

@@ -5,14 +5,12 @@ import { ClassificationEvaluator } from "./ClassificationEvaluator";
 import { createClassificationEvaluator } from "./createClassificationEvaluator";
 
 export interface DocumentRelevanceEvaluatorArgs<
-  RecordType extends Record<
-    string,
-    unknown
-  > = DocumentRelevanceEvaluationRecord,
+  RecordType extends Record<string, unknown> =
+    DocumentRelevanceEvaluationRecord,
 > extends Omit<
-    CreateClassificationEvaluatorArgs<RecordType>,
-    "promptTemplate" | "choices" | "optimizationDirection" | "name"
-  > {
+  CreateClassificationEvaluatorArgs<RecordType>,
+  "promptTemplate" | "choices" | "optimizationDirection" | "name"
+> {
   optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
   name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
   choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
@@ -55,10 +53,8 @@ export interface DocumentRelevanceEvaluationRecord {
  * ```
  */
 export function createDocumentRelevanceEvaluator<
-  RecordType extends Record<
-    string,
-    unknown
-  > = DocumentRelevanceEvaluationRecord,
+  RecordType extends Record<string, unknown> =
+    DocumentRelevanceEvaluationRecord,
 >(
   args: DocumentRelevanceEvaluatorArgs<RecordType>
 ): ClassificationEvaluator<RecordType> {

package/src/llm/createFaithfulnessEvaluator.ts

@@ -0,0 +1,52 @@
+import { FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
+import { CreateClassificationEvaluatorArgs } from "../types/evals";
+
+import { ClassificationEvaluator } from "./ClassificationEvaluator";
+import { createClassificationEvaluator } from "./createClassificationEvaluator";
+
+export interface FaithfulnessEvaluatorArgs<
+  RecordType extends Record<string, unknown> = FaithfulnessEvaluationRecord,
+> extends Omit<
+  CreateClassificationEvaluatorArgs<RecordType>,
+  "promptTemplate" | "choices" | "optimizationDirection" | "name"
+> {
+  optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
+  name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
+  choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
+  promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
+}
+
+/**
+ * A record to be evaluated by the faithfulness evaluator.
+ */
+export type FaithfulnessEvaluationRecord = {
+  input: string;
+  output: string;
+  context?: string;
+};
+/**
+ * Creates a function that evaluates whether an answer is faithful or unfaithful based on a query and reference text.
+ *
+ * @param args - The arguments for creating the faithfulness evaluator.
+ * @returns A function that evaluates whether an answer is faithful or unfaithful based on a query and reference text.
+ */
+export function createFaithfulnessEvaluator<
+  RecordType extends Record<string, unknown> = FaithfulnessEvaluationRecord,
+>(
+  args: FaithfulnessEvaluatorArgs<RecordType>
+): ClassificationEvaluator<RecordType> {
+  const {
+    choices = FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.choices,
+    promptTemplate = FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.template,
+    optimizationDirection = FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection,
+    name = FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.name,
+    ...rest
+  } = args;
+  return createClassificationEvaluator<RecordType>({
+    ...rest,
+    promptTemplate,
+    choices,
+    optimizationDirection,
+    name,
+  });
+}

package/src/llm/createHallucinationEvaluator.ts

@@ -1,3 +1,10 @@
+/**
+ * @deprecated This evaluator is maintained for backwards compatibility.
+ * Please use createFaithfulnessEvaluator instead, which uses updated terminology:
+ * - 'faithful'/'unfaithful' labels instead of 'factual'/'hallucinated'
+ * - Maximizes score (1.0=faithful) instead of minimizing it
+ */
+
 import { HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
 import { CreateClassificationEvaluatorArgs } from "../types/evals";
 
@@ -7,9 +14,9 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator";
 export interface HallucinationEvaluatorArgs<
   RecordType extends Record<string, unknown> = HallucinationEvaluationRecord,
 > extends Omit<
-    CreateClassificationEvaluatorArgs<RecordType>,
-    "promptTemplate" | "choices" | "optimizationDirection" | "name"
-  > {
+  CreateClassificationEvaluatorArgs<RecordType>,
+  "promptTemplate" | "choices" | "optimizationDirection" | "name"
+> {
   optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
   name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
   choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
@@ -24,9 +31,16 @@ export type HallucinationEvaluationRecord = {
   output: string;
   context?: string;
 };
+
 /**
+ * @deprecated Use createFaithfulnessEvaluator instead.
+ *
  * Creates a function that evaluates whether an answer is factual or hallucinated based on a query and reference text.
  *
+ * Note: This is deprecated. Please use createFaithfulnessEvaluator which:
+ * - Uses 'faithful'/'unfaithful' labels instead of 'factual'/'hallucinated'
+ * - Maximizes the score (1.0 for faithful, 0.0 for unfaithful)
+ *
  * @param args - The arguments for creating the hallucination evaluator.
  * @returns A function that evaluates whether an answer is factual or hallucinated based on a query and reference text.
  */
@@ -35,6 +49,14 @@ export function createHallucinationEvaluator<
 >(
   args: HallucinationEvaluatorArgs<RecordType>
 ): ClassificationEvaluator<RecordType> {
+  // eslint-disable-next-line no-console
+  console.warn(
+    "createHallucinationEvaluator is deprecated and will be removed in a future version. " +
+      "Please use createFaithfulnessEvaluator instead. The new evaluator uses " +
+      "'faithful'/'unfaithful' labels and maximizes score (1.0=faithful) instead of " +
+      "minimizing it (0.0=factual)."
+  );
+
   const {
     choices = HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.choices,
     promptTemplate = HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.template,

package/src/llm/createToolInvocationEvaluator.ts

@@ -0,0 +1,103 @@
+import { TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
+import { CreateClassificationEvaluatorArgs } from "../types/evals";
+
+import { ClassificationEvaluator } from "./ClassificationEvaluator";
+import { createClassificationEvaluator } from "./createClassificationEvaluator";
+
+export interface ToolInvocationEvaluatorArgs<
+  RecordType extends Record<string, unknown> = ToolInvocationEvaluationRecord,
+> extends Omit<
+  CreateClassificationEvaluatorArgs<RecordType>,
+  "promptTemplate" | "choices" | "optimizationDirection" | "name"
+> {
+  optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
+  name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
+  choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
+  promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
+}
+
+/**
+ * A record to be evaluated by the tool invocation evaluator.
+ */
+export type ToolInvocationEvaluationRecord = {
+  /**
+   * The input query or conversation context.
+   */
+  input: string;
+  /**
+   * The available tool schemas, either as JSON schema or human-readable format.
+   */
+  availableTools: string;
+  /**
+   * The tool invocation(s) made by the LLM, including arguments.
+   */
+  toolSelection: string;
+};
+
+/**
+ * Creates a tool invocation evaluator function.
+ *
+ * This function returns an evaluator that determines whether a tool was invoked
+ * correctly with proper arguments, formatting, and safe content.
+ *
+ * The evaluator checks for:
+ * - Properly structured JSON (if applicable)
+ * - All required fields/parameters present
+ * - No hallucinated or nonexistent fields
+ * - Argument values matching user query and schema expectations
+ * - No unsafe content (e.g., PII) in arguments
+ *
+ * @param args - The arguments for creating the tool invocation evaluator.
+ * @param args.model - The model to use for classification.
+ * @param args.choices - The possible classification choices (defaults to correct/incorrect).
+ * @param args.promptTemplate - The prompt template to use (defaults to TOOL_INVOCATION_TEMPLATE).
+ * @param args.telemetry - The telemetry to use for the evaluator.
+ *
+ * @returns An evaluator function that takes a {@link ToolInvocationEvaluationRecord} and returns
+ * a classification result indicating whether the tool invocation is correct or incorrect.
+ *
+ * @example
+ * ```ts
+ * const evaluator = createToolInvocationEvaluator({ model: openai("gpt-4o-mini") });
+ *
+ * // Example with JSON schema format for available tools
+ * const result = await evaluator.evaluate({
+ *   input: "User: Book a flight from NYC to LA for tomorrow",
+ *   availableTools: JSON.stringify({
+ *     name: "book_flight",
+ *     description: "Book a flight between two cities",
+ *     parameters: {
+ *       type: "object",
+ *       properties: {
+ *         origin: { type: "string", description: "Departure city code" },
+ *         destination: { type: "string", description: "Arrival city code" },
+ *         date: { type: "string", description: "Flight date in YYYY-MM-DD" }
+ *       },
+ *       required: ["origin", "destination", "date"]
+ *     }
+ *   }),
+ *   toolSelection: 'book_flight(origin="NYC", destination="LA", date="2024-01-15")'
+ * });
+ * console.log(result.label); // "correct" or "incorrect"
+ * ```
+ */
+export function createToolInvocationEvaluator<
+  RecordType extends Record<string, unknown> = ToolInvocationEvaluationRecord,
+>(
+  args: ToolInvocationEvaluatorArgs<RecordType>
+): ClassificationEvaluator<RecordType> {
+  const {
+    choices = TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.choices,
+    promptTemplate = TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.template,
+    optimizationDirection = TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection,
+    name = TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.name,
+    ...rest
+  } = args;
+  return createClassificationEvaluator<RecordType>({
+    ...rest,
+    promptTemplate,
+    choices,
+    optimizationDirection,
+    name,
+  });
+}

package/src/llm/index.ts

@@ -1,8 +1,10 @@
-export * from "./generateClassification";
-export * from "./createClassifierFn";
+export * from "./ClassificationEvaluator";
 export * from "./createClassificationEvaluator";
-export * from "./createHallucinationEvaluator";
-export * from "./createDocumentRelevanceEvaluator";
+export * from "./createClassifierFn";
 export * from "./createCorrectnessEvaluator";
-export * from "./ClassificationEvaluator";
+export * from "./createDocumentRelevanceEvaluator";
+export * from "./createFaithfulnessEvaluator";
+export * from "./createHallucinationEvaluator"; // Deprecated: use createFaithfulnessEvaluator
+export * from "./createToolInvocationEvaluator";
+export * from "./generateClassification";
 export * from "./LLMEvaluator";

package/src/types/evals.ts

@@ -110,8 +110,8 @@ export type CreateLLMEvaluatorArgs<RecordType extends Record<string, unknown>> =
 
 export interface CreateClassificationEvaluatorArgs<
   RecordType extends Record<string, unknown>,
-> extends CreateClassifierArgs,
-    CreateLLMEvaluatorArgs<RecordType> {
+>
+  extends CreateClassifierArgs, CreateLLMEvaluatorArgs<RecordType> {
   /**
    * The prompt template to use for classification
    */
@@ -156,8 +156,9 @@ interface EvaluatorDescription {
  * The Base Evaluator interface
  * This is the interface that all evaluators must implement
  */
-export interface EvaluatorInterface<ExampleType extends Record<string, unknown>>
-  extends EvaluatorDescription {
+export interface EvaluatorInterface<
+  ExampleType extends Record<string, unknown>,
+> extends EvaluatorDescription {
   /**
    * The function that evaluates the example
    */