@arizeai/phoenix-evals 0.8.0 → 0.9.0

package/dist/esm/__generated__/default_templates/TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.d.ts

@@ -0,0 +1,3 @@
+import type { ClassificationEvaluatorConfig } from "../types.js";
+export declare const TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG: ClassificationEvaluatorConfig;
+//# sourceMappingURL=TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map

package/dist/esm/__generated__/default_templates/TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,sDAAsD,EAAE,6BAmFpE,CAAC"}

package/dist/esm/__generated__/default_templates/TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.js

@@ -0,0 +1,86 @@
+// This file is generated. Do not edit by hand.
+export const TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG = {
+    name: "tool_response_handling",
+    description: "For determining if an AI agent properly handled a tool's response, including error handling, data extraction, transformation, and safe information disclosure. Requires conversation context, the tool call(s), the tool result(s), and the agent's output.",
+    optimizationDirection: "MAXIMIZE",
+    template: [
+        {
+            role: "user",
+            content: `
+You are an impartial judge evaluating an AI agent's handling of a tool's response. Your task is to determine whether the agent correctly processed the tool result to produce an appropriate output.
+
+IMPORTANT - Scope of Evaluation:
+- You are ONLY evaluating how the agent handled the tool response, NOT whether the right tool was selected or whether the tool was invoked correctly.
+- This evaluation focuses on what happens AFTER the tool returns a result.
+
+IMPORTANT - Multi-Tool Handling:
+- The agent may make MULTIPLE tool calls in a single interaction. This is valid and expected.
+- When multiple tools are called, evaluate how the agent handled ALL tool results together.
+- Return "correct" only if the agent properly handled ALL tool results.
+- Return "incorrect" if the agent mishandled ANY tool result.
+
+IMPORTANT - Error Response Handling:
+- Tool results may contain errors (rate limits, timeouts, not found, invalid arguments, etc.).
+- The agent's output may include retries, follow-up tool calls, or a final response to the user.
+- Evaluate the ENTIRE handling sequence, not just the final message.
+- Appropriate error handling includes:
+  - Retrying on transient errors (rate limits, timeouts)
+  - Correcting arguments after invalid argument errors
+  - Informing the user appropriately when errors are not recoverable
+  - NOT making repeated identical calls that continue to fail
+
+Criteria for CORRECT handling:
+- Data is extracted accurately from the tool result (no hallucination of data that wasn't returned)
+- Dates, numbers, and structured fields are properly transformed and formatted
+- Results are accurately summarized to address the user's original query
+- Error responses are handled appropriately (retries for transient errors, corrections for invalid arguments)
+- No repeated identical calls after non-retryable errors
+- No disclosure of sensitive/internal information (database credentials, internal URLs, PII, API keys, etc.)
+- The agent's response actually uses the tool result rather than ignoring it
+
+Criteria for INCORRECT handling:
+- Hallucinated data: The output includes information not present in the tool result
+- Misinterpretation: The meaning of the tool result is misrepresented or reversed
+- Improper transformation: Dates, numbers, or structured data are incorrectly converted
+- Missing retry: Failed to retry on retryable errors (rate limits, timeouts)
+- Missing correction: Failed to correct arguments after invalid argument errors
+- Futile retries: Repeated identical calls that continue to fail
+- Information disclosure: Leaked sensitive information (credentials, internal URLs, PII)
+- Ignored results: The agent's response doesn't incorporate the tool result
+- Incomplete handling: Only some tool results are used when multiple tools were called
+
+Before providing your final judgment, explain your reasoning and consider:
+- Does the output accurately reflect what the tool returned?
+- Are there any fabricated details not in the tool result?
+- Were errors handled appropriately?
+- Is sensitive information properly protected?
+- Does the output actually address the user's query using the tool data?
+
+<data>
+<input>
+{{input}}
+</input>
+
+<tool_call>
+{{toolCall}}
+</tool_call>
+
+<tool_result>
+{{toolResult}}
+</tool_result>
+
+<output>
+{{output}}
+</output>
+</data>
+
+Given the above data, did the agent handle the tool response correctly or incorrectly?
+`,
+        },
+    ],
+    choices: {
+        "correct": 1,
+        "incorrect": 0
+    },
+};
+//# sourceMappingURL=TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.js.map

package/dist/esm/__generated__/default_templates/TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAAA,+CAA+C;AAI/C,MAAM,CAAC,MAAM,sDAAsD,GAAkC;IACnG,IAAI,EAAE,wBAAwB;IAC9B,WAAW,EAAE,6PAA6P;IAC1Q,qBAAqB,EAAE,UAAU;IACjC,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqEd;SACI;KACF;IACD,OAAO,EAAE;QACT,SAAS,EAAE,CAAC;QACZ,WAAW,EAAE,CAAC;KACf;CACA,CAAC"}

package/dist/esm/__generated__/default_templates/index.d.ts

@@ -3,5 +3,6 @@ export { DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG } from "./DOCUMENT_R
 export { FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG } from "./FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.js";
 export { HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG } from "./HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js";
 export { TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.js";
+export { TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.js";
 export { TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/esm/__generated__/default_templates/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,kDAAkD,EAAE,MAAM,sDAAsD,CAAC;AAC1H,OAAO,EAAE,4CAA4C,EAAE,MAAM,gDAAgD,CAAC;AAC9G,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC;AAChH,OAAO,EAAE,+CAA+C,EAAE,MAAM,mDAAmD,CAAC;AACpH,OAAO,EAAE,8CAA8C,EAAE,MAAM,kDAAkD,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,kDAAkD,EAAE,MAAM,sDAAsD,CAAC;AAC1H,OAAO,EAAE,4CAA4C,EAAE,MAAM,gDAAgD,CAAC;AAC9G,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC;AAChH,OAAO,EAAE,+CAA+C,EAAE,MAAM,mDAAmD,CAAC;AACpH,OAAO,EAAE,sDAAsD,EAAE,MAAM,0DAA0D,CAAC;AAClI,OAAO,EAAE,8CAA8C,EAAE,MAAM,kDAAkD,CAAC"}

package/dist/esm/__generated__/default_templates/index.js

@@ -4,5 +4,6 @@ export { DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG } from "./DOCUMENT_R
 export { FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG } from "./FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.js";
 export { HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG } from "./HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js";
 export { TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.js";
+export { TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.js";
 export { TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG.js";
 //# sourceMappingURL=index.js.map

package/dist/esm/__generated__/default_templates/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/index.ts"],"names":[],"mappings":"AAAA,+CAA+C;AAE/C,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,kDAAkD,EAAE,MAAM,sDAAsD,CAAC;AAC1H,OAAO,EAAE,4CAA4C,EAAE,MAAM,gDAAgD,CAAC;AAC9G,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC;AAChH,OAAO,EAAE,+CAA+C,EAAE,MAAM,mDAAmD,CAAC;AACpH,OAAO,EAAE,8CAA8C,EAAE,MAAM,kDAAkD,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/index.ts"],"names":[],"mappings":"AAAA,+CAA+C;AAE/C,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,kDAAkD,EAAE,MAAM,sDAAsD,CAAC;AAC1H,OAAO,EAAE,4CAA4C,EAAE,MAAM,gDAAgD,CAAC;AAC9G,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC;AAChH,OAAO,EAAE,+CAA+C,EAAE,MAAM,mDAAmD,CAAC;AACpH,OAAO,EAAE,sDAAsD,EAAE,MAAM,0DAA0D,CAAC;AAClI,OAAO,EAAE,8CAA8C,EAAE,MAAM,kDAAkD,CAAC"}

package/dist/esm/llm/createToolResponseHandlingEvaluator.d.ts

@@ -0,0 +1,78 @@
+import { CreateClassificationEvaluatorArgs } from "../types/evals.js";
+import { ClassificationEvaluator } from "./ClassificationEvaluator.js";
+export interface ToolResponseHandlingEvaluatorArgs<RecordType extends Record<string, unknown> = ToolResponseHandlingEvaluationRecord> extends Omit<CreateClassificationEvaluatorArgs<RecordType>, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
+    optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
+    name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
+    choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
+    promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
+}
+/**
+ * A record to be evaluated by the tool response handling evaluator.
+ */
+export type ToolResponseHandlingEvaluationRecord = {
+    /**
+     * The user query or conversation context.
+     */
+    input: string;
+    /**
+     * The tool invocation(s) made by the agent, including arguments.
+     */
+    toolCall: string;
+    /**
+     * The tool's response (data, errors, or partial results).
+     */
+    toolResult: string;
+    /**
+     * The agent's handling after receiving the tool result
+     * (may include retries, follow-ups, or final response).
+     */
+    output: string;
+};
+/**
+ * Creates a tool response handling evaluator function.
+ *
+ * This function returns an evaluator that determines whether an AI agent properly
+ * handled a tool's response, including error handling, data extraction,
+ * transformation, and safe information disclosure.
+ *
+ * @param args - The arguments for creating the tool response handling evaluator.
+ * @param args.model - The model to use for classification.
+ * @param args.choices - The possible classification choices (defaults to correct/incorrect).
+ * @param args.promptTemplate - The prompt template to use.
+ * @param args.telemetry - The telemetry to use for the evaluator.
+ *
+ * @returns An evaluator function that takes a {@link ToolResponseHandlingEvaluationRecord}
+ * and returns a classification result indicating whether the tool response handling
+ * is correct or incorrect.
+ *
+ * @example
+ * ```ts
+ * const evaluator = createToolResponseHandlingEvaluator({ model: openai("gpt-4o-mini") });
+ *
+ * // Example: Correct extraction from tool result
+ * const result = await evaluator.evaluate({
+ *   input: "What's the weather in Seattle?",
+ *   toolCall: 'get_weather(location="Seattle")',
+ *   toolResult: JSON.stringify({
+ *     temperature: 58,
+ *     unit: "fahrenheit",
+ *     conditions: "partly cloudy"
+ *   }),
+ *   output: "The weather in Seattle is 58°F and partly cloudy."
+ * });
+ * console.log(result.label); // "correct"
+ *
+ * // Example: Hallucinated data (incorrect)
+ * const resultHallucinated = await evaluator.evaluate({
+ *   input: "What restaurants are nearby?",
+ *   toolCall: 'search_restaurants(location="downtown")',
+ *   toolResult: JSON.stringify({
+ *     results: [{ name: "Cafe Luna", rating: 4.2 }]
+ *   }),
+ *   output: "I found Cafe Luna (4.2 stars) and Mario's Italian (4.8 stars) nearby."
+ * });
+ * console.log(resultHallucinated.label); // "incorrect" - Mario's was hallucinated
+ * ```
+ */
+export declare function createToolResponseHandlingEvaluator<RecordType extends Record<string, unknown> = ToolResponseHandlingEvaluationRecord>(args: ToolResponseHandlingEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType>;
+//# sourceMappingURL=createToolResponseHandlingEvaluator.d.ts.map

package/dist/esm/llm/createToolResponseHandlingEvaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"createToolResponseHandlingEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createToolResponseHandlingEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGpE,MAAM,WAAW,iCAAiC,CAChD,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,oCAAoC,CACtC,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED;;GAEG;AACH,MAAM,MAAM,oCAAoC,GAAG;IACjD;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AACH,wBAAgB,mCAAmC,CACjD,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,oCAAoC,EAEtC,IAAI,EAAE,iCAAiC,CAAC,UAAU,CAAC,GAClD,uBAAuB,CAAC,UAAU,CAAC,CAerC"}

package/dist/esm/llm/createToolResponseHandlingEvaluator.js

@@ -0,0 +1,59 @@
+import { TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates/index.js";
+import { createClassificationEvaluator } from "./createClassificationEvaluator.js";
+/**
+ * Creates a tool response handling evaluator function.
+ *
+ * This function returns an evaluator that determines whether an AI agent properly
+ * handled a tool's response, including error handling, data extraction,
+ * transformation, and safe information disclosure.
+ *
+ * @param args - The arguments for creating the tool response handling evaluator.
+ * @param args.model - The model to use for classification.
+ * @param args.choices - The possible classification choices (defaults to correct/incorrect).
+ * @param args.promptTemplate - The prompt template to use.
+ * @param args.telemetry - The telemetry to use for the evaluator.
+ *
+ * @returns An evaluator function that takes a {@link ToolResponseHandlingEvaluationRecord}
+ * and returns a classification result indicating whether the tool response handling
+ * is correct or incorrect.
+ *
+ * @example
+ * ```ts
+ * const evaluator = createToolResponseHandlingEvaluator({ model: openai("gpt-4o-mini") });
+ *
+ * // Example: Correct extraction from tool result
+ * const result = await evaluator.evaluate({
+ *   input: "What's the weather in Seattle?",
+ *   toolCall: 'get_weather(location="Seattle")',
+ *   toolResult: JSON.stringify({
+ *     temperature: 58,
+ *     unit: "fahrenheit",
+ *     conditions: "partly cloudy"
+ *   }),
+ *   output: "The weather in Seattle is 58°F and partly cloudy."
+ * });
+ * console.log(result.label); // "correct"
+ *
+ * // Example: Hallucinated data (incorrect)
+ * const resultHallucinated = await evaluator.evaluate({
+ *   input: "What restaurants are nearby?",
+ *   toolCall: 'search_restaurants(location="downtown")',
+ *   toolResult: JSON.stringify({
+ *     results: [{ name: "Cafe Luna", rating: 4.2 }]
+ *   }),
+ *   output: "I found Cafe Luna (4.2 stars) and Mario's Italian (4.8 stars) nearby."
+ * });
+ * console.log(resultHallucinated.label); // "incorrect" - Mario's was hallucinated
+ * ```
+ */
+export function createToolResponseHandlingEvaluator(args) {
+    const { choices = TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.choices, promptTemplate = TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.template, optimizationDirection = TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection, name = TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.name, ...rest } = args;
+    return createClassificationEvaluator({
+        ...rest,
+        promptTemplate,
+        choices,
+        optimizationDirection,
+        name,
+    });
+}
+//# sourceMappingURL=createToolResponseHandlingEvaluator.js.map

package/dist/esm/llm/createToolResponseHandlingEvaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createToolResponseHandlingEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createToolResponseHandlingEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sDAAsD,EAAE,MAAM,oCAAoC,CAAC;AAI5G,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAsChF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AACH,MAAM,UAAU,mCAAmC,CAIjD,IAAmD;IAEnD,MAAM,EACJ,OAAO,GAAG,sDAAsD,CAAC,OAAO,EACxE,cAAc,GAAG,sDAAsD,CAAC,QAAQ,EAChF,qBAAqB,GAAG,sDAAsD,CAAC,qBAAqB,EACpG,IAAI,GAAG,sDAAsD,CAAC,IAAI,EAClE,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}

package/dist/esm/llm/createToolSelectionEvaluator.d.ts

@@ -0,0 +1,64 @@
+import { CreateClassificationEvaluatorArgs } from "../types/evals.js";
+import { ClassificationEvaluator } from "./ClassificationEvaluator.js";
+export interface ToolSelectionEvaluatorArgs<RecordType extends Record<string, unknown> = ToolSelectionEvaluationRecord> extends Omit<CreateClassificationEvaluatorArgs<RecordType>, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
+    optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
+    name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
+    choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
+    promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
+}
+/**
+ * A record to be evaluated by the tool selection evaluator.
+ */
+export type ToolSelectionEvaluationRecord = {
+    /**
+     * The input query or conversation context.
+     */
+    input: string;
+    /**
+     * The available tools that the LLM could use.
+     */
+    availableTools: string;
+    /**
+     * The tool or tools selected by the LLM.
+     */
+    toolSelection: string;
+};
+/**
+ * Creates a tool selection evaluator function.
+ *
+ * This function returns an evaluator that determines whether the correct tool
+ * was selected for a given context. Unlike the tool invocation evaluator which
+ * checks if the tool was called correctly with proper arguments, this evaluator
+ * focuses on whether the right tool was chosen in the first place.
+ *
+ * The evaluator checks for:
+ * - Whether the LLM chose the best available tool for the user query
+ * - Whether the tool name exists in the available tools list
+ * - Whether the correct number of tools were selected for the task
+ * - Whether the tool selection is safe and appropriate
+ *
+ * @param args - The arguments for creating the tool selection evaluator.
+ * @param args.model - The model to use for classification.
+ * @param args.choices - The possible classification choices (defaults to correct/incorrect).
+ * @param args.promptTemplate - The prompt template to use (defaults to TOOL_SELECTION_TEMPLATE).
+ * @param args.telemetry - The telemetry to use for the evaluator.
+ *
+ * @returns An evaluator function that takes a {@link ToolSelectionEvaluationRecord} and returns
+ * a classification result indicating whether the tool selection is correct or incorrect.
+ *
+ * @example
+ * ```ts
+ * const evaluator = createToolSelectionEvaluator({ model: openai("gpt-4o-mini") });
+ *
+ * const result = await evaluator.evaluate({
+ *   input: "User: What is the weather in San Francisco?",
+ *   availableTools: `WeatherTool: Get the current weather for a location.
+ * NewsTool: Stay connected to global events with our up-to-date news around the world.
+ * MusicTool: Create playlists, search for music, and check the latest music trends.`,
+ *   toolSelection: "WeatherTool"
+ * });
+ * console.log(result.label); // "correct" or "incorrect"
+ * ```
+ */
+export declare function createToolSelectionEvaluator<RecordType extends Record<string, unknown> = ToolSelectionEvaluationRecord>(args: ToolSelectionEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType>;
+//# sourceMappingURL=createToolSelectionEvaluator.d.ts.map

package/dist/esm/llm/createToolSelectionEvaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"createToolSelectionEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createToolSelectionEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGpE,MAAM,WAAW,0BAA0B,CACzC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,6BAA6B,CAC1E,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED;;GAEG;AACH,MAAM,MAAM,6BAA6B,GAAG;IAC1C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,wBAAgB,4BAA4B,CAC1C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,6BAA6B,EAE1E,IAAI,EAAE,0BAA0B,CAAC,UAAU,CAAC,GAC3C,uBAAuB,CAAC,UAAU,CAAC,CAerC"}

package/dist/esm/llm/createToolSelectionEvaluator.js

@@ -0,0 +1,50 @@
+import { TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates/index.js";
+import { createClassificationEvaluator } from "./createClassificationEvaluator.js";
+/**
+ * Creates a tool selection evaluator function.
+ *
+ * This function returns an evaluator that determines whether the correct tool
+ * was selected for a given context. Unlike the tool invocation evaluator which
+ * checks if the tool was called correctly with proper arguments, this evaluator
+ * focuses on whether the right tool was chosen in the first place.
+ *
+ * The evaluator checks for:
+ * - Whether the LLM chose the best available tool for the user query
+ * - Whether the tool name exists in the available tools list
+ * - Whether the correct number of tools were selected for the task
+ * - Whether the tool selection is safe and appropriate
+ *
+ * @param args - The arguments for creating the tool selection evaluator.
+ * @param args.model - The model to use for classification.
+ * @param args.choices - The possible classification choices (defaults to correct/incorrect).
+ * @param args.promptTemplate - The prompt template to use (defaults to TOOL_SELECTION_TEMPLATE).
+ * @param args.telemetry - The telemetry to use for the evaluator.
+ *
+ * @returns An evaluator function that takes a {@link ToolSelectionEvaluationRecord} and returns
+ * a classification result indicating whether the tool selection is correct or incorrect.
+ *
+ * @example
+ * ```ts
+ * const evaluator = createToolSelectionEvaluator({ model: openai("gpt-4o-mini") });
+ *
+ * const result = await evaluator.evaluate({
+ *   input: "User: What is the weather in San Francisco?",
+ *   availableTools: `WeatherTool: Get the current weather for a location.
+ * NewsTool: Stay connected to global events with our up-to-date news around the world.
+ * MusicTool: Create playlists, search for music, and check the latest music trends.`,
+ *   toolSelection: "WeatherTool"
+ * });
+ * console.log(result.label); // "correct" or "incorrect"
+ * ```
+ */
+export function createToolSelectionEvaluator(args) {
+    const { choices = TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG.choices, promptTemplate = TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG.template, optimizationDirection = TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection, name = TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG.name, ...rest } = args;
+    return createClassificationEvaluator({
+        ...rest,
+        promptTemplate,
+        choices,
+        optimizationDirection,
+        name,
+    });
+}
+//# sourceMappingURL=createToolSelectionEvaluator.js.map

package/dist/esm/llm/createToolSelectionEvaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createToolSelectionEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createToolSelectionEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,8CAA8C,EAAE,MAAM,oCAAoC,CAAC;AAIpG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAgChF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,MAAM,UAAU,4BAA4B,CAG1C,IAA4C;IAE5C,MAAM,EACJ,OAAO,GAAG,8CAA8C,CAAC,OAAO,EAChE,cAAc,GAAG,8CAA8C,CAAC,QAAQ,EACxE,qBAAqB,GAAG,8CAA8C,CAAC,qBAAqB,EAC5F,IAAI,GAAG,8CAA8C,CAAC,IAAI,EAC1D,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}

package/dist/esm/llm/index.d.ts

@@ -6,6 +6,8 @@ export * from "./createDocumentRelevanceEvaluator.js";
 export * from "./createFaithfulnessEvaluator.js";
 export * from "./createHallucinationEvaluator.js";
 export * from "./createToolInvocationEvaluator.js";
+export * from "./createToolResponseHandlingEvaluator.js";
+export * from "./createToolSelectionEvaluator.js";
 export * from "./generateClassification.js";
 export * from "./LLMEvaluator.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/esm/llm/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,iCAAiC,CAAC;AAChD,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}

package/dist/esm/llm/index.js

@@ -6,6 +6,8 @@ export * from "./createDocumentRelevanceEvaluator.js";
 export * from "./createFaithfulnessEvaluator.js";
 export * from "./createHallucinationEvaluator.js"; // Deprecated: use createFaithfulnessEvaluator
 export * from "./createToolInvocationEvaluator.js";
+export * from "./createToolResponseHandlingEvaluator.js";
+export * from "./createToolSelectionEvaluator.js";
 export * from "./generateClassification.js";
 export * from "./LLMEvaluator.js";
 //# sourceMappingURL=index.js.map

package/dist/esm/llm/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC,CAAC,8CAA8C;AAC9F,cAAc,iCAAiC,CAAC;AAChD,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC,CAAC,8CAA8C;AAC9F,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}