@arizeai/phoenix-evals 0.6.5 → 0.8.0

package/dist/esm/llm/createToolInvocationEvaluator.d.ts

@@ -0,0 +1,74 @@
+import { CreateClassificationEvaluatorArgs } from "../types/evals.js";
+import { ClassificationEvaluator } from "./ClassificationEvaluator.js";
+export interface ToolInvocationEvaluatorArgs<RecordType extends Record<string, unknown> = ToolInvocationEvaluationRecord> extends Omit<CreateClassificationEvaluatorArgs<RecordType>, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
+    optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
+    name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
+    choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
+    promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
+}
+/**
+ * A record to be evaluated by the tool invocation evaluator.
+ */
+export type ToolInvocationEvaluationRecord = {
+    /**
+     * The input query or conversation context.
+     */
+    input: string;
+    /**
+     * The available tool schemas, either as JSON schema or human-readable format.
+     */
+    availableTools: string;
+    /**
+     * The tool invocation(s) made by the LLM, including arguments.
+     */
+    toolSelection: string;
+};
+/**
+ * Creates a tool invocation evaluator function.
+ *
+ * This function returns an evaluator that determines whether a tool was invoked
+ * correctly with proper arguments, formatting, and safe content.
+ *
+ * The evaluator checks for:
+ * - Properly structured JSON (if applicable)
+ * - All required fields/parameters present
+ * - No hallucinated or nonexistent fields
+ * - Argument values matching user query and schema expectations
+ * - No unsafe content (e.g., PII) in arguments
+ *
+ * @param args - The arguments for creating the tool invocation evaluator.
+ * @param args.model - The model to use for classification.
+ * @param args.choices - The possible classification choices (defaults to correct/incorrect).
+ * @param args.promptTemplate - The prompt template to use (defaults to TOOL_INVOCATION_TEMPLATE).
+ * @param args.telemetry - The telemetry to use for the evaluator.
+ *
+ * @returns An evaluator function that takes a {@link ToolInvocationEvaluationRecord} and returns
+ * a classification result indicating whether the tool invocation is correct or incorrect.
+ *
+ * @example
+ * ```ts
+ * const evaluator = createToolInvocationEvaluator({ model: openai("gpt-4o-mini") });
+ *
+ * // Example with JSON schema format for available tools
+ * const result = await evaluator.evaluate({
+ *   input: "User: Book a flight from NYC to LA for tomorrow",
+ *   availableTools: JSON.stringify({
+ *     name: "book_flight",
+ *     description: "Book a flight between two cities",
+ *     parameters: {
+ *       type: "object",
+ *       properties: {
+ *         origin: { type: "string", description: "Departure city code" },
+ *         destination: { type: "string", description: "Arrival city code" },
+ *         date: { type: "string", description: "Flight date in YYYY-MM-DD" }
+ *       },
+ *       required: ["origin", "destination", "date"]
+ *     }
+ *   }),
+ *   toolSelection: 'book_flight(origin="NYC", destination="LA", date="2024-01-15")'
+ * });
+ * console.log(result.label); // "correct" or "incorrect"
+ * ```
+ */
+export declare function createToolInvocationEvaluator<RecordType extends Record<string, unknown> = ToolInvocationEvaluationRecord>(args: ToolInvocationEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType>;
+//# sourceMappingURL=createToolInvocationEvaluator.d.ts.map

package/dist/esm/llm/createToolInvocationEvaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"createToolInvocationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createToolInvocationEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGpE,MAAM,WAAW,2BAA2B,CAC1C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,8BAA8B,CAC3E,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED;;GAEG;AACH,MAAM,MAAM,8BAA8B,GAAG;IAC3C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,wBAAgB,6BAA6B,CAC3C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,8BAA8B,EAE3E,IAAI,EAAE,2BAA2B,CAAC,UAAU,CAAC,GAC5C,uBAAuB,CAAC,UAAU,CAAC,CAerC"}

package/dist/esm/llm/createToolInvocationEvaluator.js

@@ -0,0 +1,60 @@
+import { TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates/index.js";
+import { createClassificationEvaluator } from "./createClassificationEvaluator.js";
+/**
+ * Creates a tool invocation evaluator function.
+ *
+ * This function returns an evaluator that determines whether a tool was invoked
+ * correctly with proper arguments, formatting, and safe content.
+ *
+ * The evaluator checks for:
+ * - Properly structured JSON (if applicable)
+ * - All required fields/parameters present
+ * - No hallucinated or nonexistent fields
+ * - Argument values matching user query and schema expectations
+ * - No unsafe content (e.g., PII) in arguments
+ *
+ * @param args - The arguments for creating the tool invocation evaluator.
+ * @param args.model - The model to use for classification.
+ * @param args.choices - The possible classification choices (defaults to correct/incorrect).
+ * @param args.promptTemplate - The prompt template to use (defaults to TOOL_INVOCATION_TEMPLATE).
+ * @param args.telemetry - The telemetry to use for the evaluator.
+ *
+ * @returns An evaluator function that takes a {@link ToolInvocationEvaluationRecord} and returns
+ * a classification result indicating whether the tool invocation is correct or incorrect.
+ *
+ * @example
+ * ```ts
+ * const evaluator = createToolInvocationEvaluator({ model: openai("gpt-4o-mini") });
+ *
+ * // Example with JSON schema format for available tools
+ * const result = await evaluator.evaluate({
+ *   input: "User: Book a flight from NYC to LA for tomorrow",
+ *   availableTools: JSON.stringify({
+ *     name: "book_flight",
+ *     description: "Book a flight between two cities",
+ *     parameters: {
+ *       type: "object",
+ *       properties: {
+ *         origin: { type: "string", description: "Departure city code" },
+ *         destination: { type: "string", description: "Arrival city code" },
+ *         date: { type: "string", description: "Flight date in YYYY-MM-DD" }
+ *       },
+ *       required: ["origin", "destination", "date"]
+ *     }
+ *   }),
+ *   toolSelection: 'book_flight(origin="NYC", destination="LA", date="2024-01-15")'
+ * });
+ * console.log(result.label); // "correct" or "incorrect"
+ * ```
+ */
+export function createToolInvocationEvaluator(args) {
+    const { choices = TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.choices, promptTemplate = TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.template, optimizationDirection = TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection, name = TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.name, ...rest } = args;
+    return createClassificationEvaluator({
+        ...rest,
+        promptTemplate,
+        choices,
+        optimizationDirection,
+        name,
+    });
+}
+//# sourceMappingURL=createToolInvocationEvaluator.js.map

package/dist/esm/llm/createToolInvocationEvaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createToolInvocationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createToolInvocationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,+CAA+C,EAAE,MAAM,oCAAoC,CAAC;AAIrG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAgChF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,MAAM,UAAU,6BAA6B,CAG3C,IAA6C;IAE7C,MAAM,EACJ,OAAO,GAAG,+CAA+C,CAAC,OAAO,EACjE,cAAc,GAAG,+CAA+C,CAAC,QAAQ,EACzE,qBAAqB,GAAG,+CAA+C,CAAC,qBAAqB,EAC7F,IAAI,GAAG,+CAA+C,CAAC,IAAI,EAC3D,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}

package/dist/esm/llm/index.d.ts

@@ -1,8 +1,11 @@
-export * from "./generateClassification.js";
-export * from "./createClassifierFn.js";
+export * from "./ClassificationEvaluator.js";
 export * from "./createClassificationEvaluator.js";
-export * from "./createHallucinationEvaluator.js";
+export * from "./createClassifierFn.js";
+export * from "./createCorrectnessEvaluator.js";
 export * from "./createDocumentRelevanceEvaluator.js";
-export * from "./ClassificationEvaluator.js";
+export * from "./createFaithfulnessEvaluator.js";
+export * from "./createHallucinationEvaluator.js";
+export * from "./createToolInvocationEvaluator.js";
+export * from "./generateClassification.js";
 export * from "./LLMEvaluator.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/esm/llm/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAC;AACzC,cAAc,sBAAsB,CAAC;AACrC,cAAc,iCAAiC,CAAC;AAChD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,oCAAoC,CAAC;AACnD,cAAc,2BAA2B,CAAC;AAC1C,cAAc,gBAAgB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,iCAAiC,CAAC;AAChD,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}

package/dist/esm/llm/index.js

@@ -1,8 +1,11 @@
-export * from "./generateClassification.js";
-export * from "./createClassifierFn.js";
+export * from "./ClassificationEvaluator.js";
 export * from "./createClassificationEvaluator.js";
-export * from "./createHallucinationEvaluator.js";
+export * from "./createClassifierFn.js";
+export * from "./createCorrectnessEvaluator.js";
 export * from "./createDocumentRelevanceEvaluator.js";
-export * from "./ClassificationEvaluator.js";
+export * from "./createFaithfulnessEvaluator.js";
+export * from "./createHallucinationEvaluator.js"; // Deprecated: use createFaithfulnessEvaluator
+export * from "./createToolInvocationEvaluator.js";
+export * from "./generateClassification.js";
 export * from "./LLMEvaluator.js";
 //# sourceMappingURL=index.js.map

package/dist/esm/llm/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAC;AACzC,cAAc,sBAAsB,CAAC;AACrC,cAAc,iCAAiC,CAAC;AAChD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,oCAAoC,CAAC;AACnD,cAAc,2BAA2B,CAAC;AAC1C,cAAc,gBAAgB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC,CAAC,8CAA8C;AAC9F,cAAc,iCAAiC,CAAC;AAChD,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}