@arizeai/phoenix-evals 0.10.0 → 1.0.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arizeai/phoenix-evals",
-  "version": "0.10.0",
+  "version": "1.0.1",
   "description": "A library for running evaluations for AI use cases",
   "keywords": [
     "arize",
@@ -71,8 +71,8 @@
     "msw": "^2.11.6",
     "nock": "^14.0.5",
     "tsx": "^4.19.3",
-    "typedoc": "^0.27.9",
-    "vitest": "^4.0.10"
+    "typedoc": "^0.28.17",
+    "vitest": "^4.1.0"
   },
   "engines": {
     "node": ">=18"

package/src/__generated__/default_templates/REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG.ts

@@ -0,0 +1,61 @@
+// This file is generated. Do not edit by hand.
+
+import type { ClassificationEvaluatorConfig } from "../types";
+
+export const REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG: ClassificationEvaluatorConfig = {
+  name: "refusal",
+  description: "Detect when an LLM refuses or declines to answer a query.",
+  optimizationDirection: "NEUTRAL",
+  template: [
+    {
+      role: "user",
+      content: `
+You are an expert evaluator that detects when an LLM refuses, declines, or avoids answering a user query. Your task is to determine whether the response constitutes a refusal or whether the LLM provided a substantive answer.
+
+<rubric>
+
+REFUSED - The response contains any form of refusal, deflection, or avoidance, including but not limited to:
+
+- Explicit refusals (e.g., "I can't help with that", "I'm unable to answer")
+- Scope disclaimers (e.g., "That's outside my area of expertise", "I'm just a chatbot that handles X")
+- Lack-of-information responses (e.g., "I don't have that information", "Nothing in the search results addresses that")
+- Safety or policy refusals (e.g., "I'm not able to provide that kind of advice")
+- Redirections without answering (e.g., "You should ask a professional about that")
+- Apologetic non-answers (e.g., "I'm sorry, but I can't assist with that request")
+- Partial refusals where the LLM acknowledges the question but ultimately does not provide an answer
+
+
+ANSWERED - The response provides a substantive answer to the query, even if:
+
+- The answer is incorrect, incomplete, or low quality
+- The answer includes caveats or disclaimers alongside a genuine attempt to respond
+- The answer is speculative or hedged but still addresses the question
+- The answer acknowledges limitations but still provides relevant information
+
+</rubric>
+
+IMPORTANT: This evaluation is purely about whether a refusal occurred, not whether the refusal was appropriate. A refusal to an out-of-scope question is still a refusal. An incorrect answer to a simple question is still an answer.
+
+<data>
+
+<input>
+{{input}}
+</input>
+
+<output>
+{{output}}
+</output>
+
+</data>
+
+Carefully read the input and output. Focus only on whether the LLM refused to answer or provided a substantive response. Do not evaluate correctness, helpfulness, or quality.
+
+Did the LLM refuse to answer or did it provide an answer?
+`,
+    },
+  ],
+  choices: {
+  "refused": 1,
+  "answered": 0
+},
+};

package/src/__generated__/default_templates/index.ts

@@ -5,6 +5,7 @@ export { CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG } from "./CORRECTNESS_CLASS
 export { DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG } from "./DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG";
 export { FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG } from "./FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG";
 export { HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG } from "./HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG";
+export { REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG } from "./REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG";
 export { TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG";
 export { TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG";
 export { TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG";

package/src/__generated__/types.ts

@@ -5,7 +5,7 @@ import type { PromptTemplate } from "../types/templating";
 export type ClassificationEvaluatorConfig = {
   name: string;
   description: string;
-  optimizationDirection: "MINIMIZE" | "MAXIMIZE";
+  optimizationDirection: "MINIMIZE" | "MAXIMIZE" | "NEUTRAL";
   template: PromptTemplate;
   choices: Record<string, number>;
 };

package/src/llm/createRefusalEvaluator.ts

@@ -0,0 +1,70 @@
+import { REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
+import type { CreateClassificationEvaluatorArgs } from "../types/evals";
+import type { ClassificationEvaluator } from "./ClassificationEvaluator";
+import { createClassificationEvaluator } from "./createClassificationEvaluator";
+
+export interface RefusalEvaluatorArgs<
+  RecordType extends Record<string, unknown> = RefusalEvaluationRecord,
+> extends Omit<
+  CreateClassificationEvaluatorArgs<RecordType>,
+  "promptTemplate" | "choices" | "optimizationDirection" | "name"
+> {
+  optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
+  name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
+  choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
+  promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
+}
+
+/**
+ * A record to be evaluated by the refusal evaluator.
+ */
+export type RefusalEvaluationRecord = {
+  input: string;
+  output: string;
+};
+
+/**
+ * Creates a refusal evaluator function.
+ *
+ * This function returns an evaluator that detects when an LLM refuses,
+ * declines, or avoids answering a user query. It is use-case agnostic:
+ * it only detects whether a refusal occurred, not whether the refusal
+ * was appropriate.
+ *
+ * @param args - The arguments for creating the refusal evaluator.
+ * @param args.model - The model to use for classification.
+ * @param args.choices - The possible classification choices (defaults to REFUSAL_CHOICES).
+ * @param args.promptTemplate - The prompt template to use (defaults to REFUSAL_TEMPLATE).
+ * @param args.telemetry - The telemetry to use for the evaluator.
+ *
+ * @returns An evaluator function that takes a {@link RefusalEvaluationRecord} and returns a classification result
+ * indicating whether the output is a refusal or an answer.
+ *
+ * @example
+ * ```ts
+ * const evaluator = createRefusalEvaluator({ model: openai("gpt-4o-mini") });
+ * const result = await evaluator.evaluate({
+ *   input: "What is the capital of France?",
+ *   output: "I'm sorry, I can only help with technical questions.",
+ * });
+ * console.log(result.label); // "refused" or "answered"
+ * ```
+ */
+export function createRefusalEvaluator<
+  RecordType extends Record<string, unknown> = RefusalEvaluationRecord,
+>(args: RefusalEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType> {
+  const {
+    choices = REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG.choices,
+    promptTemplate = REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG.template,
+    optimizationDirection = REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection,
+    name = REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG.name,
+    ...rest
+  } = args;
+  return createClassificationEvaluator<RecordType>({
+    ...rest,
+    promptTemplate,
+    choices,
+    optimizationDirection,
+    name,
+  });
+}

package/src/llm/index.ts

@@ -6,6 +6,7 @@ export * from "./createCorrectnessEvaluator";
 export * from "./createDocumentRelevanceEvaluator";
 export * from "./createFaithfulnessEvaluator";
 export * from "./createHallucinationEvaluator"; // Deprecated: use createFaithfulnessEvaluator
+export * from "./createRefusalEvaluator";
 export * from "./createToolInvocationEvaluator";
 export * from "./createToolResponseHandlingEvaluator";
 export * from "./createToolSelectionEvaluator";

package/src/types/evals.ts

@@ -131,7 +131,7 @@ export type EvaluationKind = "LLM" | "CODE";
  * The direction to optimize the numeric evaluation score
  * E.x. "MAXIMIZE" means that the higher the score, the better the evaluation
  */
-export type OptimizationDirection = "MAXIMIZE" | "MINIMIZE";
+export type OptimizationDirection = "MAXIMIZE" | "MINIMIZE" | "NEUTRAL";
 
 /**
  * The description of an evaluator