@arizeai/phoenix-evals 0.9.0 → 1.0.0

package/src/llm/ClassificationEvaluator.ts

@@ -1,19 +1,18 @@
+import type { LanguageModel } from "ai";
+
 import { getTemplateVariables } from "../template";
-import {
+import type {
   ClassificationChoicesMap,
   CreateClassificationEvaluatorArgs,
   EvaluatorFn,
   PromptTemplate,
   WithPromptTemplate,
 } from "../types";
-import { ObjectMapping } from "../types/data";
+import type { ObjectMapping } from "../types/data";
 import { remapObject } from "../utils/objectMappingUtils";
-
 import { createClassifierFn } from "./createClassifierFn";
 import { LLMEvaluator } from "./LLMEvaluator";
 
-import { LanguageModel } from "ai";
-
 /**
  * An LLM evaluator that performs evaluation via classification
  */

package/src/llm/LLMEvaluator.ts

@@ -1,5 +1,5 @@
 import { EvaluatorBase } from "../core/EvaluatorBase";
-import { CreateLLMEvaluatorArgs } from "../types";
+import type { CreateLLMEvaluatorArgs } from "../types";
 
 /**
  * Base class for llm evaluation metrics / scores

package/src/llm/createClassificationEvaluator.ts

@@ -1,5 +1,4 @@
-import { CreateClassificationEvaluatorArgs } from "../types/evals";
-
+import type { CreateClassificationEvaluatorArgs } from "../types/evals";
 import { ClassificationEvaluator } from "./ClassificationEvaluator";
 
 export function createClassificationEvaluator<

package/src/llm/createClassifierFn.ts

@@ -1,11 +1,10 @@
 import { formatTemplate } from "../template";
-import {
+import type {
   ClassificationChoicesMap,
   CreateClassifierArgs,
   EvaluationResult,
   EvaluatorFn,
 } from "../types/evals";
-
 import { generateClassification } from "./generateClassification";
 
 /**

package/src/llm/createConcisenessEvaluator.ts

@@ -0,0 +1,71 @@
+import { CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
+import type { CreateClassificationEvaluatorArgs } from "../types/evals";
+import type { ClassificationEvaluator } from "./ClassificationEvaluator";
+import { createClassificationEvaluator } from "./createClassificationEvaluator";
+
+export interface ConcisenessEvaluatorArgs<
+  RecordType extends Record<string, unknown> = ConcisenessEvaluationRecord,
+> extends Omit<
+  CreateClassificationEvaluatorArgs<RecordType>,
+  "promptTemplate" | "choices" | "optimizationDirection" | "name"
+> {
+  optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
+  name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
+  choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
+  promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
+}
+
+/**
+ * A record to be evaluated by the conciseness evaluator.
+ */
+export type ConcisenessEvaluationRecord = {
+  input: string;
+  output: string;
+};
+
+/**
+ * Creates a conciseness evaluator function.
+ *
+ * This function returns an evaluator that determines whether a given output
+ * is concise and free of unnecessary content such as pleasantries, hedging,
+ * meta-commentary, or redundant information.
+ *
+ * @param args - The arguments for creating the conciseness evaluator.
+ * @param args.model - The model to use for classification.
+ * @param args.choices - The possible classification choices (defaults to CONCISENESS_CHOICES).
+ * @param args.promptTemplate - The prompt template to use (defaults to CONCISENESS_TEMPLATE).
+ * @param args.telemetry - The telemetry to use for the evaluator.
+ *
+ * @returns An evaluator function that takes a {@link ConcisenessEvaluationRecord} and returns a classification result
+ * indicating whether the output is concise or verbose.
+ *
+ * @example
+ * ```ts
+ * const evaluator = createConcisenessEvaluator({ model: openai("gpt-4o-mini") });
+ * const result = await evaluator.evaluate({
+ *   input: "What is the capital of France?",
+ *   output: "Paris.",
+ * });
+ * console.log(result.label); // "concise" or "verbose"
+ * ```
+ */
+export function createConcisenessEvaluator<
+  RecordType extends Record<string, unknown> = ConcisenessEvaluationRecord,
+>(
+  args: ConcisenessEvaluatorArgs<RecordType>
+): ClassificationEvaluator<RecordType> {
+  const {
+    choices = CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG.choices,
+    promptTemplate = CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG.template,
+    optimizationDirection = CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection,
+    name = CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG.name,
+    ...rest
+  } = args;
+  return createClassificationEvaluator<RecordType>({
+    ...rest,
+    promptTemplate,
+    choices,
+    optimizationDirection,
+    name,
+  });
+}

package/src/llm/createCorrectnessEvaluator.ts

@@ -1,7 +1,6 @@
 import { CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
-import { CreateClassificationEvaluatorArgs } from "../types/evals";
-
-import { ClassificationEvaluator } from "./ClassificationEvaluator";
+import type { CreateClassificationEvaluatorArgs } from "../types/evals";
+import type { ClassificationEvaluator } from "./ClassificationEvaluator";
 import { createClassificationEvaluator } from "./createClassificationEvaluator";
 
 export interface CorrectnessEvaluatorArgs<

package/src/llm/createDocumentRelevanceEvaluator.ts

@@ -1,7 +1,6 @@
 import { DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
-import { CreateClassificationEvaluatorArgs } from "../types/evals";
-
-import { ClassificationEvaluator } from "./ClassificationEvaluator";
+import type { CreateClassificationEvaluatorArgs } from "../types/evals";
+import type { ClassificationEvaluator } from "./ClassificationEvaluator";
 import { createClassificationEvaluator } from "./createClassificationEvaluator";
 
 export interface DocumentRelevanceEvaluatorArgs<

package/src/llm/createFaithfulnessEvaluator.ts

@@ -1,7 +1,6 @@
 import { FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
-import { CreateClassificationEvaluatorArgs } from "../types/evals";
-
-import { ClassificationEvaluator } from "./ClassificationEvaluator";
+import type { CreateClassificationEvaluatorArgs } from "../types/evals";
+import type { ClassificationEvaluator } from "./ClassificationEvaluator";
 import { createClassificationEvaluator } from "./createClassificationEvaluator";
 
 export interface FaithfulnessEvaluatorArgs<

package/src/llm/createHallucinationEvaluator.ts

@@ -6,9 +6,8 @@
  */
 
 import { HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
-import { CreateClassificationEvaluatorArgs } from "../types/evals";
-
-import { ClassificationEvaluator } from "./ClassificationEvaluator";
+import type { CreateClassificationEvaluatorArgs } from "../types/evals";
+import type { ClassificationEvaluator } from "./ClassificationEvaluator";
 import { createClassificationEvaluator } from "./createClassificationEvaluator";
 
 export interface HallucinationEvaluatorArgs<

package/src/llm/createRefusalEvaluator.ts

@@ -0,0 +1,70 @@
+import { REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
+import type { CreateClassificationEvaluatorArgs } from "../types/evals";
+import type { ClassificationEvaluator } from "./ClassificationEvaluator";
+import { createClassificationEvaluator } from "./createClassificationEvaluator";
+
+export interface RefusalEvaluatorArgs<
+  RecordType extends Record<string, unknown> = RefusalEvaluationRecord,
+> extends Omit<
+  CreateClassificationEvaluatorArgs<RecordType>,
+  "promptTemplate" | "choices" | "optimizationDirection" | "name"
+> {
+  optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
+  name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
+  choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
+  promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
+}
+
+/**
+ * A record to be evaluated by the refusal evaluator.
+ */
+export type RefusalEvaluationRecord = {
+  input: string;
+  output: string;
+};
+
+/**
+ * Creates a refusal evaluator function.
+ *
+ * This function returns an evaluator that detects when an LLM refuses,
+ * declines, or avoids answering a user query. It is use-case agnostic:
+ * it only detects whether a refusal occurred, not whether the refusal
+ * was appropriate.
+ *
+ * @param args - The arguments for creating the refusal evaluator.
+ * @param args.model - The model to use for classification.
+ * @param args.choices - The possible classification choices (defaults to REFUSAL_CHOICES).
+ * @param args.promptTemplate - The prompt template to use (defaults to REFUSAL_TEMPLATE).
+ * @param args.telemetry - The telemetry to use for the evaluator.
+ *
+ * @returns An evaluator function that takes a {@link RefusalEvaluationRecord} and returns a classification result
+ * indicating whether the output is a refusal or an answer.
+ *
+ * @example
+ * ```ts
+ * const evaluator = createRefusalEvaluator({ model: openai("gpt-4o-mini") });
+ * const result = await evaluator.evaluate({
+ *   input: "What is the capital of France?",
+ *   output: "I'm sorry, I can only help with technical questions.",
+ * });
+ * console.log(result.label); // "refused" or "answered"
+ * ```
+ */
+export function createRefusalEvaluator<
+  RecordType extends Record<string, unknown> = RefusalEvaluationRecord,
+>(args: RefusalEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType> {
+  const {
+    choices = REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG.choices,
+    promptTemplate = REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG.template,
+    optimizationDirection = REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection,
+    name = REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG.name,
+    ...rest
+  } = args;
+  return createClassificationEvaluator<RecordType>({
+    ...rest,
+    promptTemplate,
+    choices,
+    optimizationDirection,
+    name,
+  });
+}

package/src/llm/createToolInvocationEvaluator.ts

@@ -1,7 +1,6 @@
 import { TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
-import { CreateClassificationEvaluatorArgs } from "../types/evals";
-
-import { ClassificationEvaluator } from "./ClassificationEvaluator";
+import type { CreateClassificationEvaluatorArgs } from "../types/evals";
+import type { ClassificationEvaluator } from "./ClassificationEvaluator";
 import { createClassificationEvaluator } from "./createClassificationEvaluator";
 
 export interface ToolInvocationEvaluatorArgs<

package/src/llm/createToolResponseHandlingEvaluator.ts

@@ -1,7 +1,6 @@
 import { TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
-import { CreateClassificationEvaluatorArgs } from "../types/evals";
-
-import { ClassificationEvaluator } from "./ClassificationEvaluator";
+import type { CreateClassificationEvaluatorArgs } from "../types/evals";
+import type { ClassificationEvaluator } from "./ClassificationEvaluator";
 import { createClassificationEvaluator } from "./createClassificationEvaluator";
 
 export interface ToolResponseHandlingEvaluatorArgs<

package/src/llm/createToolSelectionEvaluator.ts

@@ -1,7 +1,6 @@
 import { TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
-import { CreateClassificationEvaluatorArgs } from "../types/evals";
-
-import { ClassificationEvaluator } from "./ClassificationEvaluator";
+import type { CreateClassificationEvaluatorArgs } from "../types/evals";
+import type { ClassificationEvaluator } from "./ClassificationEvaluator";
 import { createClassificationEvaluator } from "./createClassificationEvaluator";
 
 export interface ToolSelectionEvaluatorArgs<

package/src/llm/generateClassification.ts

@@ -1,10 +1,10 @@
-import { tracer } from "../telemetry";
-import { ClassificationResult, WithLLM } from "../types/evals";
-import { WithTelemetry } from "../types/otel";
-import type { WithPrompt } from "../types/prompts";
-
 import { generateObject } from "ai";
 import { z } from "zod";
+
+import { tracer } from "../telemetry";
+import type { ClassificationResult, WithLLM } from "../types/evals";
+import type { WithTelemetry } from "../types/otel";
+import type { WithPrompt } from "../types/prompts";
 export type ClassifyArgs = WithLLM &
   WithTelemetry &
   WithPrompt & {

package/src/llm/index.ts

@@ -1,10 +1,12 @@
 export * from "./ClassificationEvaluator";
 export * from "./createClassificationEvaluator";
 export * from "./createClassifierFn";
+export * from "./createConcisenessEvaluator";
 export * from "./createCorrectnessEvaluator";
 export * from "./createDocumentRelevanceEvaluator";
 export * from "./createFaithfulnessEvaluator";
 export * from "./createHallucinationEvaluator"; // Deprecated: use createFaithfulnessEvaluator
+export * from "./createRefusalEvaluator";
 export * from "./createToolInvocationEvaluator";
 export * from "./createToolResponseHandlingEvaluator";
 export * from "./createToolSelectionEvaluator";

package/src/template/applyTemplate.ts

@@ -1,9 +1,8 @@
-import { PromptTemplate, RenderedPrompt } from "../types/templating";
+import Mustache from "mustache";
 
+import type { PromptTemplate, RenderedPrompt } from "../types/templating";
 import { createTemplateVariablesProxy } from "./createTemplateVariablesProxy";
 
-import Mustache from "mustache";
-
 /**
  * A function that applies a set of variables to a template (e.g. a prompt)
  * Uses the Mustache library to apply the variables to the template

package/src/template/getTemplateVariables.ts

@@ -1,7 +1,7 @@
-import { PromptTemplate } from "../types/templating";
-
 import Mustache from "mustache";
 
+import type { PromptTemplate } from "../types/templating";
+
 type GetTemplateVariableArgs = {
   template: PromptTemplate;
 };

package/src/types/evals.ts

@@ -1,8 +1,8 @@
-import { ObjectMapping } from "./data";
-import { WithTelemetry } from "./otel";
-import { PromptTemplate } from "./templating";
+import type { LanguageModel } from "ai";
 
-import { LanguageModel } from "ai";
+import type { ObjectMapping } from "./data";
+import type { WithTelemetry } from "./otel";
+import type { PromptTemplate } from "./templating";
 
 /**
  * A specific AI example that is under evaluation
@@ -131,7 +131,7 @@ export type EvaluationKind = "LLM" | "CODE";
  * The direction to optimize the numeric evaluation score
  * E.x. "MAXIMIZE" means that the higher the score, the better the evaluation
  */
-export type OptimizationDirection = "MAXIMIZE" | "MINIMIZE";
+export type OptimizationDirection = "MAXIMIZE" | "MINIMIZE" | "NEUTRAL";
 
 /**
  * The description of an evaluator

package/src/types/otel.ts

@@ -1,4 +1,4 @@
-import { Tracer } from "@opentelemetry/api";
+import type { Tracer } from "@opentelemetry/api";
 
 export type TelemetryConfig = {
   /**

package/src/utils/bindEvaluator.ts

@@ -1,5 +1,5 @@
-import { EvaluatorBase } from "../core/EvaluatorBase";
-import { ObjectMapping } from "../types/data";
+import type { EvaluatorBase } from "../core/EvaluatorBase";
+import type { ObjectMapping } from "../types/data";
 
 /**
  * Context for binding an evaluator with input mapping configuration.

package/src/utils/objectMappingUtils.ts

@@ -1,7 +1,7 @@
-import { ObjectMapping, ValueGetter } from "../types/data";
-
 import { JSONPath } from "jsonpath-plus";
 
+import type { ObjectMapping, ValueGetter } from "../types/data";
+
 /**
  * Remaps an object by applying field mappings while preserving original data.
  *