@arizeai/phoenix-evals 0.0.7 → 0.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arizeai/phoenix-evals",
-  "version": "0.0.7",
+  "version": "0.1.0",
   "description": "A library for running evaluations for AI use cases",
   "main": "dist/src/index.js",
   "module": "dist/esm/index.js",
@@ -41,7 +41,7 @@
   "author": "oss@arize.com",
   "license": "ELv2",
   "devDependencies": {
-    "@ai-sdk/openai": "^1.1.15",
+    "@ai-sdk/openai": "^2.0.0",
     "@arizeai/openinference-instrumentation-openai": "^2.3.0",
     "@types/mustache": "^4.2.6",
     "@types/node": "^24.0.10",
@@ -50,16 +50,16 @@
     "typedoc": "^0.27.9",
     "typescript": "^5.8.2",
     "vitest": "^2.1.9",
-    "@arizeai/phoenix-client": "2.3.5"
+    "@arizeai/phoenix-client": "3.0.0"
   },
   "engines": {
     "node": ">=18"
   },
   "dependencies": {
     "@opentelemetry/api": "^1.9.0",
-    "ai": "^4.1.47",
+    "ai": "^5.0.0",
     "mustache": "^4.2.0",
-    "zod": "^3.25.75"
+    "zod": "^4.0.14"
   },
   "scripts": {
     "clean": "rimraf dist",

package/src/default_templates/DOCUMENT_RELEVANCY_TEMPLATE.ts

@@ -0,0 +1,25 @@
+export const DOCUMENT_RELEVANCY_TEMPLATE = `
+You are comparing a document to a question and trying to determine if the document text
+contains information relevant to answering the question. Here is the data:
+
+    [BEGIN DATA]
+    ************
+    [Question]: {{input}}
+    ************
+    [Document text]: {{documentText}}
+    ************
+    [END DATA]
+
+Compare the Question above to the Document text. You must determine whether the Document text
+contains information that can answer the Question. Please focus on whether the very specific
+question can be answered by the information in the Document text.
+Your response must be single word, either "relevant" or "unrelated",
+and should not contain any text or characters aside from that word.
+"unrelated" means that the document text does not contain an answer to the Question.
+"relevant" means the document text contains an answer to the Question.
+`;
+
+export const DOCUMENT_RELEVANCY_CHOICES = {
+  relevant: 1,
+  unrelated: 0,
+};

package/src/default_templates/HALLUCINATION_TEMPLATE.ts

@@ -26,6 +26,6 @@ Is the answer above factual or hallucinated based on the query and reference tex
 `;
 
 export const HALLUCINATION_CHOICES = {
-  factual: 1,
-  hallucinated: 0,
+  hallucinated: 1,
+  factual: 0,
 };

package/src/default_templates/index.ts

@@ -1 +1,2 @@
 export * from "./HALLUCINATION_TEMPLATE";
+export * from "./DOCUMENT_RELEVANCY_TEMPLATE";

package/src/llm/createClassificationEvaluator.ts

@@ -0,0 +1,13 @@
+import { CreateClassificationEvaluatorArgs, Evaluator } from "../types/evals";
+import { createClassifierFn } from "./createClassifierFn";
+
+export function createClassificationEvaluator<
+  ExampleType extends Record<string, unknown>,
+>(args: CreateClassificationEvaluatorArgs): Evaluator<ExampleType> {
+  return {
+    name: args.name,
+    source: "LLM",
+    optimizationDirection: args.optimizationDirection,
+    evaluate: createClassifierFn(args),
+  };
+}

package/src/llm/{createClassifier.ts → createClassifierFn.ts}

@@ -1,6 +1,5 @@
 import {
   ClassificationChoicesMap,
-  EvaluationArgs,
   EvaluationResult,
   CreateClassifierArgs,
   EvaluatorFn,
@@ -23,16 +22,14 @@ function choicesToLabels(
 }
 
 /**
- * A function that serves as a factory that will output a classification evaluator
+ * A function that serves as a factory that will output a classification evaluator function
  */
-export function createClassifier<OutputType, InputType>(
+export function createClassifierFn<ExampleType extends Record<string, unknown>>(
   args: CreateClassifierArgs
-): EvaluatorFn<OutputType, InputType> {
+): EvaluatorFn<ExampleType> {
   const { model, choices, promptTemplate, ...rest } = args;
 
-  return async (
-    args: EvaluationArgs<OutputType, InputType>
-  ): Promise<EvaluationResult> => {
+  return async (args: ExampleType): Promise<EvaluationResult> => {
     const templateVariables = {
       ...args,
     };

package/src/llm/createDocumentRelevancyEvaluator.ts

@@ -0,0 +1,72 @@
+import { CreateClassificationEvaluatorArgs, Evaluator } from "../types/evals";
+import {
+  DOCUMENT_RELEVANCY_TEMPLATE,
+  DOCUMENT_RELEVANCY_CHOICES,
+} from "../default_templates/DOCUMENT_RELEVANCY_TEMPLATE";
+import { createClassificationEvaluator } from "./createClassificationEvaluator";
+
+export interface DocumentRelevancyEvaluatorArgs
+  extends Omit<
+    CreateClassificationEvaluatorArgs,
+    "promptTemplate" | "choices" | "optimizationDirection" | "name"
+  > {
+  optimizationDirection?: CreateClassificationEvaluatorArgs["optimizationDirection"];
+  name?: CreateClassificationEvaluatorArgs["name"];
+  choices?: CreateClassificationEvaluatorArgs["choices"];
+  promptTemplate?: CreateClassificationEvaluatorArgs["promptTemplate"];
+}
+
+/**
+ * An example to be evaluated by the document relevancy evaluator.
+ */
+export type DocumentRelevancyExample = {
+  input: string;
+  documentText: string;
+};
+
+/**
+ * Creates a document relevancy evaluator function.
+ *
+ * This function returns an evaluator that determines whether a given document text
+ * is relevant to a provided input question. The evaluator uses a classification model
+ * and a prompt template to make its determination.
+ *
+ * @param args - The arguments for creating the document relevancy evaluator.
+ * @param args.model - The model to use for classification.
+ * @param args.choices - The possible classification choices (defaults to DOCUMENT_RELEVANCY_CHOICES).
+ * @param args.promptTemplate - The prompt template to use (defaults to DOCUMENT_RELEVANCY_TEMPLATE).
+ * @param args.telemetry - The telemetry to use for the evaluator.
+ *
+ * @returns An evaluator function that takes a {@link DocumentRelevancyExample} and returns a classification result
+ * indicating whether the document is relevant to the input question.
+ *
+ * @example
+ * ```ts
+ * const evaluator = createDocumentRelevancyEvaluator({ model: openai("gpt-4o-mini") });
+ * const result = await evaluator.evaluate({
+ *   input: "What is the capital of France?",
+ *   documentText: "Paris is the capital and most populous city of France.",
+ * });
+ * console.log(result.label); // "relevant" or "unrelated"
+ * ```
+ */
+export function createDocumentRelevancyEvaluator(
+  args: DocumentRelevancyEvaluatorArgs
+): Evaluator<DocumentRelevancyExample> {
+  const {
+    choices = DOCUMENT_RELEVANCY_CHOICES,
+    promptTemplate = DOCUMENT_RELEVANCY_TEMPLATE,
+    optimizationDirection = "MAXIMIZE",
+    name = "document_relevancy",
+    ...rest
+  } = args;
+
+  return createClassificationEvaluator<DocumentRelevancyExample>({
+    ...args,
+    promptTemplate,
+    choices,
+    optimizationDirection,
+    name,
+    ...rest,
+  });
+}

package/src/llm/createHallucinationEvaluator.ts

@@ -1,15 +1,30 @@
-import { createClassifier } from "./createClassifier";
-import { CreateClassifierArgs, EvaluatorFn } from "../types/evals";
+import { CreateClassificationEvaluatorArgs, Evaluator } from "../types/evals";
 import {
   HALLUCINATION_TEMPLATE,
   HALLUCINATION_CHOICES,
 } from "../default_templates/HALLUCINATION_TEMPLATE";
+import { createClassificationEvaluator } from "./createClassificationEvaluator";
 
 export interface HallucinationEvaluatorArgs
-  extends Omit<CreateClassifierArgs, "promptTemplate" | "choices"> {
-  choices?: CreateClassifierArgs["choices"];
-  promptTemplate?: CreateClassifierArgs["promptTemplate"];
+  extends Omit<
+    CreateClassificationEvaluatorArgs,
+    "promptTemplate" | "choices" | "optimizationDirection" | "name"
+  > {
+  optimizationDirection?: CreateClassificationEvaluatorArgs["optimizationDirection"];
+  name?: CreateClassificationEvaluatorArgs["name"];
+  choices?: CreateClassificationEvaluatorArgs["choices"];
+  promptTemplate?: CreateClassificationEvaluatorArgs["promptTemplate"];
 }
+
+/**
+ * An example to be evaluated by the hallucination evaluator.
+ */
+export type HallucinationExample = {
+  input: string;
+  output: string;
+  reference?: string;
+  context?: string;
+};
 /**
  * Creates a function that evaluates whether an answer is factual or hallucinated based on a query and reference text.
  *
@@ -18,17 +33,20 @@ export interface HallucinationEvaluatorArgs
  */
 export function createHallucinationEvaluator(
   args: HallucinationEvaluatorArgs
-): EvaluatorFn<string, string> {
+): Evaluator<HallucinationExample> {
   const {
     choices = HALLUCINATION_CHOICES,
     promptTemplate = HALLUCINATION_TEMPLATE,
+    optimizationDirection = "MINIMIZE",
+    name = "hallucination",
     ...rest
   } = args;
-  const hallucinationEvaluatorFn = createClassifier<string, string>({
+  return createClassificationEvaluator<HallucinationExample>({
     ...args,
     promptTemplate,
     choices,
+    optimizationDirection,
+    name,
     ...rest,
   });
-  return hallucinationEvaluatorFn;
 }

package/src/llm/index.ts

@@ -1,3 +1,5 @@
 export * from "./generateClassification";
-export * from "./createClassifier";
+export * from "./createClassifierFn";
+export * from "./createClassificationEvaluator";
 export * from "./createHallucinationEvaluator";
+export * from "./createDocumentRelevancyEvaluator";

package/src/types/evals.ts

@@ -2,9 +2,9 @@ import { LanguageModel } from "ai";
 import { WithTelemetry } from "./otel";
 
 /**
- * The arguments for an evaluation
+ * A specific AI example that is under evaluation
  */
-export interface EvaluationArgs<OutputType, InputType> {
+export interface ExampleRecord<OutputType, InputType> {
   output: OutputType;
   expected?: OutputType;
   input?: InputType;
@@ -15,9 +15,7 @@ export interface WithLLM {
   model: LanguageModel;
 }
 
-export interface LLMEvaluationArgs<OutputType, InputType>
-  extends EvaluationArgs<OutputType, InputType>,
-    WithLLM {}
+export interface LLMEvaluationArgs extends WithLLM {}
 
 /**
  * The result of an evaluation
@@ -80,6 +78,55 @@ export interface CreateClassifierArgs extends WithTelemetry {
   promptTemplate: string;
 }
 
-export type EvaluatorFn<OutputType, InputType> = (
-  args: EvaluationArgs<OutputType, InputType>
+export interface CreateClassificationEvaluatorArgs
+  extends CreateClassifierArgs {
+  /**
+   * The name of the metric that the evaluator produces
+   * E.x. "correctness"
+   */
+  name: string;
+  /**
+   * If present, represents the direction in which you want the metric to be optimized
+   * E.x. "MAXIMIZE" means you want the number to be higher.
+   */
+  optimizationDirection?: OptimizationDirection;
+}
+
+export type EvaluatorFn<ExampleType extends Record<string, unknown>> = (
+  args: ExampleType
 ) => Promise<EvaluationResult>;
+
+/**
+ * The source of the evaluation
+ */
+type EvaluationSource = "LLM" | "CODE";
+
+/**
+ * The direction to optimize the numeric evaluation score
+ * E.x. "MAXIMIZE" means that the higher the score, the better the evaluation
+ */
+type OptimizationDirection = "MAXIMIZE" | "MINIMIZE";
+
+/**
+ * The Base Evaluator interface
+ * This is the interface that all evaluators must implement
+ */
+export interface Evaluator<ExampleType extends Record<string, unknown>> {
+  /**
+   * The name of the evaluator / the metric that it measures
+   */
+  name: string;
+  /**
+   * The source of the evaluation. Also known as the "kind" of evaluator.
+   */
+  source: EvaluationSource;
+  /**
+   * The direction to optimize the numeric evaluation score
+   * E.x. "MAXIMIZE" means that the higher the score, the better the evaluation
+   */
+  optimizationDirection?: OptimizationDirection;
+  /**
+   * The function that evaluates the example
+   */
+  evaluate: EvaluatorFn<ExampleType>;
+}

package/src/types/prompts.ts

@@ -1,9 +1,9 @@
-import { CoreMessage, Message } from "ai";
+import { ModelMessage } from "ai";
 
 /**
- * Prompt part of the AI function options.
- * It contains a system message, a simple text prompt, or a list of messages.
- * Note: this is pulled from the `ai` package and is used as a compatibility type.
+ * Prompt part of the AI function options for model generation.
+ * It contains a system message, a simple text prompt, or a list of model messages.
+ * Uses ModelMessage format compatible with AI SDK v5 generateObject function.
  */
 export interface WithPrompt {
   /**
@@ -15,7 +15,8 @@ export interface WithPrompt {
    */
   prompt?: string;
   /**
-   * A list of messages. You can either use `prompt` or `messages` but not both.
+   * A list of model messages. You can either use `prompt` or `messages` but not both.
+   * Uses ModelMessage format for compatibility with AI SDK v5.
    */
-  messages?: Array<CoreMessage> | Array<Omit<Message, "id">>;
+  messages?: Array<ModelMessage>;
 }

package/dist/esm/llm/createClassifier.d.ts

@@ -1,6 +0,0 @@
-import { CreateClassifierArgs, EvaluatorFn } from "../types/evals.js";
-/**
- * A function that serves as a factory that will output a classification evaluator
- */
-export declare function createClassifier<OutputType, InputType>(args: CreateClassifierArgs): EvaluatorFn<OutputType, InputType>;
-//# sourceMappingURL=createClassifier.d.ts.map

package/dist/esm/llm/createClassifier.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"createClassifier.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassifier.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,oBAAoB,EACpB,WAAW,EACZ,MAAM,gBAAgB,CAAC;AAkBxB;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,SAAS,EACpD,IAAI,EAAE,oBAAoB,GACzB,WAAW,CAAC,UAAU,EAAE,SAAS,CAAC,CA8BpC"}

package/dist/esm/llm/createClassifier.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"createClassifier.js","sourceRoot":"","sources":["../../../src/llm/createClassifier.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE7C;;;GAGG;AACH,SAAS,eAAe,CACtB,OAAiC;IAEjC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,MAA+B,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAC9B,IAA0B;IAE1B,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC;IAEzD,OAAO,KAAK,EACV,IAA2C,EAChB,EAAE;QAC7B,MAAM,iBAAiB,GAAG;YACxB,GAAG,IAAI;SACR,CAAC;QAEF,MAAM,MAAM,GAAG,cAAc,CAAC;YAC5B,QAAQ,EAAE,cAAc;YACxB,SAAS,EAAE,iBAAiB;SAC7B,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,MAAM,sBAAsB,CAAC;YAClD,KAAK;YACL,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC;YAChC,MAAM;YACN,GAAG,IAAI;SACR,CAAC,CAAC;QAEH,mEAAmE;QACnE,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAE5C,OAAO;YACL,KAAK;YACL,GAAG,cAAc;SAClB,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}

package/dist/src/llm/createClassifier.d.ts

@@ -1,6 +0,0 @@
-import { CreateClassifierArgs, EvaluatorFn } from "../types/evals";
-/**
- * A function that serves as a factory that will output a classification evaluator
- */
-export declare function createClassifier<OutputType, InputType>(args: CreateClassifierArgs): EvaluatorFn<OutputType, InputType>;
-//# sourceMappingURL=createClassifier.d.ts.map

package/dist/src/llm/createClassifier.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"createClassifier.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassifier.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,oBAAoB,EACpB,WAAW,EACZ,MAAM,gBAAgB,CAAC;AAkBxB;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,SAAS,EACpD,IAAI,EAAE,oBAAoB,GACzB,WAAW,CAAC,UAAU,EAAE,SAAS,CAAC,CA8BpC"}

package/dist/src/llm/createClassifier.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"createClassifier.js","sourceRoot":"","sources":["../../../src/llm/createClassifier.ts"],"names":[],"mappings":";;;;;;;;;;;;;AA2BA,4CAgCC;AApDD,qEAAkE;AAClE,0CAA6C;AAE7C;;;GAGG;AACH,SAAS,eAAe,CACtB,OAAiC;IAEjC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,MAA+B,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,SAAgB,gBAAgB,CAC9B,IAA0B;IAE1B,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,KAAc,IAAI,EAAb,IAAI,UAAK,IAAI,EAAlD,sCAA2C,CAAO,CAAC;IAEzD,OAAO,KAAK,EACV,IAA2C,EAChB,EAAE;QAC7B,MAAM,iBAAiB,qBAClB,IAAI,CACR,CAAC;QAEF,MAAM,MAAM,GAAG,IAAA,yBAAc,EAAC;YAC5B,QAAQ,EAAE,cAAc;YACxB,SAAS,EAAE,iBAAiB;SAC7B,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,MAAM,IAAA,+CAAsB,kBACjD,KAAK,EACL,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,EAChC,MAAM,IACH,IAAI,EACP,CAAC;QAEH,mEAAmE;QACnE,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAE5C,uBACE,KAAK,IACF,cAAc,EACjB;IACJ,CAAC,CAAC;AACJ,CAAC"}