@arizeai/phoenix-evals 0.0.7 → 0.0.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arizeai/phoenix-evals",
-  "version": "0.0.7",
+  "version": "0.0.8",
   "description": "A library for running evaluations for AI use cases",
   "main": "dist/src/index.js",
   "module": "dist/esm/index.js",
@@ -41,7 +41,7 @@
   "author": "oss@arize.com",
   "license": "ELv2",
   "devDependencies": {
-    "@ai-sdk/openai": "^1.1.15",
+    "@ai-sdk/openai": "^2.0.0",
     "@arizeai/openinference-instrumentation-openai": "^2.3.0",
     "@types/mustache": "^4.2.6",
     "@types/node": "^24.0.10",
@@ -50,16 +50,16 @@
     "typedoc": "^0.27.9",
     "typescript": "^5.8.2",
     "vitest": "^2.1.9",
-    "@arizeai/phoenix-client": "2.3.5"
+    "@arizeai/phoenix-client": "2.4.0"
   },
   "engines": {
     "node": ">=18"
   },
   "dependencies": {
     "@opentelemetry/api": "^1.9.0",
-    "ai": "^4.1.47",
+    "ai": "^5.0.0",
     "mustache": "^4.2.0",
-    "zod": "^3.25.75"
+    "zod": "^4.0.14"
   },
   "scripts": {
     "clean": "rimraf dist",

package/src/default_templates/DOCUMENT_RELEVANCY_TEMPLATE.ts

@@ -0,0 +1,25 @@
+export const DOCUMENT_RELEVANCY_TEMPLATE = `
+You are comparing a document to a question and trying to determine if the document text
+contains information relevant to answering the question. Here is the data:
+
+    [BEGIN DATA]
+    ************
+    [Question]: {{input}}
+    ************
+    [Document text]: {{documentText}}
+    ************
+    [END DATA]
+
+Compare the Question above to the Document text. You must determine whether the Document text
+contains information that can answer the Question. Please focus on whether the very specific
+question can be answered by the information in the Document text.
+Your response must be single word, either "relevant" or "unrelated",
+and should not contain any text or characters aside from that word.
+"unrelated" means that the document text does not contain an answer to the Question.
+"relevant" means the document text contains an answer to the Question.
+`;
+
+export const DOCUMENT_RELEVANCY_CHOICES = {
+  relevant: 1,
+  unrelated: 0,
+};

package/src/default_templates/index.ts

@@ -1 +1,2 @@
 export * from "./HALLUCINATION_TEMPLATE";
+export * from "./DOCUMENT_RELEVANCY_TEMPLATE";

package/src/llm/createClassifier.ts

@@ -1,6 +1,5 @@
 import {
   ClassificationChoicesMap,
-  EvaluationArgs,
   EvaluationResult,
   CreateClassifierArgs,
   EvaluatorFn,
@@ -25,14 +24,12 @@ function choicesToLabels(
 /**
  * A function that serves as a factory that will output a classification evaluator
  */
-export function createClassifier<OutputType, InputType>(
+export function createClassifier<ExampleType extends Record<string, unknown>>(
   args: CreateClassifierArgs
-): EvaluatorFn<OutputType, InputType> {
+): EvaluatorFn<ExampleType> {
   const { model, choices, promptTemplate, ...rest } = args;
 
-  return async (
-    args: EvaluationArgs<OutputType, InputType>
-  ): Promise<EvaluationResult> => {
+  return async (args: ExampleType): Promise<EvaluationResult> => {
     const templateVariables = {
       ...args,
     };

package/src/llm/createDocumentRelevancyEvaluator.ts

@@ -0,0 +1,64 @@
+import { createClassifier } from "./createClassifier";
+import { CreateClassifierArgs, EvaluatorFn } from "../types/evals";
+import {
+  DOCUMENT_RELEVANCY_TEMPLATE,
+  DOCUMENT_RELEVANCY_CHOICES,
+} from "../default_templates/DOCUMENT_RELEVANCY_TEMPLATE";
+
+export interface DocumentRelevancyEvaluatorArgs
+  extends Omit<CreateClassifierArgs, "promptTemplate" | "choices"> {
+  choices?: CreateClassifierArgs["choices"];
+  promptTemplate?: CreateClassifierArgs["promptTemplate"];
+}
+
+/**
+ * An example to be evaluated by the document relevancy evaluator.
+ */
+export type DocumentRelevancyExample = {
+  input: string;
+  documentText: string;
+};
+
+/**
+ * Creates a document relevancy evaluator function.
+ *
+ * This function returns an evaluator that determines whether a given document text
+ * is relevant to a provided input question. The evaluator uses a classification model
+ * and a prompt template to make its determination.
+ *
+ * @param args - The arguments for creating the document relevancy evaluator.
+ * @param args.model - The model to use for classification.
+ * @param args.choices - The possible classification choices (defaults to DOCUMENT_RELEVANCY_CHOICES).
+ * @param args.promptTemplate - The prompt template to use (defaults to DOCUMENT_RELEVANCY_TEMPLATE).
+ * @param args.telemetry - The telemetry to use for the evaluator.
+ *
+ * @returns An evaluator function that takes a {@link DocumentRelevancyExample} and returns a classification result
+ * indicating whether the document is relevant to the input question.
+ *
+ * @example
+ * ```ts
+ * const evaluator = createDocumentRelevancyEvaluator({ model: openai("gpt-4o-mini") });
+ * const result = await evaluator({
+ *   input: "What is the capital of France?",
+ *   documentText: "Paris is the capital and most populous city of France.",
+ * });
+ * console.log(result.label); // "relevant" or "unrelated"
+ * ```
+ */
+export function createDocumentRelevancyEvaluator(
+  args: DocumentRelevancyEvaluatorArgs
+): EvaluatorFn<DocumentRelevancyExample> {
+  const {
+    choices = DOCUMENT_RELEVANCY_CHOICES,
+    promptTemplate = DOCUMENT_RELEVANCY_TEMPLATE,
+    ...rest
+  } = args;
+  const documentRelevancyEvaluatorFn =
+    createClassifier<DocumentRelevancyExample>({
+      ...args,
+      promptTemplate,
+      choices,
+      ...rest,
+    });
+  return documentRelevancyEvaluatorFn;
+}

package/src/llm/createHallucinationEvaluator.ts

@@ -10,6 +10,16 @@ export interface HallucinationEvaluatorArgs
   choices?: CreateClassifierArgs["choices"];
   promptTemplate?: CreateClassifierArgs["promptTemplate"];
 }
+
+/**
+ * An example to be evaluated by the hallucination evaluator.
+ */
+export type HallucinationExample = {
+  input: string;
+  output: string;
+  reference?: string;
+  context?: string;
+};
 /**
  * Creates a function that evaluates whether an answer is factual or hallucinated based on a query and reference text.
  *
@@ -18,13 +28,13 @@ export interface HallucinationEvaluatorArgs
  */
 export function createHallucinationEvaluator(
   args: HallucinationEvaluatorArgs
-): EvaluatorFn<string, string> {
+): EvaluatorFn<HallucinationExample> {
   const {
     choices = HALLUCINATION_CHOICES,
     promptTemplate = HALLUCINATION_TEMPLATE,
     ...rest
   } = args;
-  const hallucinationEvaluatorFn = createClassifier<string, string>({
+  const hallucinationEvaluatorFn = createClassifier<HallucinationExample>({
     ...args,
     promptTemplate,
     choices,

package/src/llm/index.ts

@@ -1,3 +1,4 @@
 export * from "./generateClassification";
 export * from "./createClassifier";
 export * from "./createHallucinationEvaluator";
+export * from "./createDocumentRelevancyEvaluator";

package/src/types/evals.ts

@@ -2,9 +2,9 @@ import { LanguageModel } from "ai";
 import { WithTelemetry } from "./otel";
 
 /**
- * The arguments for an evaluation
+ * A specific AI example that is under evaluation
  */
-export interface EvaluationArgs<OutputType, InputType> {
+export interface ExampleRecord<OutputType, InputType> {
   output: OutputType;
   expected?: OutputType;
   input?: InputType;
@@ -15,9 +15,7 @@ export interface WithLLM {
   model: LanguageModel;
 }
 
-export interface LLMEvaluationArgs<OutputType, InputType>
-  extends EvaluationArgs<OutputType, InputType>,
-    WithLLM {}
+export interface LLMEvaluationArgs extends WithLLM {}
 
 /**
  * The result of an evaluation
@@ -80,6 +78,6 @@ export interface CreateClassifierArgs extends WithTelemetry {
   promptTemplate: string;
 }
 
-export type EvaluatorFn<OutputType, InputType> = (
-  args: EvaluationArgs<OutputType, InputType>
+export type EvaluatorFn<ExampleType extends Record<string, unknown>> = (
+  args: ExampleType
 ) => Promise<EvaluationResult>;

package/src/types/prompts.ts

@@ -1,9 +1,9 @@
-import { CoreMessage, Message } from "ai";
+import { ModelMessage } from "ai";
 
 /**
- * Prompt part of the AI function options.
- * It contains a system message, a simple text prompt, or a list of messages.
- * Note: this is pulled from the `ai` package and is used as a compatibility type.
+ * Prompt part of the AI function options for model generation.
+ * It contains a system message, a simple text prompt, or a list of model messages.
+ * Uses ModelMessage format compatible with AI SDK v5 generateObject function.
  */
 export interface WithPrompt {
   /**
@@ -15,7 +15,8 @@ export interface WithPrompt {
    */
   prompt?: string;
   /**
-   * A list of messages. You can either use `prompt` or `messages` but not both.
+   * A list of model messages. You can either use `prompt` or `messages` but not both.
+   * Uses ModelMessage format for compatibility with AI SDK v5.
    */
-  messages?: Array<CoreMessage> | Array<Omit<Message, "id">>;
+  messages?: Array<ModelMessage>;
 }