@arizeai/phoenix-evals 0.5.0 → 0.6.0

package/src/llm/ClassificationEvaluator.ts

@@ -1,14 +1,19 @@
 import { getTemplateVariables } from "../template";
 import {
+  ClassificationChoicesMap,
   CreateClassificationEvaluatorArgs,
   EvaluatorFn,
-  Template,
+  PromptTemplate,
   WithPromptTemplate,
 } from "../types";
+import { ObjectMapping } from "../types/data";
+import { remapObject } from "../utils/objectMappingUtils";
 
 import { createClassifierFn } from "./createClassifierFn";
 import { LLMEvaluator } from "./LLMEvaluator";
 
+import { LanguageModel } from "ai";
+
 /**
  * An LLM evaluator that performs evaluation via classification
  */
@@ -17,15 +22,35 @@ export class ClassificationEvaluator<RecordType extends Record<string, unknown>>
   implements WithPromptTemplate
 {
   readonly evaluatorFn: EvaluatorFn<RecordType>;
-  readonly promptTemplate: Template;
+  readonly promptTemplate: PromptTemplate;
+  /**
+   * A dynamically computed set of prompt template variables
+   */
   private _promptTemplateVariables: string[] | undefined;
-  constructor(args: CreateClassificationEvaluatorArgs) {
+  /**
+   * The model to use for classification
+   */
+  readonly model: LanguageModel;
+  /**
+   * The choices to classify the example into
+   */
+  readonly choices: ClassificationChoicesMap;
+
+  constructor(args: CreateClassificationEvaluatorArgs<RecordType>) {
     super(args);
     this.promptTemplate = args.promptTemplate;
-    this.evaluatorFn = createClassifierFn<RecordType>(args);
+    this.model = args.model;
+    this.choices = args.choices;
+    this.evaluatorFn = createClassifierFn<RecordType>({
+      ...args,
+    });
   }
   evaluate = (example: RecordType) => {
-    return this.evaluatorFn(example);
+    return this.evaluatorFn(
+      this.inputMapping
+        ? remapObject<RecordType>(example, this.inputMapping)
+        : example
+    );
   };
   /**
    * List out the prompt template variables needed to perform evaluation
@@ -40,4 +65,15 @@ export class ClassificationEvaluator<RecordType extends Record<string, unknown>>
     // Give a copy of the variables
     return [...this._promptTemplateVariables];
   }
+  /**
+   * Binds the input mapping to the evaluator. It makes a a copy of the evaluator and returns it.
+   */
+  bindInputMapping(
+    inputMapping: ObjectMapping<RecordType>
+  ): ClassificationEvaluator<RecordType> {
+    return new ClassificationEvaluator({
+      ...this,
+      inputMapping,
+    });
+  }
 }

package/src/llm/LLMEvaluator.ts

@@ -1,22 +1,13 @@
-import {
-  CreateEvaluatorArgs,
-  EvaluationResult,
-  Evaluator,
-  OptimizationDirection,
-} from "../types";
+import { EvaluatorBase } from "../core/EvaluatorBase";
+import { CreateLLMEvaluatorArgs } from "../types";
 
 /**
  * Base class for llm evaluation metrics / scores
  */
-export abstract class LLMEvaluator<RecordType extends Record<string, unknown>>
-  implements Evaluator<RecordType>
-{
-  readonly name: string;
-  readonly kind = "LLM" as const;
-  readonly optimizationDirection?: OptimizationDirection;
-  constructor({ name, optimizationDirection }: CreateEvaluatorArgs) {
-    this.name = name;
-    this.optimizationDirection = optimizationDirection;
+export abstract class LLMEvaluator<
+  RecordType extends Record<string, unknown>,
+> extends EvaluatorBase<RecordType> {
+  constructor({ ...args }: CreateLLMEvaluatorArgs<RecordType>) {
+    super({ kind: "LLM", ...args });
   }
-  abstract evaluate(_example: RecordType): Promise<EvaluationResult>;
 }

package/src/llm/createClassificationEvaluator.ts

@@ -5,7 +5,7 @@ import { ClassificationEvaluator } from "./ClassificationEvaluator";
 export function createClassificationEvaluator<
   RecordType extends Record<string, unknown>,
 >(
-  args: CreateClassificationEvaluatorArgs
+  args: CreateClassificationEvaluatorArgs<RecordType>
 ): ClassificationEvaluator<RecordType> {
   return new ClassificationEvaluator<RecordType>(args);
 }

package/src/llm/createDocumentRelevancyEvaluator.ts

@@ -2,19 +2,24 @@ import {
   DOCUMENT_RELEVANCY_CHOICES,
   DOCUMENT_RELEVANCY_TEMPLATE,
 } from "../default_templates/DOCUMENT_RELEVANCY_TEMPLATE";
-import { CreateClassificationEvaluatorArgs, Evaluator } from "../types/evals";
+import { CreateClassificationEvaluatorArgs } from "../types/evals";
 
+import { ClassificationEvaluator } from "./ClassificationEvaluator";
 import { createClassificationEvaluator } from "./createClassificationEvaluator";
 
-export interface DocumentRelevancyEvaluatorArgs
-  extends Omit<
-    CreateClassificationEvaluatorArgs,
+export interface DocumentRelevancyEvaluatorArgs<
+  RecordType extends Record<
+    string,
+    unknown
+  > = DocumentRelevancyEvaluationRecord,
+> extends Omit<
+    CreateClassificationEvaluatorArgs<RecordType>,
     "promptTemplate" | "choices" | "optimizationDirection" | "name"
   > {
-  optimizationDirection?: CreateClassificationEvaluatorArgs["optimizationDirection"];
-  name?: CreateClassificationEvaluatorArgs["name"];
-  choices?: CreateClassificationEvaluatorArgs["choices"];
-  promptTemplate?: CreateClassificationEvaluatorArgs["promptTemplate"];
+  optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
+  name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
+  choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
+  promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
 }
 
 /**
@@ -57,7 +62,9 @@ export function createDocumentRelevancyEvaluator<
     string,
     unknown
   > = DocumentRelevancyEvaluationRecord,
->(args: DocumentRelevancyEvaluatorArgs): Evaluator<RecordType> {
+>(
+  args: DocumentRelevancyEvaluatorArgs<RecordType>
+): ClassificationEvaluator<RecordType> {
   const {
     choices = DOCUMENT_RELEVANCY_CHOICES,
     promptTemplate = DOCUMENT_RELEVANCY_TEMPLATE,
@@ -65,13 +72,11 @@ export function createDocumentRelevancyEvaluator<
     name = "document_relevancy",
     ...rest
   } = args;
-
   return createClassificationEvaluator<RecordType>({
-    ...args,
+    ...rest,
     promptTemplate,
     choices,
     optimizationDirection,
     name,
-    ...rest,
   });
 }

package/src/llm/createHallucinationEvaluator.ts

@@ -7,15 +7,16 @@ import { CreateClassificationEvaluatorArgs } from "../types/evals";
 import { ClassificationEvaluator } from "./ClassificationEvaluator";
 import { createClassificationEvaluator } from "./createClassificationEvaluator";
 
-export interface HallucinationEvaluatorArgs
-  extends Omit<
-    CreateClassificationEvaluatorArgs,
+export interface HallucinationEvaluatorArgs<
+  RecordType extends Record<string, unknown> = HallucinationEvaluationRecord,
+> extends Omit<
+    CreateClassificationEvaluatorArgs<RecordType>,
     "promptTemplate" | "choices" | "optimizationDirection" | "name"
   > {
-  optimizationDirection?: CreateClassificationEvaluatorArgs["optimizationDirection"];
-  name?: CreateClassificationEvaluatorArgs["name"];
-  choices?: CreateClassificationEvaluatorArgs["choices"];
-  promptTemplate?: CreateClassificationEvaluatorArgs["promptTemplate"];
+  optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
+  name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
+  choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
+  promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
 }
 
 /**
@@ -35,7 +36,9 @@ export type HallucinationEvaluationRecord = {
  */
 export function createHallucinationEvaluator<
   RecordType extends Record<string, unknown> = HallucinationEvaluationRecord,
->(args: HallucinationEvaluatorArgs): ClassificationEvaluator<RecordType> {
+>(
+  args: HallucinationEvaluatorArgs<RecordType>
+): ClassificationEvaluator<RecordType> {
   const {
     choices = HALLUCINATION_CHOICES,
     promptTemplate = HALLUCINATION_TEMPLATE,
@@ -44,11 +47,10 @@ export function createHallucinationEvaluator<
     ...rest
   } = args;
   return createClassificationEvaluator<RecordType>({
-    ...args,
+    ...rest,
     promptTemplate,
     choices,
     optimizationDirection,
     name,
-    ...rest,
   });
 }

package/src/template/applyTemplate.ts

@@ -1,4 +1,4 @@
-import { Template } from "../types/templating";
+import { PromptTemplate, RenderedPrompt } from "../types/templating";
 
 import { createTemplateVariablesProxy } from "./createTemplateVariablesProxy";
 
@@ -9,16 +9,29 @@ import Mustache from "mustache";
  * Uses the Mustache library to apply the variables to the template
  */
 export function formatTemplate(args: {
-  template: Template;
+  template: PromptTemplate;
   variables: Record<string, unknown>;
-}) {
+}): RenderedPrompt {
   const { template, variables } = args;
   const variablesProxy = createTemplateVariablesProxy(variables);
+  if (typeof template === "string") {
+    return renderTemplateString(template, variablesProxy);
+  }
+  return template.map((message) => {
+    if (message.content !== undefined && typeof message.content === "string") {
+      return {
+        ...message,
+        content: renderTemplateString(message.content, variablesProxy),
+      };
+    }
+    return message;
+  }) as RenderedPrompt;
+}
+
+function renderTemplateString(
+  template: string,
+  variables: Record<string, unknown>
+) {
   // Disable HTML escaping by providing a custom escape function that returns text as-is
-  return Mustache.render(
-    template,
-    variablesProxy,
-    {},
-    { escape: (text) => text }
-  );
+  return Mustache.render(template, variables, {}, { escape: (text) => text });
 }

package/src/template/createTemplateVariablesProxy.ts

@@ -3,17 +3,17 @@
  * This allows Mustache to access properties of objects (e.g., {{user.name}}) while
  * stringifying objects that are accessed as leaf values (e.g., {{user.profile}}).
  */
-export function createTemplateVariablesProxy(obj: unknown): unknown {
+export function createTemplateVariablesProxy<T>(obj: T): T {
   if (obj === null || obj === undefined) {
     return obj;
   }
 
   if (Array.isArray(obj)) {
-    return obj.map(createTemplateVariablesProxy);
+    return obj.map(createTemplateVariablesProxy) as T;
   }
 
   if (typeof obj === "object") {
-    return new Proxy(obj as Record<string, unknown>, {
+    return new Proxy(obj, {
       get(target, prop: string | symbol) {
         // Handle toString and valueOf to stringify the object when accessed directly
         if (prop === "toString") {

package/src/template/getTemplateVariables.ts

@@ -1,9 +1,9 @@
-import { Template } from "../types/templating";
+import { PromptTemplate } from "../types/templating";
 
 import Mustache from "mustache";
 
 type GetTemplateVariableArgs = {
-  template: Template;
+  template: PromptTemplate;
 };
 /**
  * Parse out the template variables of a prompt
@@ -12,6 +12,22 @@ type GetTemplateVariableArgs = {
  */
 export function getTemplateVariables(args: GetTemplateVariableArgs): string[] {
   const { template } = args;
+  if (typeof template === "string") {
+    return getTemplateVariablesFromString(template);
+  }
+  return template.reduce((acc, message) => {
+    if (message.content !== undefined && typeof message.content === "string") {
+      return [...acc, ...getTemplateVariablesFromString(message.content)];
+    }
+    return acc;
+  }, [] as string[]);
+}
+/**
+ * Parse out the template variables of a string template
+ * @param template - The template to get the variables from
+ * @returns {string[]} a list of prompt template variables
+ */
+function getTemplateVariablesFromString(template: string): string[] {
   const templateSpans = Mustache.parse(template);
   return templateSpans.reduce((acc, templateSpan) => {
     const [spanType, value] = templateSpan;

package/src/types/data.ts

@@ -0,0 +1,200 @@
+/**
+ * A value extractor that can retrieve data from an object using various methods.
+ *
+ * This type supports multiple ways to extract values from your data structure:
+ * - **String paths**: Simple property names, dot notation, or JSONPath expressions
+ * - **Function extractors**: Custom transformation functions
+ *
+ * @example
+ * **Simple property access:**
+ * ```typescript
+ * const getter: ValueGetter<{ name: string }> = "name";
+ * ```
+ *
+ * @example
+ * **Dot notation for nested properties:**
+ * ```typescript
+ * const getter: ValueGetter<{ user: { profile: { name: string } } }> = "user.profile.name";
+ * ```
+ *
+ * @example
+ * **Array element access:**
+ * ```typescript
+ * const getter: ValueGetter<{ items: string[] }> = "items[0]";
+ * ```
+ *
+ * @example
+ * **JSONPath expression:**
+ * ```typescript
+ * const getter: ValueGetter<{ items: Array<{ id: number }> }> = "$.items[*].id";
+ * ```
+ *
+ * @example
+ * **Function-based extraction:**
+ * ```typescript
+ * const getter: ValueGetter<{ firstName: string; lastName: string }> =
+ *   (data) => `${data.firstName} ${data.lastName}`;
+ * ```
+ *
+ * @example
+ * **Complex transformation:**
+ * ```typescript
+ * const getter: ValueGetter<{ scores: number[] }> =
+ *   (data) => data.scores.reduce((a, b) => a + b, 0) / data.scores.length;
+ * ```
+ *
+ * @typeParam DataType - The type of the data object to extract values from
+ */
+export type ValueGetter<DataType extends Record<string, unknown>> =
+  | string
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  | ((data: DataType) => any);
+
+/**
+ * A mapping configuration that transforms data from one structure to another.
+ *
+ * This type defines how to map fields from your data structure to the fields
+ * expected by an evaluator or other component. The mapping is flexible and
+ * supports multiple extraction methods.
+ *
+ * **Key Features:**
+ * - Preserves original data fields
+ * - Adds/overrides fields with mapped values
+ * - Supports nested property access
+ * - Supports array element access
+ * - Supports JSONPath expressions for complex queries
+ * - Supports function-based transformations
+ *
+ * @example
+ * **Basic field mapping:**
+ * ```typescript
+ * type MyData = {
+ *   userQuery: string;
+ *   context: string;
+ *   response: string;
+ * };
+ *
+ * const mapping: ObjectMapping<MyData> = {
+ *   input: "userQuery",      // Map "input" to "userQuery"
+ *   reference: "context",    // Map "reference" to "context"
+ *   output: "response",       // Map "output" to "response"
+ * };
+ * ```
+ *
+ * @example
+ * **Nested property mapping:**
+ * ```typescript
+ * type ApiData = {
+ *   request: {
+ *     body: {
+ *       query: string;
+ *       context: string;
+ *     };
+ *   };
+ *   response: {
+ *     data: {
+ *       text: string;
+ *     };
+ *   };
+ * };
+ *
+ * const mapping: ObjectMapping<ApiData> = {
+ *   input: "request.body.query",
+ *   reference: "request.body.context",
+ *   output: "response.data.text",
+ * };
+ * ```
+ *
+ * @example
+ * **Array element access:**
+ * ```typescript
+ * type DataWithArrays = {
+ *   messages: Array<{ role: string; content: string }>;
+ *   sources: string[];
+ * };
+ *
+ * const mapping: ObjectMapping<DataWithArrays> = {
+ *   firstMessage: "messages[0].content",
+ *   lastSource: "sources[-1]",  // Last element
+ *   allRoles: "$.messages[*].role",  // JSONPath for all roles
+ * };
+ * ```
+ *
+ * @example
+ * **Function-based transformations:**
+ * ```typescript
+ * type RawData = {
+ *   firstName: string;
+ *   lastName: string;
+ *   contexts: string[];
+ *   scores: number[];
+ * };
+ *
+ * const mapping: ObjectMapping<RawData> = {
+ *   // Combine fields
+ *   fullName: (data) => `${data.firstName} ${data.lastName}`,
+ *   // Transform array to string
+ *   contextText: (data) => data.contexts.join("\n\n"),
+ *   // Calculate derived value
+ *   averageScore: (data) =>
+ *     data.scores.reduce((a, b) => a + b, 0) / data.scores.length,
+ *   // Conditional logic
+ *   status: (data) => data.scores.length > 0 ? "active" : "inactive",
+ * };
+ * ```
+ *
+ * @example
+ * **Mixed mapping types:**
+ * ```typescript
+ * type ComplexData = {
+ *   user: {
+ *     name: string;
+ *     email: string;
+ *   };
+ *   items: Array<{ id: number; name: string }>;
+ *   metadata: {
+ *     tags: string[];
+ *   };
+ * };
+ *
+ * const mapping: ObjectMapping<ComplexData> = {
+ *   // Simple dot notation
+ *   userName: "user.name",
+ *   // Array access
+ *   firstItemId: "items[0].id",
+ *   // JSONPath for complex query
+ *   allItemNames: "$.items[*].name",
+ *   // Function for transformation
+ *   formattedTags: (data) => data.metadata.tags.map(t => `#${t}`).join(" "),
+ * };
+ * ```
+ *
+ * @example
+ * **Real-world evaluator binding:**
+ * ```typescript
+ * import { bindEvaluator, createHallucinationEvaluator } from "@arizeai/phoenix-evals";
+ *
+ * type QAData = {
+ *   question: string;
+ *   context: string;
+ *   answer: string;
+ * };
+ *
+ * const mapping: ObjectMapping<QAData> = {
+ *   input: "question",      // Evaluator expects "input"
+ *   reference: "context",   // Evaluator expects "reference"
+ *   output: "answer",       // Evaluator expects "output"
+ * };
+ *
+ * const evaluator = bindEvaluator(
+ *   createHallucinationEvaluator({ model: openai("gpt-4") }),
+ *   { inputMapping: mapping }
+ * );
+ * ```
+ *
+ * @typeParam DataType - The type of the data object being mapped
+ */
+export type ObjectMapping<DataType extends Record<string, unknown>> = Record<
+  string,
+  ValueGetter<DataType>
+>;

package/src/types/evals.ts

@@ -1,5 +1,6 @@
+import { ObjectMapping } from "./data";
 import { WithTelemetry } from "./otel";
-import { Template } from "./templating";
+import { PromptTemplate } from "./templating";
 
 import { LanguageModel } from "ai";
 
@@ -78,25 +79,44 @@ export interface CreateClassifierArgs extends WithTelemetry {
   /**
    * The prompt template to use for classification
    */
-  promptTemplate: Template;
+  promptTemplate: PromptTemplate;
 }
 
-export interface CreateEvaluatorArgs {
+export interface CreateEvaluatorArgs<
+  ExampleType extends Record<string, unknown> = Record<string, unknown>,
+> extends WithTelemetry {
   /**
    * The name of the metric that the evaluator produces
    * E.x. "correctness"
    */
   name: string;
+  /**
+   * The kind of the evaluation. Also known as the "kind" of evaluator.
+   */
+  kind: EvaluationKind;
   /**
    * If present, represents the direction in which you want the metric to be optimized
    * E.x. "MAXIMIZE" means you want the number to be higher.
    */
   optimizationDirection?: OptimizationDirection;
+  /**
+   * The mapping of the input to evaluate to the shape that the evaluator expects
+   */
+  inputMapping?: ObjectMapping<ExampleType>;
 }
 
-export interface CreateClassificationEvaluatorArgs
-  extends CreateClassifierArgs,
-    CreateEvaluatorArgs {}
+export type CreateLLMEvaluatorArgs<RecordType extends Record<string, unknown>> =
+  Omit<CreateEvaluatorArgs<RecordType>, "kind">;
+
+export interface CreateClassificationEvaluatorArgs<
+  RecordType extends Record<string, unknown>,
+> extends CreateClassifierArgs,
+    CreateLLMEvaluatorArgs<RecordType> {
+  /**
+   * The prompt template to use for classification
+   */
+  promptTemplate: PromptTemplate;
+}
 
 export type EvaluatorFn<ExampleType extends Record<string, unknown>> = (
   args: ExampleType
@@ -136,7 +156,7 @@ interface EvaluatorDescription {
  * The Base Evaluator interface
  * This is the interface that all evaluators must implement
  */
-export interface Evaluator<ExampleType extends Record<string, unknown>>
+export interface EvaluatorInterface<ExampleType extends Record<string, unknown>>
   extends EvaluatorDescription {
   /**
    * The function that evaluates the example

package/src/types/index.ts

@@ -1,3 +1,4 @@
 export * from "./evals";
 export * from "./templating";
 export * from "./otel";
+export * from "./data";

package/src/types/otel.ts

@@ -1,17 +1,19 @@
 import { Tracer } from "@opentelemetry/api";
 
+export type TelemetryConfig = {
+  /**
+   * Whether OpenTelemetry is enabled on the call.
+   * Defaults to true for visibility into the evals calls.
+   * @default true
+   */
+  isEnabled?: boolean;
+  /**
+   * The tracer to use for the call.
+   * If not provided, the traces will get picked up by the global tracer.
+   */
+  tracer?: Tracer;
+};
+
 export type WithTelemetry = {
-  telemetry?: {
-    /**
-     * Whether OpenTelemetry is enabled on the call.
-     * Defaults to true for visibility into the evals calls.
-     * @default true
-     */
-    isEnabled?: boolean;
-    /**
-     * The tracer to use for the call.
-     * If not provided, the traces will get picked up by the global tracer.
-     */
-    tracer?: Tracer;
-  };
+  telemetry?: TelemetryConfig;
 };

package/src/types/templating.ts

@@ -1,10 +1,13 @@
-export type Template = string;
+import type { ModelMessage } from "ai";
+
+export type PromptTemplate = string | Array<ModelMessage>;
+export type RenderedPrompt = string | Array<ModelMessage>;
 export type TemplateVariables = Record<string, unknown>;
 
-/**
- * A class or object that has a prompt template
- */
 export interface WithPromptTemplate {
-  readonly promptTemplate: Template;
-  get promptTemplateVariables(): string[];
+  readonly promptTemplate: PromptTemplate;
+  /**
+   * List out the prompt template variables needed to perform evaluation
+   */
+  readonly promptTemplateVariables: string[];
 }