@arizeai/phoenix-evals 0.5.1 → 0.6.0

package/src/utils/bindEvaluator.ts

@@ -0,0 +1,229 @@
+import { EvaluatorBase } from "../core/EvaluatorBase";
+import { ObjectMapping } from "../types/data";
+
+/**
+ * Context for binding an evaluator with input mapping configuration.
+ *
+ * This type defines the structure for binding an evaluator to a specific data shape
+ * by mapping the evaluator's expected input fields to the actual data structure.
+ *
+ * @example
+ * ```typescript
+ * // Map evaluator fields to your data structure
+ * const context: BindingContext<MyDataType> = {
+ *   inputMapping: {
+ *     input: "userQuery",        // Maps "input" to "userQuery" field
+ *     reference: "context",      // Maps "reference" to "context" field
+ *     output: "modelResponse",   // Maps "output" to "modelResponse" field
+ *   },
+ * };
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Using nested property access
+ * const context: BindingContext<ApiResponse> = {
+ *   inputMapping: {
+ *     input: "request.body.query",
+ *     reference: "request.body.context",
+ *     output: "response.data.text",
+ *   },
+ * };
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Using function-based mapping for transformations
+ * const context: BindingContext<RawData> = {
+ *   inputMapping: {
+ *     input: "question",
+ *     reference: (data) => data.context.join("\n"),  // Transform array to string
+ *     output: "answer",
+ *   },
+ * };
+ * ```
+ *
+ * @typeParam RecordType - The type of the data record that will be evaluated
+ */
+export type BindingContext<RecordType extends Record<string, unknown>> = {
+  /**
+   * Mapping of evaluator input fields to data source fields.
+   *
+   * The keys represent the field names expected by the evaluator (e.g., "input", "output", "reference"),
+   * and the values specify how to extract those fields from your data structure.
+   *
+   * Supports:
+   * - Simple property names: `"fieldName"`
+   * - Dot notation: `"user.profile.name"`
+   * - Array access: `"items[0].id"`
+   * - JSONPath expressions: `"$.items[*].id"`
+   * - Function extractors: `(data) => data.customField.toUpperCase()`
+   */
+  inputMapping: ObjectMapping<RecordType>;
+};
+
+/**
+ * Binds an evaluator to a specific data structure using input mapping.
+ *
+ * This function creates a new evaluator instance that automatically transforms
+ * your data structure to match what the evaluator expects. This is particularly
+ * useful when your data schema doesn't match the evaluator's expected input format.
+ *
+ * @param evaluator - The evaluator to bind (e.g., a hallucination evaluator)
+ * @param context - The binding context containing the input mapping configuration
+ * @returns A new evaluator instance with the input mapping applied
+ *
+ * @example
+ * **Basic usage with simple field mapping:**
+ * ```typescript
+ * import { bindEvaluator, createHallucinationEvaluator } from "@arizeai/phoenix-evals";
+ * import { openai } from "@ai-sdk/openai";
+ *
+ * type MyData = {
+ *   question: string;
+ *   context: string;
+ *   answer: string;
+ * };
+ *
+ * const evaluator = bindEvaluator<MyData>(
+ *   createHallucinationEvaluator({ model: openai("gpt-4") }),
+ *   {
+ *     inputMapping: {
+ *       input: "question",      // Evaluator expects "input", map from "question"
+ *       reference: "context",   // Evaluator expects "reference", map from "context"
+ *       output: "answer",        // Evaluator expects "output", map from "answer"
+ *     },
+ *   }
+ * );
+ *
+ * // Now you can evaluate with your data structure
+ * const result = await evaluator.evaluate({
+ *   question: "What is AI?",
+ *   context: "AI is artificial intelligence...",
+ *   answer: "AI stands for artificial intelligence",
+ * });
+ * ```
+ *
+ * @example
+ * **Using nested property access:**
+ * ```typescript
+ * type ApiResponse = {
+ *   request: {
+ *     body: {
+ *       query: string;
+ *       context: string;
+ *     };
+ *   };
+ *   response: {
+ *     data: {
+ *       text: string;
+ *     };
+ *   };
+ * };
+ *
+ * const evaluator = bindEvaluator<ApiResponse>(
+ *   createHallucinationEvaluator({ model: openai("gpt-4") }),
+ *   {
+ *     inputMapping: {
+ *       input: "request.body.query",
+ *       reference: "request.body.context",
+ *       output: "response.data.text",
+ *     },
+ *   }
+ * );
+ * ```
+ *
+ * @example
+ * **Using function-based mapping for data transformation:**
+ * ```typescript
+ * type RawData = {
+ *   question: string;
+ *   contexts: string[];  // Array of context strings
+ *   answer: string;
+ * };
+ *
+ * const evaluator = bindEvaluator<RawData>(
+ *   createHallucinationEvaluator({ model: openai("gpt-4") }),
+ *   {
+ *     inputMapping: {
+ *       input: "question",
+ *       // Transform array to single string
+ *       reference: (data) => data.contexts.join("\n\n"),
+ *       output: "answer",
+ *     },
+ *   }
+ * );
+ * ```
+ *
+ * @example
+ * **Using JSONPath for complex queries:**
+ * ```typescript
+ * type ComplexData = {
+ *   conversation: {
+ *     messages: Array<{ role: string; content: string }>;
+ *   };
+ *   metadata: {
+ *     sources: string[];
+ *   };
+ * };
+ *
+ * const evaluator = bindEvaluator<ComplexData>(
+ *   createHallucinationEvaluator({ model: openai("gpt-4") }),
+ *   {
+ *     inputMapping: {
+ *       // Extract last user message
+ *       input: "$.conversation.messages[?(@.role=='user')].content[-1]",
+ *       // Extract all sources
+ *       reference: "$.metadata.sources[*]",
+ *       // Extract last assistant message
+ *       output: "$.conversation.messages[?(@.role=='assistant')].content[-1]",
+ *     },
+ *   }
+ * );
+ * ```
+ *
+ * @example
+ * **Binding multiple evaluators with different mappings:**
+ * ```typescript
+ * type EvaluationData = {
+ *   userQuery: string;
+ *   systemContext: string;
+ *   modelOutput: string;
+ *   expectedOutput?: string;
+ * };
+ *
+ * // Hallucination evaluator
+ * const hallucinationEvaluator = bindEvaluator<EvaluationData>(
+ *   createHallucinationEvaluator({ model: openai("gpt-4") }),
+ *   {
+ *     inputMapping: {
+ *       input: "userQuery",
+ *       reference: "systemContext",
+ *       output: "modelOutput",
+ *     },
+ *   }
+ * );
+ *
+ * // Document relevancy evaluator (if it exists)
+ * const relevancyEvaluator = bindEvaluator<EvaluationData>(
+ *   createDocumentRelevancyEvaluator({ model: openai("gpt-4") }),
+ *   {
+ *     inputMapping: {
+ *       query: "userQuery",
+ *       document: "systemContext",
+ *       output: "modelOutput",
+ *     },
+ *   }
+ * );
+ * ```
+ */
+export function bindEvaluator<RecordType extends Record<string, unknown>>(
+  evaluator: EvaluatorBase<RecordType>,
+  context: BindingContext<RecordType>
+): EvaluatorBase<RecordType> {
+  let boundEvaluator: EvaluatorBase<RecordType> = evaluator;
+  if (context.inputMapping) {
+    boundEvaluator = boundEvaluator.bindInputMapping(context.inputMapping);
+  }
+  return boundEvaluator;
+}

package/src/utils/index.ts

@@ -0,0 +1 @@
+export * from "./bindEvaluator";

package/src/utils/objectMappingUtils.ts

@@ -0,0 +1,202 @@
+import { ObjectMapping, ValueGetter } from "../types/data";
+
+import { JSONPath } from "jsonpath-plus";
+
+/**
+ * Remaps an object by applying field mappings while preserving original data.
+ *
+ * This function takes your original data object and a mapping configuration,
+ * then returns a new object that contains:
+ * - All original fields from the input data
+ * - Additional/overridden fields based on the mapping
+ *
+ * The mapping allows you to extract values using:
+ * - Simple property names: `"fieldName"`
+ * - Dot notation: `"user.profile.name"`
+ * - Array access: `"items[0].id"`
+ * - JSONPath expressions: `"$.items[*].id"`
+ * - Function extractors: `(data) => data.customField`
+ *
+ * @param data - The input data object to remap
+ * @param mapping - The mapping configuration defining how to extract/transform values
+ * @returns A new object with original fields plus mapped fields
+ *
+ * @example
+ * **Basic remapping:**
+ * ```typescript
+ * const data = {
+ *   name: "John",
+ *   age: 30,
+ *   email: "john@example.com",
+ * };
+ *
+ * const mapping: ObjectMapping<typeof data> = {
+ *   userName: "name",
+ *   userAge: "age",
+ * };
+ *
+ * const result = remapObject(data, mapping);
+ * // Result: {
+ * //   name: "John",
+ * //   age: 30,
+ * //   email: "john@example.com",
+ * //   userName: "John",      // Added from mapping
+ * //   userAge: 30,            // Added from mapping
+ * // }
+ * ```
+ *
+ * @example
+ * **Nested property extraction:**
+ * ```typescript
+ * const data = {
+ *   user: {
+ *     profile: {
+ *       firstName: "John",
+ *       lastName: "Doe",
+ *     },
+ *     email: "john@example.com",
+ *   },
+ * };
+ *
+ * const mapping: ObjectMapping<typeof data> = {
+ *   firstName: "user.profile.firstName",
+ *   lastName: "user.profile.lastName",
+ *   email: "user.email",
+ * };
+ *
+ * const result = remapObject(data, mapping);
+ * // Result includes original data plus:
+ * // {
+ * //   firstName: "John",
+ * //   lastName: "Doe",
+ * //   email: "john@example.com",
+ * // }
+ * ```
+ *
+ * @example
+ * **Array element access:**
+ * ```typescript
+ * const data = {
+ *   items: [
+ *     { id: 1, name: "Apple" },
+ *     { id: 2, name: "Banana" },
+ *   ],
+ * };
+ *
+ * const mapping: ObjectMapping<typeof data> = {
+ *   firstItemId: "items[0].id",
+ *   firstItemName: "items[0].name",
+ * };
+ *
+ * const result = remapObject(data, mapping);
+ * // Result includes:
+ * // {
+ * //   firstItemId: 1,
+ * //   firstItemName: "Apple",
+ * // }
+ * ```
+ *
+ * @example
+ * **Function-based transformation:**
+ * ```typescript
+ * const data = {
+ *   firstName: "John",
+ *   lastName: "Doe",
+ *   scores: [85, 92, 78],
+ * };
+ *
+ * const mapping: ObjectMapping<typeof data> = {
+ *   fullName: (data) => `${data.firstName} ${data.lastName}`,
+ *   averageScore: (data) =>
+ *     data.scores.reduce((a, b) => a + b, 0) / data.scores.length,
+ * };
+ *
+ * const result = remapObject(data, mapping);
+ * // Result includes:
+ * // {
+ * //   fullName: "John Doe",
+ * //   averageScore: 85,
+ * // }
+ * ```
+ *
+ * @example
+ * **Field override:**
+ * ```typescript
+ * const data = {
+ *   name: "John",
+ *   status: "inactive",
+ * };
+ *
+ * const mapping: ObjectMapping<typeof data> = {
+ *   // Override existing field
+ *   status: (data) => data.name === "John" ? "active" : "inactive",
+ *   // Add new field
+ *   displayName: (data) => `User: ${data.name}`,
+ * };
+ *
+ * const result = remapObject(data, mapping);
+ * // Result:
+ * // {
+ * //   name: "John",
+ * //   status: "active",        // Overridden
+ * //   displayName: "User: John", // Added
+ * // }
+ * ```
+ *
+ * @example
+ * **Real-world evaluator usage:**
+ * ```typescript
+ * // Your data structure
+ * const example = {
+ *   question: "What is AI?",
+ *   context: "AI is artificial intelligence...",
+ *   answer: "AI stands for artificial intelligence",
+ * };
+ *
+ * // Evaluator expects: { input, reference, output }
+ * const mapping: ObjectMapping<typeof example> = {
+ *   input: "question",
+ *   reference: "context",
+ *   output: "answer",
+ * };
+ *
+ * const remapped = remapObject(example, mapping);
+ * // Now remapped has: { question, context, answer, input, reference, output }
+ * // The evaluator can access input, reference, and output fields
+ * ```
+ */
+export function remapObject<DataType extends Record<string, unknown>>(
+  data: DataType,
+  mapping: ObjectMapping<DataType>
+): DataType {
+  return {
+    ...data,
+    ...Object.fromEntries(
+      Object.entries(mapping).map(([key, value]) => [
+        key,
+        getMappedObjectValue(data, value),
+      ])
+    ),
+  };
+}
+
+/**
+ * Extracts a value from a data object using a value getter.
+ *
+ * This internal function handles the actual extraction logic, supporting both
+ * string-based paths (including JSONPath) and function-based extractors.
+ *
+ * @param data - The data object to extract from
+ * @param valueGetter - The value getter (string path or function)
+ * @returns The extracted value
+ *
+ * @internal
+ */
+function getMappedObjectValue<DataType extends Record<string, unknown>>(
+  data: DataType,
+  valueGetter: ValueGetter<DataType>
+): DataType[keyof DataType] {
+  return typeof valueGetter === "function"
+    ? valueGetter(data)
+    : JSONPath({ path: valueGetter, json: data, wrap: false });
+}