@arizeai/phoenix-evals 0.5.1 → 0.6.1

package/dist/src/utils/bindEvaluator.js

@@ -0,0 +1,166 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.bindEvaluator = bindEvaluator;
+/**
+ * Binds an evaluator to a specific data structure using input mapping.
+ *
+ * This function creates a new evaluator instance that automatically transforms
+ * your data structure to match what the evaluator expects. This is particularly
+ * useful when your data schema doesn't match the evaluator's expected input format.
+ *
+ * @param evaluator - The evaluator to bind (e.g., a hallucination evaluator)
+ * @param context - The binding context containing the input mapping configuration
+ * @returns A new evaluator instance with the input mapping applied
+ *
+ * @example
+ * **Basic usage with simple field mapping:**
+ * ```typescript
+ * import { bindEvaluator, createHallucinationEvaluator } from "@arizeai/phoenix-evals";
+ * import { openai } from "@ai-sdk/openai";
+ *
+ * type MyData = {
+ *   question: string;
+ *   context: string;
+ *   answer: string;
+ * };
+ *
+ * const evaluator = bindEvaluator<MyData>(
+ *   createHallucinationEvaluator({ model: openai("gpt-4") }),
+ *   {
+ *     inputMapping: {
+ *       input: "question",      // Evaluator expects "input", map from "question"
+ *       reference: "context",   // Evaluator expects "reference", map from "context"
+ *       output: "answer",        // Evaluator expects "output", map from "answer"
+ *     },
+ *   }
+ * );
+ *
+ * // Now you can evaluate with your data structure
+ * const result = await evaluator.evaluate({
+ *   question: "What is AI?",
+ *   context: "AI is artificial intelligence...",
+ *   answer: "AI stands for artificial intelligence",
+ * });
+ * ```
+ *
+ * @example
+ * **Using nested property access:**
+ * ```typescript
+ * type ApiResponse = {
+ *   request: {
+ *     body: {
+ *       query: string;
+ *       context: string;
+ *     };
+ *   };
+ *   response: {
+ *     data: {
+ *       text: string;
+ *     };
+ *   };
+ * };
+ *
+ * const evaluator = bindEvaluator<ApiResponse>(
+ *   createHallucinationEvaluator({ model: openai("gpt-4") }),
+ *   {
+ *     inputMapping: {
+ *       input: "request.body.query",
+ *       reference: "request.body.context",
+ *       output: "response.data.text",
+ *     },
+ *   }
+ * );
+ * ```
+ *
+ * @example
+ * **Using function-based mapping for data transformation:**
+ * ```typescript
+ * type RawData = {
+ *   question: string;
+ *   contexts: string[];  // Array of context strings
+ *   answer: string;
+ * };
+ *
+ * const evaluator = bindEvaluator<RawData>(
+ *   createHallucinationEvaluator({ model: openai("gpt-4") }),
+ *   {
+ *     inputMapping: {
+ *       input: "question",
+ *       // Transform array to single string
+ *       reference: (data) => data.contexts.join("\n\n"),
+ *       output: "answer",
+ *     },
+ *   }
+ * );
+ * ```
+ *
+ * @example
+ * **Using JSONPath for complex queries:**
+ * ```typescript
+ * type ComplexData = {
+ *   conversation: {
+ *     messages: Array<{ role: string; content: string }>;
+ *   };
+ *   metadata: {
+ *     sources: string[];
+ *   };
+ * };
+ *
+ * const evaluator = bindEvaluator<ComplexData>(
+ *   createHallucinationEvaluator({ model: openai("gpt-4") }),
+ *   {
+ *     inputMapping: {
+ *       // Extract last user message
+ *       input: "$.conversation.messages[?(@.role=='user')].content[-1]",
+ *       // Extract all sources
+ *       reference: "$.metadata.sources[*]",
+ *       // Extract last assistant message
+ *       output: "$.conversation.messages[?(@.role=='assistant')].content[-1]",
+ *     },
+ *   }
+ * );
+ * ```
+ *
+ * @example
+ * **Binding multiple evaluators with different mappings:**
+ * ```typescript
+ * type EvaluationData = {
+ *   userQuery: string;
+ *   systemContext: string;
+ *   modelOutput: string;
+ *   expectedOutput?: string;
+ * };
+ *
+ * // Hallucination evaluator
+ * const hallucinationEvaluator = bindEvaluator<EvaluationData>(
+ *   createHallucinationEvaluator({ model: openai("gpt-4") }),
+ *   {
+ *     inputMapping: {
+ *       input: "userQuery",
+ *       reference: "systemContext",
+ *       output: "modelOutput",
+ *     },
+ *   }
+ * );
+ *
+ * // Document relevancy evaluator (if it exists)
+ * const relevancyEvaluator = bindEvaluator<EvaluationData>(
+ *   createDocumentRelevanceEvaluator({ model: openai("gpt-4") }),
+ *   {
+ *     inputMapping: {
+ *       query: "userQuery",
+ *       document: "systemContext",
+ *       output: "modelOutput",
+ *     },
+ *   }
+ * );
+ * ```
+ */
+function bindEvaluator(evaluator, context) {
+    let boundEvaluator = evaluator;
+    if (context.inputMapping) {
+        boundEvaluator = boundEvaluator.bindInputMapping(context.inputMapping);
+    }
+    return boundEvaluator;
+}
+//# sourceMappingURL=bindEvaluator.js.map

package/dist/src/utils/bindEvaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bindEvaluator.js","sourceRoot":"","sources":["../../../src/utils/bindEvaluator.ts"],"names":[],"mappings":";;AA2NA,sCASC;AApKD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0JG;AACH,SAAgB,aAAa,CAC3B,SAAoC,EACpC,OAAmC;IAEnC,IAAI,cAAc,GAA8B,SAAS,CAAC;IAC1D,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QACzB,cAAc,GAAG,cAAc,CAAC,gBAAgB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,cAAc,CAAC;AACxB,CAAC"}

package/dist/src/utils/index.d.ts

@@ -0,0 +1,2 @@
+export * from "./bindEvaluator";
+//# sourceMappingURL=index.d.ts.map

package/dist/src/utils/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC"}

package/dist/src/{default_templates → utils}/index.js

@@ -14,6 +14,5 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./HALLUCINATION_TEMPLATE"), exports);
-__exportStar(require("./DOCUMENT_RELEVANCY_TEMPLATE"), exports);
+__exportStar(require("./bindEvaluator"), exports);
 //# sourceMappingURL=index.js.map

package/dist/src/utils/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/utils/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,kDAAgC"}

package/dist/src/utils/objectMappingUtils.d.ts

@@ -0,0 +1,166 @@
+import { ObjectMapping } from "../types/data";
+/**
+ * Remaps an object by applying field mappings while preserving original data.
+ *
+ * This function takes your original data object and a mapping configuration,
+ * then returns a new object that contains:
+ * - All original fields from the input data
+ * - Additional/overridden fields based on the mapping
+ *
+ * The mapping allows you to extract values using:
+ * - Simple property names: `"fieldName"`
+ * - Dot notation: `"user.profile.name"`
+ * - Array access: `"items[0].id"`
+ * - JSONPath expressions: `"$.items[*].id"`
+ * - Function extractors: `(data) => data.customField`
+ *
+ * @param data - The input data object to remap
+ * @param mapping - The mapping configuration defining how to extract/transform values
+ * @returns A new object with original fields plus mapped fields
+ *
+ * @example
+ * **Basic remapping:**
+ * ```typescript
+ * const data = {
+ *   name: "John",
+ *   age: 30,
+ *   email: "john@example.com",
+ * };
+ *
+ * const mapping: ObjectMapping<typeof data> = {
+ *   userName: "name",
+ *   userAge: "age",
+ * };
+ *
+ * const result = remapObject(data, mapping);
+ * // Result: {
+ * //   name: "John",
+ * //   age: 30,
+ * //   email: "john@example.com",
+ * //   userName: "John",      // Added from mapping
+ * //   userAge: 30,            // Added from mapping
+ * // }
+ * ```
+ *
+ * @example
+ * **Nested property extraction:**
+ * ```typescript
+ * const data = {
+ *   user: {
+ *     profile: {
+ *       firstName: "John",
+ *       lastName: "Doe",
+ *     },
+ *     email: "john@example.com",
+ *   },
+ * };
+ *
+ * const mapping: ObjectMapping<typeof data> = {
+ *   firstName: "user.profile.firstName",
+ *   lastName: "user.profile.lastName",
+ *   email: "user.email",
+ * };
+ *
+ * const result = remapObject(data, mapping);
+ * // Result includes original data plus:
+ * // {
+ * //   firstName: "John",
+ * //   lastName: "Doe",
+ * //   email: "john@example.com",
+ * // }
+ * ```
+ *
+ * @example
+ * **Array element access:**
+ * ```typescript
+ * const data = {
+ *   items: [
+ *     { id: 1, name: "Apple" },
+ *     { id: 2, name: "Banana" },
+ *   ],
+ * };
+ *
+ * const mapping: ObjectMapping<typeof data> = {
+ *   firstItemId: "items[0].id",
+ *   firstItemName: "items[0].name",
+ * };
+ *
+ * const result = remapObject(data, mapping);
+ * // Result includes:
+ * // {
+ * //   firstItemId: 1,
+ * //   firstItemName: "Apple",
+ * // }
+ * ```
+ *
+ * @example
+ * **Function-based transformation:**
+ * ```typescript
+ * const data = {
+ *   firstName: "John",
+ *   lastName: "Doe",
+ *   scores: [85, 92, 78],
+ * };
+ *
+ * const mapping: ObjectMapping<typeof data> = {
+ *   fullName: (data) => `${data.firstName} ${data.lastName}`,
+ *   averageScore: (data) =>
+ *     data.scores.reduce((a, b) => a + b, 0) / data.scores.length,
+ * };
+ *
+ * const result = remapObject(data, mapping);
+ * // Result includes:
+ * // {
+ * //   fullName: "John Doe",
+ * //   averageScore: 85,
+ * // }
+ * ```
+ *
+ * @example
+ * **Field override:**
+ * ```typescript
+ * const data = {
+ *   name: "John",
+ *   status: "inactive",
+ * };
+ *
+ * const mapping: ObjectMapping<typeof data> = {
+ *   // Override existing field
+ *   status: (data) => data.name === "John" ? "active" : "inactive",
+ *   // Add new field
+ *   displayName: (data) => `User: ${data.name}`,
+ * };
+ *
+ * const result = remapObject(data, mapping);
+ * // Result:
+ * // {
+ * //   name: "John",
+ * //   status: "active",        // Overridden
+ * //   displayName: "User: John", // Added
+ * // }
+ * ```
+ *
+ * @example
+ * **Real-world evaluator usage:**
+ * ```typescript
+ * // Your data structure
+ * const example = {
+ *   question: "What is AI?",
+ *   context: "AI is artificial intelligence...",
+ *   answer: "AI stands for artificial intelligence",
+ * };
+ *
+ * // Evaluator expects: { input, reference, output }
+ * const mapping: ObjectMapping<typeof example> = {
+ *   input: "question",
+ *   reference: "context",
+ *   output: "answer",
+ * };
+ *
+ * const remapped = remapObject(example, mapping);
+ * // Now remapped has: { question, context, answer, input, reference, output }
+ * // The evaluator can access input, reference, and output fields
+ * ```
+ */
+export declare function remapObject<DataType extends Record<string, unknown>>(data: DataType, mapping: ObjectMapping<DataType>): DataType;
+//# sourceMappingURL=objectMappingUtils.d.ts.map

package/dist/src/utils/objectMappingUtils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"objectMappingUtils.d.ts","sourceRoot":"","sources":["../../../src/utils/objectMappingUtils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAe,MAAM,eAAe,CAAC;AAI3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkKG;AACH,wBAAgB,WAAW,CAAC,QAAQ,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAClE,IAAI,EAAE,QAAQ,EACd,OAAO,EAAE,aAAa,CAAC,QAAQ,CAAC,GAC/B,QAAQ,CAUV"}

package/dist/src/utils/objectMappingUtils.js

@@ -0,0 +1,191 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.remapObject = remapObject;
+const jsonpath_plus_1 = require("jsonpath-plus");
+/**
+ * Remaps an object by applying field mappings while preserving original data.
+ *
+ * This function takes your original data object and a mapping configuration,
+ * then returns a new object that contains:
+ * - All original fields from the input data
+ * - Additional/overridden fields based on the mapping
+ *
+ * The mapping allows you to extract values using:
+ * - Simple property names: `"fieldName"`
+ * - Dot notation: `"user.profile.name"`
+ * - Array access: `"items[0].id"`
+ * - JSONPath expressions: `"$.items[*].id"`
+ * - Function extractors: `(data) => data.customField`
+ *
+ * @param data - The input data object to remap
+ * @param mapping - The mapping configuration defining how to extract/transform values
+ * @returns A new object with original fields plus mapped fields
+ *
+ * @example
+ * **Basic remapping:**
+ * ```typescript
+ * const data = {
+ *   name: "John",
+ *   age: 30,
+ *   email: "john@example.com",
+ * };
+ *
+ * const mapping: ObjectMapping<typeof data> = {
+ *   userName: "name",
+ *   userAge: "age",
+ * };
+ *
+ * const result = remapObject(data, mapping);
+ * // Result: {
+ * //   name: "John",
+ * //   age: 30,
+ * //   email: "john@example.com",
+ * //   userName: "John",      // Added from mapping
+ * //   userAge: 30,            // Added from mapping
+ * // }
+ * ```
+ *
+ * @example
+ * **Nested property extraction:**
+ * ```typescript
+ * const data = {
+ *   user: {
+ *     profile: {
+ *       firstName: "John",
+ *       lastName: "Doe",
+ *     },
+ *     email: "john@example.com",
+ *   },
+ * };
+ *
+ * const mapping: ObjectMapping<typeof data> = {
+ *   firstName: "user.profile.firstName",
+ *   lastName: "user.profile.lastName",
+ *   email: "user.email",
+ * };
+ *
+ * const result = remapObject(data, mapping);
+ * // Result includes original data plus:
+ * // {
+ * //   firstName: "John",
+ * //   lastName: "Doe",
+ * //   email: "john@example.com",
+ * // }
+ * ```
+ *
+ * @example
+ * **Array element access:**
+ * ```typescript
+ * const data = {
+ *   items: [
+ *     { id: 1, name: "Apple" },
+ *     { id: 2, name: "Banana" },
+ *   ],
+ * };
+ *
+ * const mapping: ObjectMapping<typeof data> = {
+ *   firstItemId: "items[0].id",
+ *   firstItemName: "items[0].name",
+ * };
+ *
+ * const result = remapObject(data, mapping);
+ * // Result includes:
+ * // {
+ * //   firstItemId: 1,
+ * //   firstItemName: "Apple",
+ * // }
+ * ```
+ *
+ * @example
+ * **Function-based transformation:**
+ * ```typescript
+ * const data = {
+ *   firstName: "John",
+ *   lastName: "Doe",
+ *   scores: [85, 92, 78],
+ * };
+ *
+ * const mapping: ObjectMapping<typeof data> = {
+ *   fullName: (data) => `${data.firstName} ${data.lastName}`,
+ *   averageScore: (data) =>
+ *     data.scores.reduce((a, b) => a + b, 0) / data.scores.length,
+ * };
+ *
+ * const result = remapObject(data, mapping);
+ * // Result includes:
+ * // {
+ * //   fullName: "John Doe",
+ * //   averageScore: 85,
+ * // }
+ * ```
+ *
+ * @example
+ * **Field override:**
+ * ```typescript
+ * const data = {
+ *   name: "John",
+ *   status: "inactive",
+ * };
+ *
+ * const mapping: ObjectMapping<typeof data> = {
+ *   // Override existing field
+ *   status: (data) => data.name === "John" ? "active" : "inactive",
+ *   // Add new field
+ *   displayName: (data) => `User: ${data.name}`,
+ * };
+ *
+ * const result = remapObject(data, mapping);
+ * // Result:
+ * // {
+ * //   name: "John",
+ * //   status: "active",        // Overridden
+ * //   displayName: "User: John", // Added
+ * // }
+ * ```
+ *
+ * @example
+ * **Real-world evaluator usage:**
+ * ```typescript
+ * // Your data structure
+ * const example = {
+ *   question: "What is AI?",
+ *   context: "AI is artificial intelligence...",
+ *   answer: "AI stands for artificial intelligence",
+ * };
+ *
+ * // Evaluator expects: { input, reference, output }
+ * const mapping: ObjectMapping<typeof example> = {
+ *   input: "question",
+ *   reference: "context",
+ *   output: "answer",
+ * };
+ *
+ * const remapped = remapObject(example, mapping);
+ * // Now remapped has: { question, context, answer, input, reference, output }
+ * // The evaluator can access input, reference, and output fields
+ * ```
+ */
+function remapObject(data, mapping) {
+    return Object.assign(Object.assign({}, data), Object.fromEntries(Object.entries(mapping).map(([key, value]) => [
+        key,
+        getMappedObjectValue(data, value),
+    ])));
+}
+/**
+ * Extracts a value from a data object using a value getter.
+ *
+ * This internal function handles the actual extraction logic, supporting both
+ * string-based paths (including JSONPath) and function-based extractors.
+ *
+ * @param data - The data object to extract from
+ * @param valueGetter - The value getter (string path or function)
+ * @returns The extracted value
+ *
+ * @internal
+ */
+function getMappedObjectValue(data, valueGetter) {
+    return typeof valueGetter === "function"
+        ? valueGetter(data)
+        : (0, jsonpath_plus_1.JSONPath)({ path: valueGetter, json: data, wrap: false });
+}
+//# sourceMappingURL=objectMappingUtils.js.map

package/dist/src/utils/objectMappingUtils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"objectMappingUtils.js","sourceRoot":"","sources":["../../../src/utils/objectMappingUtils.ts"],"names":[],"mappings":";;AAuKA,kCAaC;AAlLD,iDAAyC;AAEzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkKG;AACH,SAAgB,WAAW,CACzB,IAAc,EACd,OAAgC;IAEhC,uCACK,IAAI,GACJ,MAAM,CAAC,WAAW,CACnB,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;QAC5C,GAAG;QACH,oBAAoB,CAAC,IAAI,EAAE,KAAK,CAAC;KAClC,CAAC,CACH,EACD;AACJ,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,oBAAoB,CAC3B,IAAc,EACd,WAAkC;IAElC,OAAO,OAAO,WAAW,KAAK,UAAU;QACtC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC;QACnB,CAAC,CAAC,IAAA,wBAAQ,EAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;AAC/D,CAAC"}