@arizeai/phoenix-evals 0.5.0 → 0.6.0

package/dist/esm/types/data.d.ts

@@ -0,0 +1,194 @@
+/**
+ * A value extractor that can retrieve data from an object using various methods.
+ *
+ * This type supports multiple ways to extract values from your data structure:
+ * - **String paths**: Simple property names, dot notation, or JSONPath expressions
+ * - **Function extractors**: Custom transformation functions
+ *
+ * @example
+ * **Simple property access:**
+ * ```typescript
+ * const getter: ValueGetter<{ name: string }> = "name";
+ * ```
+ *
+ * @example
+ * **Dot notation for nested properties:**
+ * ```typescript
+ * const getter: ValueGetter<{ user: { profile: { name: string } } }> = "user.profile.name";
+ * ```
+ *
+ * @example
+ * **Array element access:**
+ * ```typescript
+ * const getter: ValueGetter<{ items: string[] }> = "items[0]";
+ * ```
+ *
+ * @example
+ * **JSONPath expression:**
+ * ```typescript
+ * const getter: ValueGetter<{ items: Array<{ id: number }> }> = "$.items[*].id";
+ * ```
+ *
+ * @example
+ * **Function-based extraction:**
+ * ```typescript
+ * const getter: ValueGetter<{ firstName: string; lastName: string }> =
+ *   (data) => `${data.firstName} ${data.lastName}`;
+ * ```
+ *
+ * @example
+ * **Complex transformation:**
+ * ```typescript
+ * const getter: ValueGetter<{ scores: number[] }> =
+ *   (data) => data.scores.reduce((a, b) => a + b, 0) / data.scores.length;
+ * ```
+ *
+ * @typeParam DataType - The type of the data object to extract values from
+ */
+export type ValueGetter<DataType extends Record<string, unknown>> = string | ((data: DataType) => any);
+/**
+ * A mapping configuration that transforms data from one structure to another.
+ *
+ * This type defines how to map fields from your data structure to the fields
+ * expected by an evaluator or other component. The mapping is flexible and
+ * supports multiple extraction methods.
+ *
+ * **Key Features:**
+ * - Preserves original data fields
+ * - Adds/overrides fields with mapped values
+ * - Supports nested property access
+ * - Supports array element access
+ * - Supports JSONPath expressions for complex queries
+ * - Supports function-based transformations
+ *
+ * @example
+ * **Basic field mapping:**
+ * ```typescript
+ * type MyData = {
+ *   userQuery: string;
+ *   context: string;
+ *   response: string;
+ * };
+ *
+ * const mapping: ObjectMapping<MyData> = {
+ *   input: "userQuery",      // Map "input" to "userQuery"
+ *   reference: "context",    // Map "reference" to "context"
+ *   output: "response",       // Map "output" to "response"
+ * };
+ * ```
+ *
+ * @example
+ * **Nested property mapping:**
+ * ```typescript
+ * type ApiData = {
+ *   request: {
+ *     body: {
+ *       query: string;
+ *       context: string;
+ *     };
+ *   };
+ *   response: {
+ *     data: {
+ *       text: string;
+ *     };
+ *   };
+ * };
+ *
+ * const mapping: ObjectMapping<ApiData> = {
+ *   input: "request.body.query",
+ *   reference: "request.body.context",
+ *   output: "response.data.text",
+ * };
+ * ```
+ *
+ * @example
+ * **Array element access:**
+ * ```typescript
+ * type DataWithArrays = {
+ *   messages: Array<{ role: string; content: string }>;
+ *   sources: string[];
+ * };
+ *
+ * const mapping: ObjectMapping<DataWithArrays> = {
+ *   firstMessage: "messages[0].content",
+ *   lastSource: "sources[-1]",  // Last element
+ *   allRoles: "$.messages[*].role",  // JSONPath for all roles
+ * };
+ * ```
+ *
+ * @example
+ * **Function-based transformations:**
+ * ```typescript
+ * type RawData = {
+ *   firstName: string;
+ *   lastName: string;
+ *   contexts: string[];
+ *   scores: number[];
+ * };
+ *
+ * const mapping: ObjectMapping<RawData> = {
+ *   // Combine fields
+ *   fullName: (data) => `${data.firstName} ${data.lastName}`,
+ *   // Transform array to string
+ *   contextText: (data) => data.contexts.join("\n\n"),
+ *   // Calculate derived value
+ *   averageScore: (data) =>
+ *     data.scores.reduce((a, b) => a + b, 0) / data.scores.length,
+ *   // Conditional logic
+ *   status: (data) => data.scores.length > 0 ? "active" : "inactive",
+ * };
+ * ```
+ *
+ * @example
+ * **Mixed mapping types:**
+ * ```typescript
+ * type ComplexData = {
+ *   user: {
+ *     name: string;
+ *     email: string;
+ *   };
+ *   items: Array<{ id: number; name: string }>;
+ *   metadata: {
+ *     tags: string[];
+ *   };
+ * };
+ *
+ * const mapping: ObjectMapping<ComplexData> = {
+ *   // Simple dot notation
+ *   userName: "user.name",
+ *   // Array access
+ *   firstItemId: "items[0].id",
+ *   // JSONPath for complex query
+ *   allItemNames: "$.items[*].name",
+ *   // Function for transformation
+ *   formattedTags: (data) => data.metadata.tags.map(t => `#${t}`).join(" "),
+ * };
+ * ```
+ *
+ * @example
+ * **Real-world evaluator binding:**
+ * ```typescript
+ * import { bindEvaluator, createHallucinationEvaluator } from "@arizeai/phoenix-evals";
+ *
+ * type QAData = {
+ *   question: string;
+ *   context: string;
+ *   answer: string;
+ * };
+ *
+ * const mapping: ObjectMapping<QAData> = {
+ *   input: "question",      // Evaluator expects "input"
+ *   reference: "context",   // Evaluator expects "reference"
+ *   output: "answer",       // Evaluator expects "output"
+ * };
+ *
+ * const evaluator = bindEvaluator(
+ *   createHallucinationEvaluator({ model: openai("gpt-4") }),
+ *   { inputMapping: mapping }
+ * );
+ * ```
+ *
+ * @typeParam DataType - The type of the data object being mapped
+ */
+export type ObjectMapping<DataType extends Record<string, unknown>> = Record<string, ValueGetter<DataType>>;
+//# sourceMappingURL=data.d.ts.map

package/dist/esm/types/data.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../src/types/data.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,MAAM,MAAM,WAAW,CAAC,QAAQ,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAC5D,MAAM,GAEN,CAAC,CAAC,IAAI,EAAE,QAAQ,KAAK,GAAG,CAAC,CAAC;AAE9B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+IG;AACH,MAAM,MAAM,aAAa,CAAC,QAAQ,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,MAAM,CAC1E,MAAM,EACN,WAAW,CAAC,QAAQ,CAAC,CACtB,CAAC"}

package/dist/esm/types/data.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=data.js.map

package/dist/esm/types/data.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"data.js","sourceRoot":"","sources":["../../../src/types/data.ts"],"names":[],"mappings":""}

package/dist/esm/types/evals.d.ts

@@ -1,5 +1,6 @@
+import { ObjectMapping } from "./data.js";
 import { WithTelemetry } from "./otel.js";
-import { Template } from "./templating.js";
+import { PromptTemplate } from "./templating.js";
 import { LanguageModel } from "ai";
 /**
  * A specific AI example that is under evaluation
@@ -66,21 +67,34 @@ export interface CreateClassifierArgs extends WithTelemetry {
     /**
      * The prompt template to use for classification
      */
-    promptTemplate: Template;
+    promptTemplate: PromptTemplate;
 }
-export interface CreateEvaluatorArgs {
+export interface CreateEvaluatorArgs<ExampleType extends Record<string, unknown> = Record<string, unknown>> extends WithTelemetry {
     /**
      * The name of the metric that the evaluator produces
      * E.x. "correctness"
      */
     name: string;
+    /**
+     * The kind of the evaluation. Also known as the "kind" of evaluator.
+     */
+    kind: EvaluationKind;
     /**
      * If present, represents the direction in which you want the metric to be optimized
      * E.x. "MAXIMIZE" means you want the number to be higher.
      */
     optimizationDirection?: OptimizationDirection;
+    /**
+     * The mapping of the input to evaluate to the shape that the evaluator expects
+     */
+    inputMapping?: ObjectMapping<ExampleType>;
 }
-export interface CreateClassificationEvaluatorArgs extends CreateClassifierArgs, CreateEvaluatorArgs {
+export type CreateLLMEvaluatorArgs<RecordType extends Record<string, unknown>> = Omit<CreateEvaluatorArgs<RecordType>, "kind">;
+export interface CreateClassificationEvaluatorArgs<RecordType extends Record<string, unknown>> extends CreateClassifierArgs, CreateLLMEvaluatorArgs<RecordType> {
+    /**
+     * The prompt template to use for classification
+     */
+    promptTemplate: PromptTemplate;
 }
 export type EvaluatorFn<ExampleType extends Record<string, unknown>> = (args: ExampleType) => Promise<EvaluationResult>;
 /**
@@ -114,7 +128,7 @@ interface EvaluatorDescription {
  * The Base Evaluator interface
  * This is the interface that all evaluators must implement
  */
-export interface Evaluator<ExampleType extends Record<string, unknown>> extends EvaluatorDescription {
+export interface EvaluatorInterface<ExampleType extends Record<string, unknown>> extends EvaluatorDescription {
     /**
      * The function that evaluates the example
      */

package/dist/esm/types/evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAExC,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAEnC;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,aAAa,CAAC;CACtB;AAGD,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,QAAQ,CAAC;CAC1B;AAED,MAAM,WAAW,mBAAmB;IAClC;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED,MAAM,WAAW,iCACf,SAAQ,oBAAoB,EAC1B,mBAAmB;CAAG;AAE1B,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,MAAM,CAAC;AAE5C;;;GAGG;AACH,MAAM,MAAM,qBAAqB,GAAG,UAAU,GAAG,UAAU,CAAC;AAE5D;;GAEG;AACH,UAAU,oBAAoB;IAC5B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CACpE,SAAQ,oBAAoB;IAC5B;;OAEG;IACH,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;CACpC"}
+{"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAEnC;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,aAAa,CAAC;CACtB;AAGD,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,WAAW,mBAAmB,CAClC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CACrE,SAAQ,aAAa;IACrB;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;OAEG;IACH,YAAY,CAAC,EAAE,aAAa,CAAC,WAAW,CAAC,CAAC;CAC3C;AAED,MAAM,MAAM,sBAAsB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAC3E,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,iCAAiC,CAChD,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC1C,SAAQ,oBAAoB,EAC1B,sBAAsB,CAAC,UAAU,CAAC;IACpC;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,MAAM,CAAC;AAE5C;;;GAGG;AACH,MAAM,MAAM,qBAAqB,GAAG,UAAU,GAAG,UAAU,CAAC;AAE5D;;GAEG;AACH,UAAU,oBAAoB;IAC5B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC7E,SAAQ,oBAAoB;IAC5B;;OAEG;IACH,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;CACpC"}

package/dist/esm/types/index.d.ts

@@ -1,4 +1,5 @@
 export * from "./evals.js";
 export * from "./templating.js";
 export * from "./otel.js";
+export * from "./data.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/esm/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/types/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,cAAc,CAAC;AAC7B,cAAc,QAAQ,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/types/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,cAAc,CAAC;AAC7B,cAAc,QAAQ,CAAC;AACvB,cAAc,QAAQ,CAAC"}

package/dist/esm/types/index.js

@@ -1,4 +1,5 @@
 export * from "./evals.js";
 export * from "./templating.js";
 export * from "./otel.js";
+export * from "./data.js";
 //# sourceMappingURL=index.js.map

package/dist/esm/types/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/types/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,cAAc,CAAC;AAC7B,cAAc,QAAQ,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/types/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,cAAc,CAAC;AAC7B,cAAc,QAAQ,CAAC;AACvB,cAAc,QAAQ,CAAC"}

package/dist/esm/types/otel.d.ts

@@ -1,17 +1,18 @@
 import { Tracer } from "@opentelemetry/api";
+export type TelemetryConfig = {
+    /**
+     * Whether OpenTelemetry is enabled on the call.
+     * Defaults to true for visibility into the evals calls.
+     * @default true
+     */
+    isEnabled?: boolean;
+    /**
+     * The tracer to use for the call.
+     * If not provided, the traces will get picked up by the global tracer.
+     */
+    tracer?: Tracer;
+};
 export type WithTelemetry = {
-    telemetry?: {
-        /**
-         * Whether OpenTelemetry is enabled on the call.
-         * Defaults to true for visibility into the evals calls.
-         * @default true
-         */
-        isEnabled?: boolean;
-        /**
-         * The tracer to use for the call.
-         * If not provided, the traces will get picked up by the global tracer.
-         */
-        tracer?: Tracer;
-    };
+    telemetry?: TelemetryConfig;
 };
 //# sourceMappingURL=otel.d.ts.map

package/dist/esm/types/otel.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"otel.d.ts","sourceRoot":"","sources":["../../../src/types/otel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,CAAC,EAAE;QACV;;;;WAIG;QACH,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB;;;WAGG;QACH,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACH,CAAC"}
+{"version":3,"file":"otel.d.ts","sourceRoot":"","sources":["../../../src/types/otel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,MAAM,MAAM,eAAe,GAAG;IAC5B;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B,CAAC"}

package/dist/esm/types/templating.d.ts

@@ -1,10 +1,12 @@
-export type Template = string;
+import type { ModelMessage } from "ai";
+export type PromptTemplate = string | Array<ModelMessage>;
+export type RenderedPrompt = string | Array<ModelMessage>;
 export type TemplateVariables = Record<string, unknown>;
-/**
- * A class or object that has a prompt template
- */
 export interface WithPromptTemplate {
-    readonly promptTemplate: Template;
-    get promptTemplateVariables(): string[];
+    readonly promptTemplate: PromptTemplate;
+    /**
+     * List out the prompt template variables needed to perform evaluation
+     */
+    readonly promptTemplateVariables: string[];
 }
 //# sourceMappingURL=templating.d.ts.map

package/dist/esm/types/templating.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"templating.d.ts","sourceRoot":"","sources":["../../../src/types/templating.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,QAAQ,GAAG,MAAM,CAAC;AAC9B,MAAM,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC;IAClC,IAAI,uBAAuB,IAAI,MAAM,EAAE,CAAC;CACzC"}
+{"version":3,"file":"templating.d.ts","sourceRoot":"","sources":["../../../src/types/templating.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAEvC,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;AAC1D,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;AAC1D,MAAM,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAExD,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC;;OAEG;IACH,QAAQ,CAAC,uBAAuB,EAAE,MAAM,EAAE,CAAC;CAC5C"}

package/dist/esm/utils/bindEvaluator.d.ts

@@ -0,0 +1,219 @@
+import { EvaluatorBase } from "../core/EvaluatorBase.js";
+import { ObjectMapping } from "../types/data.js";
+/**
+ * Context for binding an evaluator with input mapping configuration.
+ *
+ * This type defines the structure for binding an evaluator to a specific data shape
+ * by mapping the evaluator's expected input fields to the actual data structure.
+ *
+ * @example
+ * ```typescript
+ * // Map evaluator fields to your data structure
+ * const context: BindingContext<MyDataType> = {
+ *   inputMapping: {
+ *     input: "userQuery",        // Maps "input" to "userQuery" field
+ *     reference: "context",      // Maps "reference" to "context" field
+ *     output: "modelResponse",   // Maps "output" to "modelResponse" field
+ *   },
+ * };
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Using nested property access
+ * const context: BindingContext<ApiResponse> = {
+ *   inputMapping: {
+ *     input: "request.body.query",
+ *     reference: "request.body.context",
+ *     output: "response.data.text",
+ *   },
+ * };
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Using function-based mapping for transformations
+ * const context: BindingContext<RawData> = {
+ *   inputMapping: {
+ *     input: "question",
+ *     reference: (data) => data.context.join("\n"),  // Transform array to string
+ *     output: "answer",
+ *   },
+ * };
+ * ```
+ *
+ * @typeParam RecordType - The type of the data record that will be evaluated
+ */
+export type BindingContext<RecordType extends Record<string, unknown>> = {
+    /**
+     * Mapping of evaluator input fields to data source fields.
+     *
+     * The keys represent the field names expected by the evaluator (e.g., "input", "output", "reference"),
+     * and the values specify how to extract those fields from your data structure.
+     *
+     * Supports:
+     * - Simple property names: `"fieldName"`
+     * - Dot notation: `"user.profile.name"`
+     * - Array access: `"items[0].id"`
+     * - JSONPath expressions: `"$.items[*].id"`
+     * - Function extractors: `(data) => data.customField.toUpperCase()`
+     */
+    inputMapping: ObjectMapping<RecordType>;
+};
+/**
+ * Binds an evaluator to a specific data structure using input mapping.
+ *
+ * This function creates a new evaluator instance that automatically transforms
+ * your data structure to match what the evaluator expects. This is particularly
+ * useful when your data schema doesn't match the evaluator's expected input format.
+ *
+ * @param evaluator - The evaluator to bind (e.g., a hallucination evaluator)
+ * @param context - The binding context containing the input mapping configuration
+ * @returns A new evaluator instance with the input mapping applied
+ *
+ * @example
+ * **Basic usage with simple field mapping:**
+ * ```typescript
+ * import { bindEvaluator, createHallucinationEvaluator } from "@arizeai/phoenix-evals";
+ * import { openai } from "@ai-sdk/openai";
+ *
+ * type MyData = {
+ *   question: string;
+ *   context: string;
+ *   answer: string;
+ * };
+ *
+ * const evaluator = bindEvaluator<MyData>(
+ *   createHallucinationEvaluator({ model: openai("gpt-4") }),
+ *   {
+ *     inputMapping: {
+ *       input: "question",      // Evaluator expects "input", map from "question"
+ *       reference: "context",   // Evaluator expects "reference", map from "context"
+ *       output: "answer",        // Evaluator expects "output", map from "answer"
+ *     },
+ *   }
+ * );
+ *
+ * // Now you can evaluate with your data structure
+ * const result = await evaluator.evaluate({
+ *   question: "What is AI?",
+ *   context: "AI is artificial intelligence...",
+ *   answer: "AI stands for artificial intelligence",
+ * });
+ * ```
+ *
+ * @example
+ * **Using nested property access:**
+ * ```typescript
+ * type ApiResponse = {
+ *   request: {
+ *     body: {
+ *       query: string;
+ *       context: string;
+ *     };
+ *   };
+ *   response: {
+ *     data: {
+ *       text: string;
+ *     };
+ *   };
+ * };
+ *
+ * const evaluator = bindEvaluator<ApiResponse>(
+ *   createHallucinationEvaluator({ model: openai("gpt-4") }),
+ *   {
+ *     inputMapping: {
+ *       input: "request.body.query",
+ *       reference: "request.body.context",
+ *       output: "response.data.text",
+ *     },
+ *   }
+ * );
+ * ```
+ *
+ * @example
+ * **Using function-based mapping for data transformation:**
+ * ```typescript
+ * type RawData = {
+ *   question: string;
+ *   contexts: string[];  // Array of context strings
+ *   answer: string;
+ * };
+ *
+ * const evaluator = bindEvaluator<RawData>(
+ *   createHallucinationEvaluator({ model: openai("gpt-4") }),
+ *   {
+ *     inputMapping: {
+ *       input: "question",
+ *       // Transform array to single string
+ *       reference: (data) => data.contexts.join("\n\n"),
+ *       output: "answer",
+ *     },
+ *   }
+ * );
+ * ```
+ *
+ * @example
+ * **Using JSONPath for complex queries:**
+ * ```typescript
+ * type ComplexData = {
+ *   conversation: {
+ *     messages: Array<{ role: string; content: string }>;
+ *   };
+ *   metadata: {
+ *     sources: string[];
+ *   };
+ * };
+ *
+ * const evaluator = bindEvaluator<ComplexData>(
+ *   createHallucinationEvaluator({ model: openai("gpt-4") }),
+ *   {
+ *     inputMapping: {
+ *       // Extract last user message
+ *       input: "$.conversation.messages[?(@.role=='user')].content[-1]",
+ *       // Extract all sources
+ *       reference: "$.metadata.sources[*]",
+ *       // Extract last assistant message
+ *       output: "$.conversation.messages[?(@.role=='assistant')].content[-1]",
+ *     },
+ *   }
+ * );
+ * ```
+ *
+ * @example
+ * **Binding multiple evaluators with different mappings:**
+ * ```typescript
+ * type EvaluationData = {
+ *   userQuery: string;
+ *   systemContext: string;
+ *   modelOutput: string;
+ *   expectedOutput?: string;
+ * };
+ *
+ * // Hallucination evaluator
+ * const hallucinationEvaluator = bindEvaluator<EvaluationData>(
+ *   createHallucinationEvaluator({ model: openai("gpt-4") }),
+ *   {
+ *     inputMapping: {
+ *       input: "userQuery",
+ *       reference: "systemContext",
+ *       output: "modelOutput",
+ *     },
+ *   }
+ * );
+ *
+ * // Document relevancy evaluator (if it exists)
+ * const relevancyEvaluator = bindEvaluator<EvaluationData>(
+ *   createDocumentRelevancyEvaluator({ model: openai("gpt-4") }),
+ *   {
+ *     inputMapping: {
+ *       query: "userQuery",
+ *       document: "systemContext",
+ *       output: "modelOutput",
+ *     },
+ *   }
+ * );
+ * ```
+ */
+export declare function bindEvaluator<RecordType extends Record<string, unknown>>(evaluator: EvaluatorBase<RecordType>, context: BindingContext<RecordType>): EvaluatorBase<RecordType>;
+//# sourceMappingURL=bindEvaluator.d.ts.map

package/dist/esm/utils/bindEvaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bindEvaluator.d.ts","sourceRoot":"","sources":["../../../src/utils/bindEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAE9C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,MAAM,MAAM,cAAc,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI;IACvE;;;;;;;;;;;;OAYG;IACH,YAAY,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;CACzC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0JG;AACH,wBAAgB,aAAa,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACtE,SAAS,EAAE,aAAa,CAAC,UAAU,CAAC,EACpC,OAAO,EAAE,cAAc,CAAC,UAAU,CAAC,GAClC,aAAa,CAAC,UAAU,CAAC,CAM3B"}