@arizeai/phoenix-evals 0.5.1 → 0.6.0

package/README.md

@@ -111,6 +111,48 @@ console.log(result);
 // Output: { label: "hallucinated", score: 0, explanation: "..." }
 ```
 
+### Data Mapping
+
+When your data structure doesn't match what an evaluator expects, use `bindEvaluator` to map your fields to the evaluator's expected input format:
+
+```typescript
+import {
+  bindEvaluator,
+  createHallucinationEvaluator,
+} from "@arizeai/phoenix-evals";
+import { openai } from "@ai-sdk/openai";
+
+const model = openai("gpt-4o-mini");
+
+type ExampleType = {
+  question: string;
+  context: string;
+  answer: string;
+};
+
+const evaluator = bindEvaluator<ExampleType>(
+  createHallucinationEvaluator({ model }),
+  {
+    inputMapping: {
+      input: "question", // Map "input" from "question"
+      reference: "context", // Map "reference" from "context"
+      output: "answer", // Map "output" from "answer"
+    },
+  }
+);
+
+const result = await evaluator.evaluate({
+  question: "Is Arize Phoenix Open Source?",
+  context:
+    "Arize Phoenix is a platform for building and deploying AI applications. It is open source.",
+  answer: "Arize is not open source.",
+});
+```
+
+Mapping supports simple properties (`"fieldName"`), dot notation (`"user.profile.name"`), array access (`"items[0].id"`), JSONPath expressions (`"$.items[*].id"`), and function extractors (`(data) => data.customField`).
+
+See the complete example in [`examples/bind_evaluator_example.ts`](examples/bind_evaluator_example.ts).
+
 ## Experimentation with Phoenix
 
 This package works seamlessly with [`@arizeai/phoenix-client`](https://www.npmjs.com/package/@arizeai/phoenix-client) to enable experimentation workflows. You can create datasets, run experiments, and trace evaluation calls for analysis and debugging.

package/dist/esm/core/EvaluatorBase.d.ts

@@ -0,0 +1,19 @@
+import { CreateEvaluatorArgs, EvaluationKind, EvaluationResult, EvaluatorInterface, OptimizationDirection, TelemetryConfig } from "../types/index.js";
+import { ObjectMapping } from "../types/data.js";
+/**
+ * Base class for all evaluators
+ */
+export declare abstract class EvaluatorBase<RecordType extends Record<string, unknown>> implements EvaluatorInterface<RecordType> {
+    readonly name: string;
+    readonly kind: EvaluationKind;
+    readonly optimizationDirection?: OptimizationDirection;
+    readonly inputMapping?: ObjectMapping<RecordType>;
+    readonly telemetry?: TelemetryConfig;
+    constructor({ name, kind, optimizationDirection, inputMapping, telemetry, }: CreateEvaluatorArgs<RecordType>);
+    abstract evaluate(_example: RecordType): Promise<EvaluationResult>;
+    /**
+     * Binds the input mapping to the evaluator. It makes a a copy of the evaluator and returns it.
+     */
+    abstract bindInputMapping(inputMapping: ObjectMapping<RecordType>): EvaluatorBase<RecordType>;
+}
+//# sourceMappingURL=EvaluatorBase.d.ts.map

package/dist/esm/core/EvaluatorBase.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"EvaluatorBase.d.ts","sourceRoot":"","sources":["../../../src/core/EvaluatorBase.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,mBAAmB,EACnB,cAAc,EACd,gBAAgB,EAChB,kBAAkB,EAClB,qBAAqB,EACrB,eAAe,EAChB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAE9C;;GAEG;AACH,8BAAsB,aAAa,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC5E,YAAW,kBAAkB,CAAC,UAAU,CAAC;IAEzC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;IAC9B,QAAQ,CAAC,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IACvD,QAAQ,CAAC,YAAY,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IAClD,QAAQ,CAAC,SAAS,CAAC,EAAE,eAAe,CAAC;gBACzB,EACV,IAAI,EACJ,IAAI,EACJ,qBAAqB,EACrB,YAAY,EACZ,SAAS,GACV,EAAE,mBAAmB,CAAC,UAAU,CAAC;IAOlC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAElE;;OAEG;IACH,QAAQ,CAAC,gBAAgB,CACvB,YAAY,EAAE,aAAa,CAAC,UAAU,CAAC,GACtC,aAAa,CAAC,UAAU,CAAC;CAC7B"}

package/dist/esm/core/EvaluatorBase.js

@@ -0,0 +1,18 @@
+/**
+ * Base class for all evaluators
+ */
+export class EvaluatorBase {
+    name;
+    kind;
+    optimizationDirection;
+    inputMapping;
+    telemetry;
+    constructor({ name, kind, optimizationDirection, inputMapping, telemetry, }) {
+        this.name = name;
+        this.kind = kind;
+        this.optimizationDirection = optimizationDirection;
+        this.inputMapping = inputMapping;
+        this.telemetry = telemetry;
+    }
+}
+//# sourceMappingURL=EvaluatorBase.js.map

package/dist/esm/core/EvaluatorBase.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"EvaluatorBase.js","sourceRoot":"","sources":["../../../src/core/EvaluatorBase.ts"],"names":[],"mappings":"AAUA;;GAEG;AACH,MAAM,OAAgB,aAAa;IAGxB,IAAI,CAAS;IACb,IAAI,CAAiB;IACrB,qBAAqB,CAAyB;IAC9C,YAAY,CAA6B;IACzC,SAAS,CAAmB;IACrC,YAAY,EACV,IAAI,EACJ,IAAI,EACJ,qBAAqB,EACrB,YAAY,EACZ,SAAS,GACuB;QAChC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;CASF"}

package/dist/esm/index.d.ts

@@ -1,4 +1,5 @@
 export * from "./llm/index.js";
 export * from "./template/index.js";
 export * from "./types/index.js";
+export * from "./utils/index.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/esm/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,OAAO,CAAC;AACtB,cAAc,YAAY,CAAC;AAC3B,cAAc,SAAS,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,OAAO,CAAC;AACtB,cAAc,YAAY,CAAC;AAC3B,cAAc,SAAS,CAAC;AACxB,cAAc,SAAS,CAAC"}

package/dist/esm/index.js

@@ -1,4 +1,5 @@
 export * from "./llm/index.js";
 export * from "./template/index.js";
 export * from "./types/index.js";
+export * from "./utils/index.js";
 //# sourceMappingURL=index.js.map

package/dist/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,OAAO,CAAC;AACtB,cAAc,YAAY,CAAC;AAC3B,cAAc,SAAS,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,OAAO,CAAC;AACtB,cAAc,YAAY,CAAC;AAC3B,cAAc,SAAS,CAAC;AACxB,cAAc,SAAS,CAAC"}

package/dist/esm/llm/ClassificationEvaluator.d.ts

@@ -1,17 +1,34 @@
-import { CreateClassificationEvaluatorArgs, EvaluatorFn, PromptTemplate, WithPromptTemplate } from "../types/index.js";
+import { ClassificationChoicesMap, CreateClassificationEvaluatorArgs, EvaluatorFn, PromptTemplate, WithPromptTemplate } from "../types/index.js";
+import { ObjectMapping } from "../types/data.js";
 import { LLMEvaluator } from "./LLMEvaluator.js";
+import { LanguageModel } from "ai";
 /**
  * An LLM evaluator that performs evaluation via classification
  */
 export declare class ClassificationEvaluator<RecordType extends Record<string, unknown>> extends LLMEvaluator<RecordType> implements WithPromptTemplate {
     readonly evaluatorFn: EvaluatorFn<RecordType>;
     readonly promptTemplate: PromptTemplate;
+    /**
+     * A dynamically computed set of prompt template variables
+     */
     private _promptTemplateVariables;
-    constructor(args: CreateClassificationEvaluatorArgs);
+    /**
+     * The model to use for classification
+     */
+    readonly model: LanguageModel;
+    /**
+     * The choices to classify the example into
+     */
+    readonly choices: ClassificationChoicesMap;
+    constructor(args: CreateClassificationEvaluatorArgs<RecordType>);
     evaluate: (example: RecordType) => Promise<import("../types/index.js").EvaluationResult>;
     /**
      * List out the prompt template variables needed to perform evaluation
      */
     get promptTemplateVariables(): string[];
+    /**
+     * Binds the input mapping to the evaluator. It makes a a copy of the evaluator and returns it.
+     */
+    bindInputMapping(inputMapping: ObjectMapping<RecordType>): ClassificationEvaluator<RecordType>;
 }
 //# sourceMappingURL=ClassificationEvaluator.d.ts.map

package/dist/esm/llm/ClassificationEvaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,EACL,iCAAiC,EACjC,WAAW,EACX,cAAc,EACd,kBAAkB,EACnB,MAAM,UAAU,CAAC;AAGlB,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C;;GAEG;AACH,qBAAa,uBAAuB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC7E,SAAQ,YAAY,CAAC,UAAU,CAC/B,YAAW,kBAAkB;IAE7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC;IAC9C,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC,OAAO,CAAC,wBAAwB,CAAuB;gBAC3C,IAAI,EAAE,iCAAiC;IAKnD,QAAQ,GAAI,SAAS,UAAU,kDAE7B;IACF;;OAEG;IACH,IAAI,uBAAuB,IAAI,MAAM,EAAE,CAStC;CACF"}
+{"version":3,"file":"ClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,EACL,wBAAwB,EACxB,iCAAiC,EACjC,WAAW,EACX,cAAc,EACd,kBAAkB,EACnB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAI9C,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAEnC;;GAEG;AACH,qBAAa,uBAAuB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC7E,SAAQ,YAAY,CAAC,UAAU,CAC/B,YAAW,kBAAkB;IAE7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC;IAC9C,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC;;OAEG;IACH,OAAO,CAAC,wBAAwB,CAAuB;IACvD;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAC9B;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,wBAAwB,CAAC;gBAE/B,IAAI,EAAE,iCAAiC,CAAC,UAAU,CAAC;IAS/D,QAAQ,GAAI,SAAS,UAAU,kDAM7B;IACF;;OAEG;IACH,IAAI,uBAAuB,IAAI,MAAM,EAAE,CAStC;IACD;;OAEG;IACH,gBAAgB,CACd,YAAY,EAAE,aAAa,CAAC,UAAU,CAAC,GACtC,uBAAuB,CAAC,UAAU,CAAC;CAMvC"}

package/dist/esm/llm/ClassificationEvaluator.js

@@ -1,4 +1,5 @@
 import { getTemplateVariables } from "../template/index.js";
+import { remapObject } from "../utils/objectMappingUtils.js";
 import { createClassifierFn } from "./createClassifierFn.js";
 import { LLMEvaluator } from "./LLMEvaluator.js";
 /**
@@ -7,14 +8,31 @@ import { LLMEvaluator } from "./LLMEvaluator.js";
 export class ClassificationEvaluator extends LLMEvaluator {
     evaluatorFn;
     promptTemplate;
+    /**
+     * A dynamically computed set of prompt template variables
+     */
     _promptTemplateVariables;
+    /**
+     * The model to use for classification
+     */
+    model;
+    /**
+     * The choices to classify the example into
+     */
+    choices;
     constructor(args) {
         super(args);
         this.promptTemplate = args.promptTemplate;
-        this.evaluatorFn = createClassifierFn(args);
+        this.model = args.model;
+        this.choices = args.choices;
+        this.evaluatorFn = createClassifierFn({
+            ...args,
+        });
     }
     evaluate = (example) => {
-        return this.evaluatorFn(example);
+        return this.evaluatorFn(this.inputMapping
+            ? remapObject(example, this.inputMapping)
+            : example);
     };
     /**
      * List out the prompt template variables needed to perform evaluation
@@ -29,5 +47,14 @@ export class ClassificationEvaluator extends LLMEvaluator {
         // Give a copy of the variables
         return [...this._promptTemplateVariables];
     }
+    /**
+     * Binds the input mapping to the evaluator. It makes a a copy of the evaluator and returns it.
+     */
+    bindInputMapping(inputMapping) {
+        return new ClassificationEvaluator({
+            ...this,
+            inputMapping,
+        });
+    }
 }
 //# sourceMappingURL=ClassificationEvaluator.js.map

package/dist/esm/llm/ClassificationEvaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"ClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAQnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C;;GAEG;AACH,MAAM,OAAO,uBACX,SAAQ,YAAwB;IAGvB,WAAW,CAA0B;IACrC,cAAc,CAAiB;IAChC,wBAAwB,CAAuB;IACvD,YAAY,IAAuC;QACjD,KAAK,CAAC,IAAI,CAAC,CAAC;QACZ,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,kBAAkB,CAAa,IAAI,CAAC,CAAC;IAC1D,CAAC;IACD,QAAQ,GAAG,CAAC,OAAmB,EAAE,EAAE;QACjC,OAAO,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACnC,CAAC,CAAC;IACF;;OAEG;IACH,IAAI,uBAAuB;QACzB,0DAA0D;QAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,wBAAwB,GAAG,oBAAoB,CAAC;gBACnD,QAAQ,EAAE,IAAI,CAAC,cAAc;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,+BAA+B;QAC/B,OAAO,CAAC,GAAG,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC5C,CAAC;CACF"}
+{"version":3,"file":"ClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AASnD,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAE1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAI9C;;GAEG;AACH,MAAM,OAAO,uBACX,SAAQ,YAAwB;IAGvB,WAAW,CAA0B;IACrC,cAAc,CAAiB;IACxC;;OAEG;IACK,wBAAwB,CAAuB;IACvD;;OAEG;IACM,KAAK,CAAgB;IAC9B;;OAEG;IACM,OAAO,CAA2B;IAE3C,YAAY,IAAmD;QAC7D,KAAK,CAAC,IAAI,CAAC,CAAC;QACZ,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,WAAW,GAAG,kBAAkB,CAAa;YAChD,GAAG,IAAI;SACR,CAAC,CAAC;IACL,CAAC;IACD,QAAQ,GAAG,CAAC,OAAmB,EAAE,EAAE;QACjC,OAAO,IAAI,CAAC,WAAW,CACrB,IAAI,CAAC,YAAY;YACf,CAAC,CAAC,WAAW,CAAa,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC;YACrD,CAAC,CAAC,OAAO,CACZ,CAAC;IACJ,CAAC,CAAC;IACF;;OAEG;IACH,IAAI,uBAAuB;QACzB,0DAA0D;QAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,wBAAwB,GAAG,oBAAoB,CAAC;gBACnD,QAAQ,EAAE,IAAI,CAAC,cAAc;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,+BAA+B;QAC/B,OAAO,CAAC,GAAG,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC5C,CAAC;IACD;;OAEG;IACH,gBAAgB,CACd,YAAuC;QAEvC,OAAO,IAAI,uBAAuB,CAAC;YACjC,GAAG,IAAI;YACP,YAAY;SACb,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/esm/llm/LLMEvaluator.d.ts

@@ -1,12 +1,9 @@
-import { CreateEvaluatorArgs, EvaluationResult, Evaluator, OptimizationDirection } from "../types/index.js";
+import { EvaluatorBase } from "../core/EvaluatorBase.js";
+import { CreateLLMEvaluatorArgs } from "../types/index.js";
 /**
  * Base class for llm evaluation metrics / scores
  */
-export declare abstract class LLMEvaluator<RecordType extends Record<string, unknown>> implements Evaluator<RecordType> {
-    readonly name: string;
-    readonly kind: "LLM";
-    readonly optimizationDirection?: OptimizationDirection;
-    constructor({ name, optimizationDirection }: CreateEvaluatorArgs);
-    abstract evaluate(_example: RecordType): Promise<EvaluationResult>;
+export declare abstract class LLMEvaluator<RecordType extends Record<string, unknown>> extends EvaluatorBase<RecordType> {
+    constructor({ ...args }: CreateLLMEvaluatorArgs<RecordType>);
 }
 //# sourceMappingURL=LLMEvaluator.d.ts.map

package/dist/esm/llm/LLMEvaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"LLMEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/LLMEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,mBAAmB,EACnB,gBAAgB,EAChB,SAAS,EACT,qBAAqB,EACtB,MAAM,UAAU,CAAC;AAElB;;GAEG;AACH,8BAAsB,YAAY,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC3E,YAAW,SAAS,CAAC,UAAU,CAAC;IAEhC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAG,KAAK,CAAU;IAC/B,QAAQ,CAAC,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;gBAC3C,EAAE,IAAI,EAAE,qBAAqB,EAAE,EAAE,mBAAmB;IAIhE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;CACnE"}
+{"version":3,"file":"LLMEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/LLMEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AAElD;;GAEG;AACH,8BAAsB,YAAY,CAChC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC1C,SAAQ,aAAa,CAAC,UAAU,CAAC;gBACrB,EAAE,GAAG,IAAI,EAAE,EAAE,sBAAsB,CAAC,UAAU,CAAC;CAG5D"}

package/dist/esm/llm/LLMEvaluator.js

@@ -1,13 +1,10 @@
+import { EvaluatorBase } from "../core/EvaluatorBase.js";
 /**
  * Base class for llm evaluation metrics / scores
  */
-export class LLMEvaluator {
-    name;
-    kind = "LLM";
-    optimizationDirection;
-    constructor({ name, optimizationDirection }) {
-        this.name = name;
-        this.optimizationDirection = optimizationDirection;
+export class LLMEvaluator extends EvaluatorBase {
+    constructor({ ...args }) {
+        super({ kind: "LLM", ...args });
     }
 }
 //# sourceMappingURL=LLMEvaluator.js.map

package/dist/esm/llm/LLMEvaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"LLMEvaluator.js","sourceRoot":"","sources":["../../../src/llm/LLMEvaluator.ts"],"names":[],"mappings":"AAOA;;GAEG;AACH,MAAM,OAAgB,YAAY;IAGvB,IAAI,CAAS;IACb,IAAI,GAAG,KAAc,CAAC;IACtB,qBAAqB,CAAyB;IACvD,YAAY,EAAE,IAAI,EAAE,qBAAqB,EAAuB;QAC9D,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;IACrD,CAAC;CAEF"}
+{"version":3,"file":"LLMEvaluator.js","sourceRoot":"","sources":["../../../src/llm/LLMEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAGtD;;GAEG;AACH,MAAM,OAAgB,YAEpB,SAAQ,aAAyB;IACjC,YAAY,EAAE,GAAG,IAAI,EAAsC;QACzD,KAAK,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;CACF"}

package/dist/esm/llm/createClassificationEvaluator.d.ts

@@ -1,4 +1,4 @@
 import { CreateClassificationEvaluatorArgs } from "../types/evals.js";
 import { ClassificationEvaluator } from "./ClassificationEvaluator.js";
-export declare function createClassificationEvaluator<RecordType extends Record<string, unknown>>(args: CreateClassificationEvaluatorArgs): ClassificationEvaluator<RecordType>;
+export declare function createClassificationEvaluator<RecordType extends Record<string, unknown>>(args: CreateClassificationEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType>;
 //# sourceMappingURL=createClassificationEvaluator.d.ts.map

package/dist/esm/llm/createClassificationEvaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,wBAAgB,6BAA6B,CAC3C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAE1C,IAAI,EAAE,iCAAiC,GACtC,uBAAuB,CAAC,UAAU,CAAC,CAErC"}
+{"version":3,"file":"createClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,wBAAgB,6BAA6B,CAC3C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAE1C,IAAI,EAAE,iCAAiC,CAAC,UAAU,CAAC,GAClD,uBAAuB,CAAC,UAAU,CAAC,CAErC"}

package/dist/esm/llm/createClassificationEvaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"createClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,MAAM,UAAU,6BAA6B,CAG3C,IAAuC;IAEvC,OAAO,IAAI,uBAAuB,CAAa,IAAI,CAAC,CAAC;AACvD,CAAC"}
+{"version":3,"file":"createClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,MAAM,UAAU,6BAA6B,CAG3C,IAAmD;IAEnD,OAAO,IAAI,uBAAuB,CAAa,IAAI,CAAC,CAAC;AACvD,CAAC"}

package/dist/esm/llm/createDocumentRelevancyEvaluator.d.ts

@@ -1,9 +1,10 @@
-import { CreateClassificationEvaluatorArgs, Evaluator } from "../types/evals.js";
-export interface DocumentRelevancyEvaluatorArgs extends Omit<CreateClassificationEvaluatorArgs, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
-    optimizationDirection?: CreateClassificationEvaluatorArgs["optimizationDirection"];
-    name?: CreateClassificationEvaluatorArgs["name"];
-    choices?: CreateClassificationEvaluatorArgs["choices"];
-    promptTemplate?: CreateClassificationEvaluatorArgs["promptTemplate"];
+import { CreateClassificationEvaluatorArgs } from "../types/evals.js";
+import { ClassificationEvaluator } from "./ClassificationEvaluator.js";
+export interface DocumentRelevancyEvaluatorArgs<RecordType extends Record<string, unknown> = DocumentRelevancyEvaluationRecord> extends Omit<CreateClassificationEvaluatorArgs<RecordType>, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
+    optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
+    name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
+    choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
+    promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
 }
 /**
  * A record to be evaluated by the document relevancy evaluator.
@@ -39,5 +40,5 @@ export interface DocumentRelevancyEvaluationRecord {
  * console.log(result.label); // "relevant" or "unrelated"
  * ```
  */
-export declare function createDocumentRelevancyEvaluator<RecordType extends Record<string, unknown> = DocumentRelevancyEvaluationRecord>(args: DocumentRelevancyEvaluatorArgs): Evaluator<RecordType>;
+export declare function createDocumentRelevancyEvaluator<RecordType extends Record<string, unknown> = DocumentRelevancyEvaluationRecord>(args: DocumentRelevancyEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType>;
 //# sourceMappingURL=createDocumentRelevancyEvaluator.d.ts.map

package/dist/esm/llm/createDocumentRelevancyEvaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createDocumentRelevancyEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iCAAiC,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAI9E,MAAM,WAAW,8BACf,SAAQ,IAAI,CACV,iCAAiC,EACjC,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,uBAAuB,CAAC,CAAC;IACnF,IAAI,CAAC,EAAE,iCAAiC,CAAC,MAAM,CAAC,CAAC;IACjD,OAAO,CAAC,EAAE,iCAAiC,CAAC,SAAS,CAAC,CAAC;IACvD,cAAc,CAAC,EAAE,iCAAiC,CAAC,gBAAgB,CAAC,CAAC;CACtE;AAED;;GAEG;AACH,MAAM,WAAW,iCAAiC;IAChD,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,gCAAgC,CAC9C,UAAU,SAAS,MAAM,CACvB,MAAM,EACN,OAAO,CACR,GAAG,iCAAiC,EACrC,IAAI,EAAE,8BAA8B,GAAG,SAAS,CAAC,UAAU,CAAC,CAiB7D"}
+{"version":3,"file":"createDocumentRelevancyEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGpE,MAAM,WAAW,8BAA8B,CAC7C,UAAU,SAAS,MAAM,CACvB,MAAM,EACN,OAAO,CACR,GAAG,iCAAiC,CACrC,SAAQ,IAAI,CACV,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED;;GAEG;AACH,MAAM,WAAW,iCAAiC;IAChD,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,gCAAgC,CAC9C,UAAU,SAAS,MAAM,CACvB,MAAM,EACN,OAAO,CACR,GAAG,iCAAiC,EAErC,IAAI,EAAE,8BAA8B,CAAC,UAAU,CAAC,GAC/C,uBAAuB,CAAC,UAAU,CAAC,CAerC"}

package/dist/esm/llm/createDocumentRelevancyEvaluator.js

@@ -29,12 +29,11 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
 export function createDocumentRelevancyEvaluator(args) {
     const { choices = DOCUMENT_RELEVANCY_CHOICES, promptTemplate = DOCUMENT_RELEVANCY_TEMPLATE, optimizationDirection = "MAXIMIZE", name = "document_relevancy", ...rest } = args;
     return createClassificationEvaluator({
-        ...args,
+        ...rest,
         promptTemplate,
         choices,
         optimizationDirection,
         name,
-        ...rest,
     });
 }
 //# sourceMappingURL=createDocumentRelevancyEvaluator.js.map

package/dist/esm/llm/createDocumentRelevancyEvaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"createDocumentRelevancyEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,0BAA0B,EAC1B,2BAA2B,GAC5B,MAAM,kDAAkD,CAAC;AAG1D,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAsBhF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,UAAU,gCAAgC,CAK9C,IAAoC;IACpC,MAAM,EACJ,OAAO,GAAG,0BAA0B,EACpC,cAAc,GAAG,2BAA2B,EAC5C,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,oBAAoB,EAC3B,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IAET,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;QACJ,GAAG,IAAI;KACR,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"createDocumentRelevancyEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,0BAA0B,EAC1B,2BAA2B,GAC5B,MAAM,kDAAkD,CAAC;AAI1D,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AA0BhF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,UAAU,gCAAgC,CAM9C,IAAgD;IAEhD,MAAM,EACJ,OAAO,GAAG,0BAA0B,EACpC,cAAc,GAAG,2BAA2B,EAC5C,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,oBAAoB,EAC3B,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}

package/dist/esm/llm/createHallucinationEvaluator.d.ts

@@ -1,10 +1,10 @@
 import { CreateClassificationEvaluatorArgs } from "../types/evals.js";
 import { ClassificationEvaluator } from "./ClassificationEvaluator.js";
-export interface HallucinationEvaluatorArgs extends Omit<CreateClassificationEvaluatorArgs, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
-    optimizationDirection?: CreateClassificationEvaluatorArgs["optimizationDirection"];
-    name?: CreateClassificationEvaluatorArgs["name"];
-    choices?: CreateClassificationEvaluatorArgs["choices"];
-    promptTemplate?: CreateClassificationEvaluatorArgs["promptTemplate"];
+export interface HallucinationEvaluatorArgs<RecordType extends Record<string, unknown> = HallucinationEvaluationRecord> extends Omit<CreateClassificationEvaluatorArgs<RecordType>, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
+    optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
+    name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
+    choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
+    promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
 }
 /**
  * A record to be evaluated by the hallucination evaluator.
@@ -21,5 +21,5 @@ export type HallucinationEvaluationRecord = {
  * @param args - The arguments for creating the hallucination evaluator.
  * @returns A function that evaluates whether an answer is factual or hallucinated based on a query and reference text.
  */
-export declare function createHallucinationEvaluator<RecordType extends Record<string, unknown> = HallucinationEvaluationRecord>(args: HallucinationEvaluatorArgs): ClassificationEvaluator<RecordType>;
+export declare function createHallucinationEvaluator<RecordType extends Record<string, unknown> = HallucinationEvaluationRecord>(args: HallucinationEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType>;
 //# sourceMappingURL=createHallucinationEvaluator.d.ts.map

package/dist/esm/llm/createHallucinationEvaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createHallucinationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGpE,MAAM,WAAW,0BACf,SAAQ,IAAI,CACV,iCAAiC,EACjC,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,uBAAuB,CAAC,CAAC;IACnF,IAAI,CAAC,EAAE,iCAAiC,CAAC,MAAM,CAAC,CAAC;IACjD,OAAO,CAAC,EAAE,iCAAiC,CAAC,SAAS,CAAC,CAAC;IACvD,cAAc,CAAC,EAAE,iCAAiC,CAAC,gBAAgB,CAAC,CAAC;CACtE;AAED;;GAEG;AACH,MAAM,MAAM,6BAA6B,GAAG;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AACF;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAC1C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,6BAA6B,EAC1E,IAAI,EAAE,0BAA0B,GAAG,uBAAuB,CAAC,UAAU,CAAC,CAgBvE"}
+{"version":3,"file":"createHallucinationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGpE,MAAM,WAAW,0BAA0B,CACzC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,6BAA6B,CAC1E,SAAQ,IAAI,CACV,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED;;GAEG;AACH,MAAM,MAAM,6BAA6B,GAAG;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AACF;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAC1C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,6BAA6B,EAE1E,IAAI,EAAE,0BAA0B,CAAC,UAAU,CAAC,GAC3C,uBAAuB,CAAC,UAAU,CAAC,CAerC"}

package/dist/esm/llm/createHallucinationEvaluator.js

@@ -9,12 +9,11 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
 export function createHallucinationEvaluator(args) {
     const { choices = HALLUCINATION_CHOICES, promptTemplate = HALLUCINATION_TEMPLATE, optimizationDirection = "MINIMIZE", name = "hallucination", ...rest } = args;
     return createClassificationEvaluator({
-        ...args,
+        ...rest,
         promptTemplate,
         choices,
         optimizationDirection,
         name,
-        ...rest,
     });
 }
 //# sourceMappingURL=createHallucinationEvaluator.js.map

package/dist/esm/llm/createHallucinationEvaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"createHallucinationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,6CAA6C,CAAC;AAIrD,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAsBhF;;;;;GAKG;AACH,MAAM,UAAU,4BAA4B,CAE1C,IAAgC;IAChC,MAAM,EACJ,OAAO,GAAG,qBAAqB,EAC/B,cAAc,GAAG,sBAAsB,EACvC,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,eAAe,EACtB,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;QACJ,GAAG,IAAI;KACR,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"createHallucinationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,6CAA6C,CAAC;AAIrD,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAuBhF;;;;;GAKG;AACH,MAAM,UAAU,4BAA4B,CAG1C,IAA4C;IAE5C,MAAM,EACJ,OAAO,GAAG,qBAAqB,EAC/B,cAAc,GAAG,sBAAsB,EACvC,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,eAAe,EACtB,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}