@arizeai/phoenix-evals 0.5.0 → 0.6.0

package/dist/src/llm/ClassificationEvaluator.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ClassificationEvaluator = void 0;
 const template_1 = require("../template");
+const objectMappingUtils_1 = require("../utils/objectMappingUtils");
 const createClassifierFn_1 = require("./createClassifierFn");
 const LLMEvaluator_1 = require("./LLMEvaluator");
 /**
@@ -11,10 +12,14 @@ class ClassificationEvaluator extends LLMEvaluator_1.LLMEvaluator {
     constructor(args) {
         super(args);
         this.evaluate = (example) => {
-            return this.evaluatorFn(example);
+            return this.evaluatorFn(this.inputMapping
+                ? (0, objectMappingUtils_1.remapObject)(example, this.inputMapping)
+                : example);
         };
         this.promptTemplate = args.promptTemplate;
-        this.evaluatorFn = (0, createClassifierFn_1.createClassifierFn)(args);
+        this.model = args.model;
+        this.choices = args.choices;
+        this.evaluatorFn = (0, createClassifierFn_1.createClassifierFn)(Object.assign({}, args));
     }
     /**
      * List out the prompt template variables needed to perform evaluation
@@ -29,6 +34,12 @@ class ClassificationEvaluator extends LLMEvaluator_1.LLMEvaluator {
         // Give a copy of the variables
         return [...this._promptTemplateVariables];
     }
+    /**
+     * Binds the input mapping to the evaluator. It makes a a copy of the evaluator and returns it.
+     */
+    bindInputMapping(inputMapping) {
+        return new ClassificationEvaluator(Object.assign(Object.assign({}, this), { inputMapping }));
+    }
 }
 exports.ClassificationEvaluator = ClassificationEvaluator;
 //# sourceMappingURL=ClassificationEvaluator.js.map

package/dist/src/llm/ClassificationEvaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"ClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":";;;AAAA,0CAAmD;AAQnD,6DAA0D;AAC1D,iDAA8C;AAE9C;;GAEG;AACH,MAAa,uBACX,SAAQ,2BAAwB;IAMhC,YAAY,IAAuC;QACjD,KAAK,CAAC,IAAI,CAAC,CAAC;QAId,aAAQ,GAAG,CAAC,OAAmB,EAAE,EAAE;YACjC,OAAO,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QACnC,CAAC,CAAC;QALA,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,IAAA,uCAAkB,EAAa,IAAI,CAAC,CAAC;IAC1D,CAAC;IAID;;OAEG;IACH,IAAI,uBAAuB;QACzB,0DAA0D;QAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,wBAAwB,GAAG,IAAA,+BAAoB,EAAC;gBACnD,QAAQ,EAAE,IAAI,CAAC,cAAc;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,+BAA+B;QAC/B,OAAO,CAAC,GAAG,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC5C,CAAC;CACF;AA5BD,0DA4BC"}
+{"version":3,"file":"ClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":";;;AAAA,0CAAmD;AASnD,oEAA0D;AAE1D,6DAA0D;AAC1D,iDAA8C;AAI9C;;GAEG;AACH,MAAa,uBACX,SAAQ,2BAAwB;IAkBhC,YAAY,IAAmD;QAC7D,KAAK,CAAC,IAAI,CAAC,CAAC;QAQd,aAAQ,GAAG,CAAC,OAAmB,EAAE,EAAE;YACjC,OAAO,IAAI,CAAC,WAAW,CACrB,IAAI,CAAC,YAAY;gBACf,CAAC,CAAC,IAAA,gCAAW,EAAa,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC;gBACrD,CAAC,CAAC,OAAO,CACZ,CAAC;QACJ,CAAC,CAAC;QAbA,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,WAAW,GAAG,IAAA,uCAAkB,oBAChC,IAAI,EACP,CAAC;IACL,CAAC;IAQD;;OAEG;IACH,IAAI,uBAAuB;QACzB,0DAA0D;QAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,wBAAwB,GAAG,IAAA,+BAAoB,EAAC;gBACnD,QAAQ,EAAE,IAAI,CAAC,cAAc;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,+BAA+B;QAC/B,OAAO,CAAC,GAAG,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC5C,CAAC;IACD;;OAEG;IACH,gBAAgB,CACd,YAAuC;QAEvC,OAAO,IAAI,uBAAuB,iCAC7B,IAAI,KACP,YAAY,IACZ,CAAC;IACL,CAAC;CACF;AA3DD,0DA2DC"}

package/dist/src/llm/LLMEvaluator.d.ts

@@ -1,12 +1,9 @@
-import { CreateEvaluatorArgs, EvaluationResult, Evaluator, OptimizationDirection } from "../types";
+import { EvaluatorBase } from "../core/EvaluatorBase";
+import { CreateLLMEvaluatorArgs } from "../types";
 /**
  * Base class for llm evaluation metrics / scores
  */
-export declare abstract class LLMEvaluator<RecordType extends Record<string, unknown>> implements Evaluator<RecordType> {
-    readonly name: string;
-    readonly kind: "LLM";
-    readonly optimizationDirection?: OptimizationDirection;
-    constructor({ name, optimizationDirection }: CreateEvaluatorArgs);
-    abstract evaluate(_example: RecordType): Promise<EvaluationResult>;
+export declare abstract class LLMEvaluator<RecordType extends Record<string, unknown>> extends EvaluatorBase<RecordType> {
+    constructor({ ...args }: CreateLLMEvaluatorArgs<RecordType>);
 }
 //# sourceMappingURL=LLMEvaluator.d.ts.map

package/dist/src/llm/LLMEvaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"LLMEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/LLMEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,mBAAmB,EACnB,gBAAgB,EAChB,SAAS,EACT,qBAAqB,EACtB,MAAM,UAAU,CAAC;AAElB;;GAEG;AACH,8BAAsB,YAAY,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC3E,YAAW,SAAS,CAAC,UAAU,CAAC;IAEhC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAG,KAAK,CAAU;IAC/B,QAAQ,CAAC,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;gBAC3C,EAAE,IAAI,EAAE,qBAAqB,EAAE,EAAE,mBAAmB;IAIhE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;CACnE"}
+{"version":3,"file":"LLMEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/LLMEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AAElD;;GAEG;AACH,8BAAsB,YAAY,CAChC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC1C,SAAQ,aAAa,CAAC,UAAU,CAAC;gBACrB,EAAE,GAAG,IAAI,EAAE,EAAE,sBAAsB,CAAC,UAAU,CAAC;CAG5D"}

package/dist/src/llm/LLMEvaluator.js

@@ -1,14 +1,25 @@
 "use strict";
+var __rest = (this && this.__rest) || function (s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
+            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
+                t[p[i]] = s[p[i]];
+        }
+    return t;
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LLMEvaluator = void 0;
+const EvaluatorBase_1 = require("../core/EvaluatorBase");
 /**
  * Base class for llm evaluation metrics / scores
  */
-class LLMEvaluator {
-    constructor({ name, optimizationDirection }) {
-        this.kind = "LLM";
-        this.name = name;
-        this.optimizationDirection = optimizationDirection;
+class LLMEvaluator extends EvaluatorBase_1.EvaluatorBase {
+    constructor(_a) {
+        var args = __rest(_a, []);
+        super(Object.assign({ kind: "LLM" }, args));
     }
 }
 exports.LLMEvaluator = LLMEvaluator;

package/dist/src/llm/LLMEvaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"LLMEvaluator.js","sourceRoot":"","sources":["../../../src/llm/LLMEvaluator.ts"],"names":[],"mappings":";;;AAOA;;GAEG;AACH,MAAsB,YAAY;IAMhC,YAAY,EAAE,IAAI,EAAE,qBAAqB,EAAuB;QAFvD,SAAI,GAAG,KAAc,CAAC;QAG7B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;IACrD,CAAC;CAEF;AAXD,oCAWC"}
+{"version":3,"file":"LLMEvaluator.js","sourceRoot":"","sources":["../../../src/llm/LLMEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,yDAAsD;AAGtD;;GAEG;AACH,MAAsB,YAEpB,SAAQ,6BAAyB;IACjC,YAAY,EAA+C;YAA1C,IAAI,cAAT,EAAW,CAAF;QACnB,KAAK,iBAAG,IAAI,EAAE,KAAK,IAAK,IAAI,EAAG,CAAC;IAClC,CAAC;CACF;AAND,oCAMC"}

package/dist/src/llm/createClassificationEvaluator.d.ts

@@ -1,4 +1,4 @@
 import { CreateClassificationEvaluatorArgs } from "../types/evals";
 import { ClassificationEvaluator } from "./ClassificationEvaluator";
-export declare function createClassificationEvaluator<RecordType extends Record<string, unknown>>(args: CreateClassificationEvaluatorArgs): ClassificationEvaluator<RecordType>;
+export declare function createClassificationEvaluator<RecordType extends Record<string, unknown>>(args: CreateClassificationEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType>;
 //# sourceMappingURL=createClassificationEvaluator.d.ts.map

package/dist/src/llm/createClassificationEvaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,wBAAgB,6BAA6B,CAC3C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAE1C,IAAI,EAAE,iCAAiC,GACtC,uBAAuB,CAAC,UAAU,CAAC,CAErC"}
+{"version":3,"file":"createClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,wBAAgB,6BAA6B,CAC3C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAE1C,IAAI,EAAE,iCAAiC,CAAC,UAAU,CAAC,GAClD,uBAAuB,CAAC,UAAU,CAAC,CAErC"}

package/dist/src/llm/createClassificationEvaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"createClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":";;AAIA,sEAMC;AARD,uEAAoE;AAEpE,SAAgB,6BAA6B,CAG3C,IAAuC;IAEvC,OAAO,IAAI,iDAAuB,CAAa,IAAI,CAAC,CAAC;AACvD,CAAC"}
+{"version":3,"file":"createClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":";;AAIA,sEAMC;AARD,uEAAoE;AAEpE,SAAgB,6BAA6B,CAG3C,IAAmD;IAEnD,OAAO,IAAI,iDAAuB,CAAa,IAAI,CAAC,CAAC;AACvD,CAAC"}

package/dist/src/llm/createDocumentRelevancyEvaluator.d.ts

@@ -1,9 +1,10 @@
-import { CreateClassificationEvaluatorArgs, Evaluator } from "../types/evals";
-export interface DocumentRelevancyEvaluatorArgs extends Omit<CreateClassificationEvaluatorArgs, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
-    optimizationDirection?: CreateClassificationEvaluatorArgs["optimizationDirection"];
-    name?: CreateClassificationEvaluatorArgs["name"];
-    choices?: CreateClassificationEvaluatorArgs["choices"];
-    promptTemplate?: CreateClassificationEvaluatorArgs["promptTemplate"];
+import { CreateClassificationEvaluatorArgs } from "../types/evals";
+import { ClassificationEvaluator } from "./ClassificationEvaluator";
+export interface DocumentRelevancyEvaluatorArgs<RecordType extends Record<string, unknown> = DocumentRelevancyEvaluationRecord> extends Omit<CreateClassificationEvaluatorArgs<RecordType>, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
+    optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
+    name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
+    choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
+    promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
 }
 /**
  * A record to be evaluated by the document relevancy evaluator.
@@ -39,5 +40,5 @@ export interface DocumentRelevancyEvaluationRecord {
  * console.log(result.label); // "relevant" or "unrelated"
  * ```
  */
-export declare function createDocumentRelevancyEvaluator<RecordType extends Record<string, unknown> = DocumentRelevancyEvaluationRecord>(args: DocumentRelevancyEvaluatorArgs): Evaluator<RecordType>;
+export declare function createDocumentRelevancyEvaluator<RecordType extends Record<string, unknown> = DocumentRelevancyEvaluationRecord>(args: DocumentRelevancyEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType>;
 //# sourceMappingURL=createDocumentRelevancyEvaluator.d.ts.map

package/dist/src/llm/createDocumentRelevancyEvaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createDocumentRelevancyEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iCAAiC,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAI9E,MAAM,WAAW,8BACf,SAAQ,IAAI,CACV,iCAAiC,EACjC,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,uBAAuB,CAAC,CAAC;IACnF,IAAI,CAAC,EAAE,iCAAiC,CAAC,MAAM,CAAC,CAAC;IACjD,OAAO,CAAC,EAAE,iCAAiC,CAAC,SAAS,CAAC,CAAC;IACvD,cAAc,CAAC,EAAE,iCAAiC,CAAC,gBAAgB,CAAC,CAAC;CACtE;AAED;;GAEG;AACH,MAAM,WAAW,iCAAiC;IAChD,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,gCAAgC,CAC9C,UAAU,SAAS,MAAM,CACvB,MAAM,EACN,OAAO,CACR,GAAG,iCAAiC,EACrC,IAAI,EAAE,8BAA8B,GAAG,SAAS,CAAC,UAAU,CAAC,CAiB7D"}
+{"version":3,"file":"createDocumentRelevancyEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGpE,MAAM,WAAW,8BAA8B,CAC7C,UAAU,SAAS,MAAM,CACvB,MAAM,EACN,OAAO,CACR,GAAG,iCAAiC,CACrC,SAAQ,IAAI,CACV,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED;;GAEG;AACH,MAAM,WAAW,iCAAiC;IAChD,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,gCAAgC,CAC9C,UAAU,SAAS,MAAM,CACvB,MAAM,EACN,OAAO,CACR,GAAG,iCAAiC,EAErC,IAAI,EAAE,8BAA8B,CAAC,UAAU,CAAC,GAC/C,uBAAuB,CAAC,UAAU,CAAC,CAerC"}

package/dist/src/llm/createDocumentRelevancyEvaluator.js

@@ -42,9 +42,9 @@ const createClassificationEvaluator_1 = require("./createClassificationEvaluator
  */
 function createDocumentRelevancyEvaluator(args) {
     const { choices = DOCUMENT_RELEVANCY_TEMPLATE_1.DOCUMENT_RELEVANCY_CHOICES, promptTemplate = DOCUMENT_RELEVANCY_TEMPLATE_1.DOCUMENT_RELEVANCY_TEMPLATE, optimizationDirection = "MAXIMIZE", name = "document_relevancy" } = args, rest = __rest(args, ["choices", "promptTemplate", "optimizationDirection", "name"]);
-    return (0, createClassificationEvaluator_1.createClassificationEvaluator)(Object.assign(Object.assign(Object.assign({}, args), { promptTemplate,
+    return (0, createClassificationEvaluator_1.createClassificationEvaluator)(Object.assign(Object.assign({}, rest), { promptTemplate,
         choices,
         optimizationDirection,
-        name }), rest));
+        name }));
 }
 //# sourceMappingURL=createDocumentRelevancyEvaluator.js.map

package/dist/src/llm/createDocumentRelevancyEvaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"createDocumentRelevancyEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAsDA,4EAsBC;AA5ED,kGAG0D;AAG1D,mFAAgF;AAsBhF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAgB,gCAAgC,CAK9C,IAAoC;IACpC,MAAM,EACJ,OAAO,GAAG,wDAA0B,EACpC,cAAc,GAAG,yDAA2B,EAC5C,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,oBAAoB,KAEzB,IAAI,EADH,IAAI,UACL,IAAI,EANF,8DAML,CAAO,CAAC;IAET,OAAO,IAAA,6DAA6B,gDAC/B,IAAI,KACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI,KACD,IAAI,EACP,CAAC;AACL,CAAC"}
+{"version":3,"file":"createDocumentRelevancyEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;AA2DA,4EAsBC;AAjFD,kGAG0D;AAI1D,mFAAgF;AA0BhF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAgB,gCAAgC,CAM9C,IAAgD;IAEhD,MAAM,EACJ,OAAO,GAAG,wDAA0B,EACpC,cAAc,GAAG,yDAA2B,EAC5C,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,oBAAoB,KAEzB,IAAI,EADH,IAAI,UACL,IAAI,EANF,8DAML,CAAO,CAAC;IACT,OAAO,IAAA,6DAA6B,kCAC/B,IAAI,KACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI,IACJ,CAAC;AACL,CAAC"}

package/dist/src/llm/createHallucinationEvaluator.d.ts

@@ -1,10 +1,10 @@
 import { CreateClassificationEvaluatorArgs } from "../types/evals";
 import { ClassificationEvaluator } from "./ClassificationEvaluator";
-export interface HallucinationEvaluatorArgs extends Omit<CreateClassificationEvaluatorArgs, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
-    optimizationDirection?: CreateClassificationEvaluatorArgs["optimizationDirection"];
-    name?: CreateClassificationEvaluatorArgs["name"];
-    choices?: CreateClassificationEvaluatorArgs["choices"];
-    promptTemplate?: CreateClassificationEvaluatorArgs["promptTemplate"];
+export interface HallucinationEvaluatorArgs<RecordType extends Record<string, unknown> = HallucinationEvaluationRecord> extends Omit<CreateClassificationEvaluatorArgs<RecordType>, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
+    optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
+    name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
+    choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
+    promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
 }
 /**
  * A record to be evaluated by the hallucination evaluator.
@@ -21,5 +21,5 @@ export type HallucinationEvaluationRecord = {
  * @param args - The arguments for creating the hallucination evaluator.
  * @returns A function that evaluates whether an answer is factual or hallucinated based on a query and reference text.
  */
-export declare function createHallucinationEvaluator<RecordType extends Record<string, unknown> = HallucinationEvaluationRecord>(args: HallucinationEvaluatorArgs): ClassificationEvaluator<RecordType>;
+export declare function createHallucinationEvaluator<RecordType extends Record<string, unknown> = HallucinationEvaluationRecord>(args: HallucinationEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType>;
 //# sourceMappingURL=createHallucinationEvaluator.d.ts.map

package/dist/src/llm/createHallucinationEvaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createHallucinationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGpE,MAAM,WAAW,0BACf,SAAQ,IAAI,CACV,iCAAiC,EACjC,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,uBAAuB,CAAC,CAAC;IACnF,IAAI,CAAC,EAAE,iCAAiC,CAAC,MAAM,CAAC,CAAC;IACjD,OAAO,CAAC,EAAE,iCAAiC,CAAC,SAAS,CAAC,CAAC;IACvD,cAAc,CAAC,EAAE,iCAAiC,CAAC,gBAAgB,CAAC,CAAC;CACtE;AAED;;GAEG;AACH,MAAM,MAAM,6BAA6B,GAAG;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AACF;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAC1C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,6BAA6B,EAC1E,IAAI,EAAE,0BAA0B,GAAG,uBAAuB,CAAC,UAAU,CAAC,CAgBvE"}
+{"version":3,"file":"createHallucinationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGpE,MAAM,WAAW,0BAA0B,CACzC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,6BAA6B,CAC1E,SAAQ,IAAI,CACV,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED;;GAEG;AACH,MAAM,MAAM,6BAA6B,GAAG;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AACF;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAC1C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,6BAA6B,EAE1E,IAAI,EAAE,0BAA0B,CAAC,UAAU,CAAC,GAC3C,uBAAuB,CAAC,UAAU,CAAC,CAerC"}

package/dist/src/llm/createHallucinationEvaluator.js

@@ -22,9 +22,9 @@ const createClassificationEvaluator_1 = require("./createClassificationEvaluator
  */
 function createHallucinationEvaluator(args) {
     const { choices = HALLUCINATION_TEMPLATE_1.HALLUCINATION_CHOICES, promptTemplate = HALLUCINATION_TEMPLATE_1.HALLUCINATION_TEMPLATE, optimizationDirection = "MINIMIZE", name = "hallucination" } = args, rest = __rest(args, ["choices", "promptTemplate", "optimizationDirection", "name"]);
-    return (0, createClassificationEvaluator_1.createClassificationEvaluator)(Object.assign(Object.assign(Object.assign({}, args), { promptTemplate,
+    return (0, createClassificationEvaluator_1.createClassificationEvaluator)(Object.assign(Object.assign({}, rest), { promptTemplate,
         choices,
         optimizationDirection,
-        name }), rest));
+        name }));
 }
 //# sourceMappingURL=createHallucinationEvaluator.js.map

package/dist/src/llm/createHallucinationEvaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"createHallucinationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAmCA,oEAkBC;AArDD,wFAGqD;AAIrD,mFAAgF;AAsBhF;;;;;GAKG;AACH,SAAgB,4BAA4B,CAE1C,IAAgC;IAChC,MAAM,EACJ,OAAO,GAAG,8CAAqB,EAC/B,cAAc,GAAG,+CAAsB,EACvC,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,eAAe,KAEpB,IAAI,EADH,IAAI,UACL,IAAI,EANF,8DAML,CAAO,CAAC;IACT,OAAO,IAAA,6DAA6B,gDAC/B,IAAI,KACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI,KACD,IAAI,EACP,CAAC;AACL,CAAC"}
+{"version":3,"file":"createHallucinationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAoCA,oEAmBC;AAvDD,wFAGqD;AAIrD,mFAAgF;AAuBhF;;;;;GAKG;AACH,SAAgB,4BAA4B,CAG1C,IAA4C;IAE5C,MAAM,EACJ,OAAO,GAAG,8CAAqB,EAC/B,cAAc,GAAG,+CAAsB,EACvC,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,eAAe,KAEpB,IAAI,EADH,IAAI,UACL,IAAI,EANF,8DAML,CAAO,CAAC;IACT,OAAO,IAAA,6DAA6B,kCAC/B,IAAI,KACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI,IACJ,CAAC;AACL,CAAC"}

package/dist/src/template/applyTemplate.d.ts

@@ -1,10 +1,10 @@
-import { Template } from "../types/templating";
+import { PromptTemplate, RenderedPrompt } from "../types/templating";
 /**
  * A function that applies a set of variables to a template (e.g. a prompt)
  * Uses the Mustache library to apply the variables to the template
  */
 export declare function formatTemplate(args: {
-    template: Template;
+    template: PromptTemplate;
     variables: Record<string, unknown>;
-}): string;
+}): RenderedPrompt;
 //# sourceMappingURL=applyTemplate.d.ts.map

package/dist/src/template/applyTemplate.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"applyTemplate.d.ts","sourceRoot":"","sources":["../../../src/template/applyTemplate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAM/C;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE;IACnC,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,UAUA"}
+{"version":3,"file":"applyTemplate.d.ts","sourceRoot":"","sources":["../../../src/template/applyTemplate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAMrE;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE;IACnC,QAAQ,EAAE,cAAc,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,cAAc,CAejB"}

package/dist/src/template/applyTemplate.js

@@ -13,7 +13,18 @@ const mustache_1 = __importDefault(require("mustache"));
 function formatTemplate(args) {
     const { template, variables } = args;
     const variablesProxy = (0, createTemplateVariablesProxy_1.createTemplateVariablesProxy)(variables);
+    if (typeof template === "string") {
+        return renderTemplateString(template, variablesProxy);
+    }
+    return template.map((message) => {
+        if (message.content !== undefined && typeof message.content === "string") {
+            return Object.assign(Object.assign({}, message), { content: renderTemplateString(message.content, variablesProxy) });
+        }
+        return message;
+    });
+}
+function renderTemplateString(template, variables) {
     // Disable HTML escaping by providing a custom escape function that returns text as-is
-    return mustache_1.default.render(template, variablesProxy, {}, { escape: (text) => text });
+    return mustache_1.default.render(template, variables, {}, { escape: (text) => text });
 }
 //# sourceMappingURL=applyTemplate.js.map

package/dist/src/template/applyTemplate.js.map

@@ -1 +1 @@
-{"version":3,"file":"applyTemplate.js","sourceRoot":"","sources":["../../../src/template/applyTemplate.ts"],"names":[],"mappings":";;;;;AAUA,wCAaC;AArBD,iFAA8E;AAE9E,wDAAgC;AAEhC;;;GAGG;AACH,SAAgB,cAAc,CAAC,IAG9B;IACC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IACrC,MAAM,cAAc,GAAG,IAAA,2DAA4B,EAAC,SAAS,CAAC,CAAC;IAC/D,sFAAsF;IACtF,OAAO,kBAAQ,CAAC,MAAM,CACpB,QAAQ,EACR,cAAc,EACd,EAAE,EACF,EAAE,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,CAC3B,CAAC;AACJ,CAAC"}
+{"version":3,"file":"applyTemplate.js","sourceRoot":"","sources":["../../../src/template/applyTemplate.ts"],"names":[],"mappings":";;;;;AAUA,wCAkBC;AA1BD,iFAA8E;AAE9E,wDAAgC;AAEhC;;;GAGG;AACH,SAAgB,cAAc,CAAC,IAG9B;IACC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IACrC,MAAM,cAAc,GAAG,IAAA,2DAA4B,EAAC,SAAS,CAAC,CAAC;IAC/D,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACjC,OAAO,oBAAoB,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QAC9B,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACzE,uCACK,OAAO,KACV,OAAO,EAAE,oBAAoB,CAAC,OAAO,CAAC,OAAO,EAAE,cAAc,CAAC,IAC9D;QACJ,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC,CAAmB,CAAC;AACvB,CAAC;AAED,SAAS,oBAAoB,CAC3B,QAAgB,EAChB,SAAkC;IAElC,sFAAsF;IACtF,OAAO,kBAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;AAC9E,CAAC"}

package/dist/src/template/createTemplateVariablesProxy.d.ts

@@ -3,5 +3,5 @@
  * This allows Mustache to access properties of objects (e.g., {{user.name}}) while
  * stringifying objects that are accessed as leaf values (e.g., {{user.profile}}).
  */
-export declare function createTemplateVariablesProxy(obj: unknown): unknown;
+export declare function createTemplateVariablesProxy<T>(obj: T): T;
 //# sourceMappingURL=createTemplateVariablesProxy.d.ts.map

package/dist/src/template/createTemplateVariablesProxy.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createTemplateVariablesProxy.d.ts","sourceRoot":"","sources":["../../../src/template/createTemplateVariablesProxy.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,wBAAgB,4BAA4B,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CA+DlE"}
+{"version":3,"file":"createTemplateVariablesProxy.d.ts","sourceRoot":"","sources":["../../../src/template/createTemplateVariablesProxy.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,wBAAgB,4BAA4B,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CA+DzD"}

package/dist/src/template/createTemplateVariablesProxy.js.map

@@ -1 +1 @@
-{"version":3,"file":"createTemplateVariablesProxy.js","sourceRoot":"","sources":["../../../src/template/createTemplateVariablesProxy.ts"],"names":[],"mappings":";;AAKA,oEA+DC;AApED;;;;GAIG;AACH,SAAgB,4BAA4B,CAAC,GAAY;IACvD,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtC,OAAO,GAAG,CAAC;IACb,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,GAAG,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC5B,OAAO,IAAI,KAAK,CAAC,GAA8B,EAAE;YAC/C,GAAG,CAAC,MAAM,EAAE,IAAqB;gBAC/B,6EAA6E;gBAC7E,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;oBACxB,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACtC,CAAC;gBACD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;oBACvB,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACtC,CAAC;gBAED,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAC7B,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;gBACnC,CAAC;gBAED,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;gBAExC,sEAAsE;gBACtE,iFAAiF;gBACjF,IACE,KAAK,KAAK,IAAI;oBACd,OAAO,KAAK,KAAK,QAAQ;oBACzB,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EACrB,CAAC;oBACD,OAAO,4BAA4B,CAAC,KAAK,CAAC,CAAC;gBAC7C,CAAC;gBAED,OAAO,KAAK,CAAC;YACf,CAAC;YACD,kGAAkG;YAClG,8EAA8E;YAC9E,GAAG,CAAC,MAAM,EAAE,IAAI;gBACd,IAAI,IAAI,KAAK,UAAU,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC9C,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YACnC,CAAC;YACD,OAAO,CAAC,MAAM;gBACZ,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACjC,CAAC;YACD,wBAAwB,CAAC,MAAM,EAAE,IAAI;gBACnC,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;oBACxB,OAAO;wBACL,UAAU,EAAE,KAAK;wBACjB,YAAY,EAAE,IAAI;wBAClB,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;qBACpC,CAAC;gBACJ,CAAC;gBACD,OAAO,OAAO,CAAC,wBAAwB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YACxD,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC"}
+{"version":3,"file":"createTemplateVariablesProxy.js","sourceRoot":"","sources":["../../../src/template/createTemplateVariablesProxy.ts"],"names":[],"mappings":";;AAKA,oEA+DC;AApED;;;;GAIG;AACH,SAAgB,4BAA4B,CAAI,GAAM;IACpD,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtC,OAAO,GAAG,CAAC;IACb,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,GAAG,CAAC,GAAG,CAAC,4BAA4B,CAAM,CAAC;IACpD,CAAC;IAED,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC5B,OAAO,IAAI,KAAK,CAAC,GAAG,EAAE;YACpB,GAAG,CAAC,MAAM,EAAE,IAAqB;gBAC/B,6EAA6E;gBAC7E,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;oBACxB,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACtC,CAAC;gBACD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;oBACvB,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACtC,CAAC;gBAED,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAC7B,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;gBACnC,CAAC;gBAED,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;gBAExC,sEAAsE;gBACtE,iFAAiF;gBACjF,IACE,KAAK,KAAK,IAAI;oBACd,OAAO,KAAK,KAAK,QAAQ;oBACzB,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EACrB,CAAC;oBACD,OAAO,4BAA4B,CAAC,KAAK,CAAC,CAAC;gBAC7C,CAAC;gBAED,OAAO,KAAK,CAAC;YACf,CAAC;YACD,kGAAkG;YAClG,8EAA8E;YAC9E,GAAG,CAAC,MAAM,EAAE,IAAI;gBACd,IAAI,IAAI,KAAK,UAAU,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC9C,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YACnC,CAAC;YACD,OAAO,CAAC,MAAM;gBACZ,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACjC,CAAC;YACD,wBAAwB,CAAC,MAAM,EAAE,IAAI;gBACnC,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;oBACxB,OAAO;wBACL,UAAU,EAAE,KAAK;wBACjB,YAAY,EAAE,IAAI;wBAClB,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;qBACpC,CAAC;gBACJ,CAAC;gBACD,OAAO,OAAO,CAAC,wBAAwB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YACxD,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/src/template/getTemplateVariables.d.ts

@@ -1,6 +1,6 @@
-import { Template } from "../types/templating";
+import { PromptTemplate } from "../types/templating";
 type GetTemplateVariableArgs = {
-    template: Template;
+    template: PromptTemplate;
 };
 /**
  * Parse out the template variables of a prompt

package/dist/src/template/getTemplateVariables.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"getTemplateVariables.d.ts","sourceRoot":"","sources":["../../../src/template/getTemplateVariables.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAI/C,KAAK,uBAAuB,GAAG;IAC7B,QAAQ,EAAE,QAAQ,CAAC;CACpB,CAAC;AACF;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,uBAAuB,GAAG,MAAM,EAAE,CAU5E"}
+{"version":3,"file":"getTemplateVariables.d.ts","sourceRoot":"","sources":["../../../src/template/getTemplateVariables.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAIrD,KAAK,uBAAuB,GAAG;IAC7B,QAAQ,EAAE,cAAc,CAAC;CAC1B,CAAC;AACF;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,uBAAuB,GAAG,MAAM,EAAE,CAW5E"}

package/dist/src/template/getTemplateVariables.js

@@ -12,6 +12,22 @@ const mustache_1 = __importDefault(require("mustache"));
  */
 function getTemplateVariables(args) {
     const { template } = args;
+    if (typeof template === "string") {
+        return getTemplateVariablesFromString(template);
+    }
+    return template.reduce((acc, message) => {
+        if (message.content !== undefined && typeof message.content === "string") {
+            return [...acc, ...getTemplateVariablesFromString(message.content)];
+        }
+        return acc;
+    }, []);
+}
+/**
+ * Parse out the template variables of a string template
+ * @param template - The template to get the variables from
+ * @returns {string[]} a list of prompt template variables
+ */
+function getTemplateVariablesFromString(template) {
     const templateSpans = mustache_1.default.parse(template);
     return templateSpans.reduce((acc, templateSpan) => {
         const [spanType, value] = templateSpan;

package/dist/src/template/getTemplateVariables.js.map

@@ -1 +1 @@
-{"version":3,"file":"getTemplateVariables.js","sourceRoot":"","sources":["../../../src/template/getTemplateVariables.ts"],"names":[],"mappings":";;;;;AAYA,oDAUC;AApBD,wDAAgC;AAKhC;;;;GAIG;AACH,SAAgB,oBAAoB,CAAC,IAA6B;IAChE,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAC1B,MAAM,aAAa,GAAG,kBAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,YAAY,EAAE,EAAE;QAChD,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,GAAG,YAAY,CAAC;QACvC,IAAI,QAAQ,KAAK,MAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACrD,GAAG,GAAG,CAAC,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;QACxB,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EAAE,EAAc,CAAC,CAAC;AACrB,CAAC"}
+{"version":3,"file":"getTemplateVariables.js","sourceRoot":"","sources":["../../../src/template/getTemplateVariables.ts"],"names":[],"mappings":";;;;;AAYA,oDAWC;AArBD,wDAAgC;AAKhC;;;;GAIG;AACH,SAAgB,oBAAoB,CAAC,IAA6B;IAChE,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAC1B,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACjC,OAAO,8BAA8B,CAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE;QACtC,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACzE,OAAO,CAAC,GAAG,GAAG,EAAE,GAAG,8BAA8B,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QACtE,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EAAE,EAAc,CAAC,CAAC;AACrB,CAAC;AACD;;;;GAIG;AACH,SAAS,8BAA8B,CAAC,QAAgB;IACtD,MAAM,aAAa,GAAG,kBAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,YAAY,EAAE,EAAE;QAChD,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,GAAG,YAAY,CAAC;QACvC,IAAI,QAAQ,KAAK,MAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACrD,GAAG,GAAG,CAAC,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;QACxB,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EAAE,EAAc,CAAC,CAAC;AACrB,CAAC"}

package/dist/src/types/data.d.ts

@@ -0,0 +1,194 @@
+/**
+ * A value extractor that can retrieve data from an object using various methods.
+ *
+ * This type supports multiple ways to extract values from your data structure:
+ * - **String paths**: Simple property names, dot notation, or JSONPath expressions
+ * - **Function extractors**: Custom transformation functions
+ *
+ * @example
+ * **Simple property access:**
+ * ```typescript
+ * const getter: ValueGetter<{ name: string }> = "name";
+ * ```
+ *
+ * @example
+ * **Dot notation for nested properties:**
+ * ```typescript
+ * const getter: ValueGetter<{ user: { profile: { name: string } } }> = "user.profile.name";
+ * ```
+ *
+ * @example
+ * **Array element access:**
+ * ```typescript
+ * const getter: ValueGetter<{ items: string[] }> = "items[0]";
+ * ```
+ *
+ * @example
+ * **JSONPath expression:**
+ * ```typescript
+ * const getter: ValueGetter<{ items: Array<{ id: number }> }> = "$.items[*].id";
+ * ```
+ *
+ * @example
+ * **Function-based extraction:**
+ * ```typescript
+ * const getter: ValueGetter<{ firstName: string; lastName: string }> =
+ *   (data) => `${data.firstName} ${data.lastName}`;
+ * ```
+ *
+ * @example
+ * **Complex transformation:**
+ * ```typescript
+ * const getter: ValueGetter<{ scores: number[] }> =
+ *   (data) => data.scores.reduce((a, b) => a + b, 0) / data.scores.length;
+ * ```
+ *
+ * @typeParam DataType - The type of the data object to extract values from
+ */
+export type ValueGetter<DataType extends Record<string, unknown>> = string | ((data: DataType) => any);
+/**
+ * A mapping configuration that transforms data from one structure to another.
+ *
+ * This type defines how to map fields from your data structure to the fields
+ * expected by an evaluator or other component. The mapping is flexible and
+ * supports multiple extraction methods.
+ *
+ * **Key Features:**
+ * - Preserves original data fields
+ * - Adds/overrides fields with mapped values
+ * - Supports nested property access
+ * - Supports array element access
+ * - Supports JSONPath expressions for complex queries
+ * - Supports function-based transformations
+ *
+ * @example
+ * **Basic field mapping:**
+ * ```typescript
+ * type MyData = {
+ *   userQuery: string;
+ *   context: string;
+ *   response: string;
+ * };
+ *
+ * const mapping: ObjectMapping<MyData> = {
+ *   input: "userQuery",      // Map "input" to "userQuery"
+ *   reference: "context",    // Map "reference" to "context"
+ *   output: "response",       // Map "output" to "response"
+ * };
+ * ```
+ *
+ * @example
+ * **Nested property mapping:**
+ * ```typescript
+ * type ApiData = {
+ *   request: {
+ *     body: {
+ *       query: string;
+ *       context: string;
+ *     };
+ *   };
+ *   response: {
+ *     data: {
+ *       text: string;
+ *     };
+ *   };
+ * };
+ *
+ * const mapping: ObjectMapping<ApiData> = {
+ *   input: "request.body.query",
+ *   reference: "request.body.context",
+ *   output: "response.data.text",
+ * };
+ * ```
+ *
+ * @example
+ * **Array element access:**
+ * ```typescript
+ * type DataWithArrays = {
+ *   messages: Array<{ role: string; content: string }>;
+ *   sources: string[];
+ * };
+ *
+ * const mapping: ObjectMapping<DataWithArrays> = {
+ *   firstMessage: "messages[0].content",
+ *   lastSource: "sources[-1]",  // Last element
+ *   allRoles: "$.messages[*].role",  // JSONPath for all roles
+ * };
+ * ```
+ *
+ * @example
+ * **Function-based transformations:**
+ * ```typescript
+ * type RawData = {
+ *   firstName: string;
+ *   lastName: string;
+ *   contexts: string[];
+ *   scores: number[];
+ * };
+ *
+ * const mapping: ObjectMapping<RawData> = {
+ *   // Combine fields
+ *   fullName: (data) => `${data.firstName} ${data.lastName}`,
+ *   // Transform array to string
+ *   contextText: (data) => data.contexts.join("\n\n"),
+ *   // Calculate derived value
+ *   averageScore: (data) =>
+ *     data.scores.reduce((a, b) => a + b, 0) / data.scores.length,
+ *   // Conditional logic
+ *   status: (data) => data.scores.length > 0 ? "active" : "inactive",
+ * };
+ * ```
+ *
+ * @example
+ * **Mixed mapping types:**
+ * ```typescript
+ * type ComplexData = {
+ *   user: {
+ *     name: string;
+ *     email: string;
+ *   };
+ *   items: Array<{ id: number; name: string }>;
+ *   metadata: {
+ *     tags: string[];
+ *   };
+ * };
+ *
+ * const mapping: ObjectMapping<ComplexData> = {
+ *   // Simple dot notation
+ *   userName: "user.name",
+ *   // Array access
+ *   firstItemId: "items[0].id",
+ *   // JSONPath for complex query
+ *   allItemNames: "$.items[*].name",
+ *   // Function for transformation
+ *   formattedTags: (data) => data.metadata.tags.map(t => `#${t}`).join(" "),
+ * };
+ * ```
+ *
+ * @example
+ * **Real-world evaluator binding:**
+ * ```typescript
+ * import { bindEvaluator, createHallucinationEvaluator } from "@arizeai/phoenix-evals";
+ *
+ * type QAData = {
+ *   question: string;
+ *   context: string;
+ *   answer: string;
+ * };
+ *
+ * const mapping: ObjectMapping<QAData> = {
+ *   input: "question",      // Evaluator expects "input"
+ *   reference: "context",   // Evaluator expects "reference"
+ *   output: "answer",       // Evaluator expects "output"
+ * };
+ *
+ * const evaluator = bindEvaluator(
+ *   createHallucinationEvaluator({ model: openai("gpt-4") }),
+ *   { inputMapping: mapping }
+ * );
+ * ```
+ *
+ * @typeParam DataType - The type of the data object being mapped
+ */
+export type ObjectMapping<DataType extends Record<string, unknown>> = Record<string, ValueGetter<DataType>>;
+//# sourceMappingURL=data.d.ts.map

package/dist/src/types/data.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../src/types/data.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,MAAM,MAAM,WAAW,CAAC,QAAQ,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAC5D,MAAM,GAEN,CAAC,CAAC,IAAI,EAAE,QAAQ,KAAK,GAAG,CAAC,CAAC;AAE9B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+IG;AACH,MAAM,MAAM,aAAa,CAAC,QAAQ,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,MAAM,CAC1E,MAAM,EACN,WAAW,CAAC,QAAQ,CAAC,CACtB,CAAC"}

package/dist/src/types/data.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=data.js.map

package/dist/src/types/data.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"data.js","sourceRoot":"","sources":["../../../src/types/data.ts"],"names":[],"mappings":""}

package/dist/src/types/evals.d.ts

@@ -1,5 +1,6 @@
+import { ObjectMapping } from "./data";
 import { WithTelemetry } from "./otel";
-import { Template } from "./templating";
+import { PromptTemplate } from "./templating";
 import { LanguageModel } from "ai";
 /**
  * A specific AI example that is under evaluation
@@ -66,21 +67,34 @@ export interface CreateClassifierArgs extends WithTelemetry {
     /**
      * The prompt template to use for classification
      */
-    promptTemplate: Template;
+    promptTemplate: PromptTemplate;
 }
-export interface CreateEvaluatorArgs {
+export interface CreateEvaluatorArgs<ExampleType extends Record<string, unknown> = Record<string, unknown>> extends WithTelemetry {
     /**
      * The name of the metric that the evaluator produces
      * E.x. "correctness"
      */
     name: string;
+    /**
+     * The kind of the evaluation. Also known as the "kind" of evaluator.
+     */
+    kind: EvaluationKind;
     /**
      * If present, represents the direction in which you want the metric to be optimized
      * E.x. "MAXIMIZE" means you want the number to be higher.
      */
     optimizationDirection?: OptimizationDirection;
+    /**
+     * The mapping of the input to evaluate to the shape that the evaluator expects
+     */
+    inputMapping?: ObjectMapping<ExampleType>;
 }
-export interface CreateClassificationEvaluatorArgs extends CreateClassifierArgs, CreateEvaluatorArgs {
+export type CreateLLMEvaluatorArgs<RecordType extends Record<string, unknown>> = Omit<CreateEvaluatorArgs<RecordType>, "kind">;
+export interface CreateClassificationEvaluatorArgs<RecordType extends Record<string, unknown>> extends CreateClassifierArgs, CreateLLMEvaluatorArgs<RecordType> {
+    /**
+     * The prompt template to use for classification
+     */
+    promptTemplate: PromptTemplate;
 }
 export type EvaluatorFn<ExampleType extends Record<string, unknown>> = (args: ExampleType) => Promise<EvaluationResult>;
 /**
@@ -114,7 +128,7 @@ interface EvaluatorDescription {
  * The Base Evaluator interface
  * This is the interface that all evaluators must implement
  */
-export interface Evaluator<ExampleType extends Record<string, unknown>> extends EvaluatorDescription {
+export interface EvaluatorInterface<ExampleType extends Record<string, unknown>> extends EvaluatorDescription {
     /**
      * The function that evaluates the example
      */