@arizeai/phoenix-evals 0.0.8 → 0.2.0

package/dist/esm/types/evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAEvC;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,aAAa,CAAC;CACtB;AAED,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC"}
+{"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAEvC;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,aAAa,CAAC;CACtB;AAED,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,iCACf,SAAQ,oBAAoB;IAC5B;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B;;GAEG;AACH,KAAK,gBAAgB,GAAG,KAAK,GAAG,MAAM,CAAC;AAEvC;;;GAGG;AACH,KAAK,qBAAqB,GAAG,UAAU,GAAG,UAAU,CAAC;AAErD;;;GAGG;AACH,MAAM,WAAW,SAAS,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IACpE;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,MAAM,EAAE,gBAAgB,CAAC;IACzB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;OAEG;IACH,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;CACpC"}

package/dist/src/default_templates/HALLUCINATION_TEMPLATE.d.ts

@@ -1,6 +1,6 @@
 export declare const HALLUCINATION_TEMPLATE = "\nIn this task, you will be presented with a query, a reference text and an answer. The answer is\ngenerated to the question based on the reference text. The answer may contain false information. You\nmust use the reference text to determine if the answer to the question contains false information,\nif the answer is a hallucination of facts. Your objective is to determine whether the answer text\ncontains factual information and is not a hallucination. A 'hallucination' refers to\nan answer that is not based on the reference text or assumes information that is not available in\nthe reference text. Your response should be a single word: either \"factual\" or \"hallucinated\", and\nit should not include any other text or characters. \"hallucinated\" indicates that the answer\nprovides factually inaccurate information to the query based on the reference text. \"factual\"\nindicates that the answer to the question is correct relative to the reference text, and does not\ncontain made up information. Please read the query and reference text carefully before determining\nyour response.\n\n    [BEGIN DATA]\n    ************\n    [Query]: {{input}}\n    ************\n    [Reference text]: {{reference}}\n    ************\n    [Answer]: {{output}}\n    ************\n    [END DATA]\n\nIs the answer above factual or hallucinated based on the query and reference text?\n";
 export declare const HALLUCINATION_CHOICES: {
-    factual: number;
     hallucinated: number;
+    factual: number;
 };
 //# sourceMappingURL=HALLUCINATION_TEMPLATE.d.ts.map

package/dist/src/default_templates/HALLUCINATION_TEMPLATE.js

@@ -28,7 +28,7 @@ your response.
 Is the answer above factual or hallucinated based on the query and reference text?
 `;
 exports.HALLUCINATION_CHOICES = {
-    factual: 1,
-    hallucinated: 0,
+    hallucinated: 1,
+    factual: 0,
 };
 //# sourceMappingURL=HALLUCINATION_TEMPLATE.js.map

package/dist/src/default_templates/HALLUCINATION_TEMPLATE.js.map

@@ -1 +1 @@
-{"version":3,"file":"HALLUCINATION_TEMPLATE.js","sourceRoot":"","sources":["../../../src/default_templates/HALLUCINATION_TEMPLATE.ts"],"names":[],"mappings":";;;AAAa,QAAA,sBAAsB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;CAyBrC,CAAC;AAEW,QAAA,qBAAqB,GAAG;IACnC,OAAO,EAAE,CAAC;IACV,YAAY,EAAE,CAAC;CAChB,CAAC"}
+{"version":3,"file":"HALLUCINATION_TEMPLATE.js","sourceRoot":"","sources":["../../../src/default_templates/HALLUCINATION_TEMPLATE.ts"],"names":[],"mappings":";;;AAAa,QAAA,sBAAsB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;CAyBrC,CAAC;AAEW,QAAA,qBAAqB,GAAG;IACnC,YAAY,EAAE,CAAC;IACf,OAAO,EAAE,CAAC;CACX,CAAC"}

package/dist/src/llm/createClassificationEvaluator.d.ts

@@ -0,0 +1,3 @@
+import { CreateClassificationEvaluatorArgs, Evaluator } from "../types/evals";
+export declare function createClassificationEvaluator<ExampleType extends Record<string, unknown>>(args: CreateClassificationEvaluatorArgs): Evaluator<ExampleType>;
+//# sourceMappingURL=createClassificationEvaluator.d.ts.map

package/dist/src/llm/createClassificationEvaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"createClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAG9E,wBAAgB,6BAA6B,CAC3C,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC3C,IAAI,EAAE,iCAAiC,GAAG,SAAS,CAAC,WAAW,CAAC,CAOjE"}

package/dist/src/llm/createClassificationEvaluator.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createClassificationEvaluator = createClassificationEvaluator;
+const createClassifierFn_1 = require("./createClassifierFn");
+function createClassificationEvaluator(args) {
+    return {
+        name: args.name,
+        source: "LLM",
+        optimizationDirection: args.optimizationDirection,
+        evaluate: (0, createClassifierFn_1.createClassifierFn)(args),
+    };
+}
+//# sourceMappingURL=createClassificationEvaluator.js.map

package/dist/src/llm/createClassificationEvaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":";;AAGA,sEASC;AAXD,6DAA0D;AAE1D,SAAgB,6BAA6B,CAE3C,IAAuC;IACvC,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,MAAM,EAAE,KAAK;QACb,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;QACjD,QAAQ,EAAE,IAAA,uCAAkB,EAAC,IAAI,CAAC;KACnC,CAAC;AACJ,CAAC"}

package/dist/src/llm/createClassifierFn.d.ts

@@ -0,0 +1,6 @@
+import { CreateClassifierArgs, EvaluatorFn } from "../types/evals";
+/**
+ * A function that serves as a factory that will output a classification evaluator function
+ */
+export declare function createClassifierFn<ExampleType extends Record<string, unknown>>(args: CreateClassifierArgs): EvaluatorFn<ExampleType>;
+//# sourceMappingURL=createClassifierFn.d.ts.map

package/dist/src/llm/createClassifierFn.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"createClassifierFn.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassifierFn.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,oBAAoB,EACpB,WAAW,EACZ,MAAM,gBAAgB,CAAC;AAkBxB;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC5E,IAAI,EAAE,oBAAoB,GACzB,WAAW,CAAC,WAAW,CAAC,CA4B1B"}

package/dist/src/llm/{createClassifier.js → createClassifierFn.js}

@@ -11,7 +11,7 @@ var __rest = (this && this.__rest) || function (s, e) {
     return t;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createClassifier = createClassifier;
+exports.createClassifierFn = createClassifierFn;
 const generateClassification_1 = require("./generateClassification");
 const template_1 = require("../template");
 /**
@@ -26,9 +26,9 @@ function choicesToLabels(choices) {
     return labels;
 }
 /**
- * A function that serves as a factory that will output a classification evaluator
+ * A function that serves as a factory that will output a classification evaluator function
  */
-function createClassifier(args) {
+function createClassifierFn(args) {
     const { model, choices, promptTemplate } = args, rest = __rest(args, ["model", "choices", "promptTemplate"]);
     return async (args) => {
         const templateVariables = Object.assign({}, args);
@@ -42,4 +42,4 @@ function createClassifier(args) {
         return Object.assign({ score }, classification);
     };
 }
-//# sourceMappingURL=createClassifier.js.map
+//# sourceMappingURL=createClassifierFn.js.map

package/dist/src/llm/createClassifierFn.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createClassifierFn.js","sourceRoot":"","sources":["../../../src/llm/createClassifierFn.ts"],"names":[],"mappings":";;;;;;;;;;;;;AA0BA,gDA8BC;AAlDD,qEAAkE;AAClE,0CAA6C;AAE7C;;;GAGG;AACH,SAAS,eAAe,CACtB,OAAiC;IAEjC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,MAA+B,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,SAAgB,kBAAkB,CAChC,IAA0B;IAE1B,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,KAAc,IAAI,EAAb,IAAI,UAAK,IAAI,EAAlD,sCAA2C,CAAO,CAAC;IAEzD,OAAO,KAAK,EAAE,IAAiB,EAA6B,EAAE;QAC5D,MAAM,iBAAiB,qBAClB,IAAI,CACR,CAAC;QAEF,MAAM,MAAM,GAAG,IAAA,yBAAc,EAAC;YAC5B,QAAQ,EAAE,cAAc;YACxB,SAAS,EAAE,iBAAiB;SAC7B,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,MAAM,IAAA,+CAAsB,kBACjD,KAAK,EACL,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,EAChC,MAAM,IACH,IAAI,EACP,CAAC;QAEH,mEAAmE;QACnE,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAE5C,uBACE,KAAK,IACF,cAAc,EACjB;IACJ,CAAC,CAAC;AACJ,CAAC"}

package/dist/src/llm/createDocumentRelevancyEvaluator.d.ts

@@ -1,7 +1,9 @@
-import { CreateClassifierArgs, EvaluatorFn } from "../types/evals";
-export interface DocumentRelevancyEvaluatorArgs extends Omit<CreateClassifierArgs, "promptTemplate" | "choices"> {
-    choices?: CreateClassifierArgs["choices"];
-    promptTemplate?: CreateClassifierArgs["promptTemplate"];
+import { CreateClassificationEvaluatorArgs, Evaluator } from "../types/evals";
+export interface DocumentRelevancyEvaluatorArgs extends Omit<CreateClassificationEvaluatorArgs, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
+    optimizationDirection?: CreateClassificationEvaluatorArgs["optimizationDirection"];
+    name?: CreateClassificationEvaluatorArgs["name"];
+    choices?: CreateClassificationEvaluatorArgs["choices"];
+    promptTemplate?: CreateClassificationEvaluatorArgs["promptTemplate"];
 }
 /**
  * An example to be evaluated by the document relevancy evaluator.
@@ -29,12 +31,12 @@ export type DocumentRelevancyExample = {
  * @example
  * ```ts
  * const evaluator = createDocumentRelevancyEvaluator({ model: openai("gpt-4o-mini") });
- * const result = await evaluator({
+ * const result = await evaluator.evaluate({
  *   input: "What is the capital of France?",
  *   documentText: "Paris is the capital and most populous city of France.",
  * });
  * console.log(result.label); // "relevant" or "unrelated"
  * ```
  */
-export declare function createDocumentRelevancyEvaluator(args: DocumentRelevancyEvaluatorArgs): EvaluatorFn<DocumentRelevancyExample>;
+export declare function createDocumentRelevancyEvaluator(args: DocumentRelevancyEvaluatorArgs): Evaluator<DocumentRelevancyExample>;
 //# sourceMappingURL=createDocumentRelevancyEvaluator.d.ts.map

package/dist/src/llm/createDocumentRelevancyEvaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createDocumentRelevancyEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAMnE,MAAM,WAAW,8BACf,SAAQ,IAAI,CAAC,oBAAoB,EAAE,gBAAgB,GAAG,SAAS,CAAC;IAChE,OAAO,CAAC,EAAE,oBAAoB,CAAC,SAAS,CAAC,CAAC;IAC1C,cAAc,CAAC,EAAE,oBAAoB,CAAC,gBAAgB,CAAC,CAAC;CACzD;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,8BAA8B,GACnC,WAAW,CAAC,wBAAwB,CAAC,CAcvC"}
+{"version":3,"file":"createDocumentRelevancyEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAO9E,MAAM,WAAW,8BACf,SAAQ,IAAI,CACV,iCAAiC,EACjC,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,uBAAuB,CAAC,CAAC;IACnF,IAAI,CAAC,EAAE,iCAAiC,CAAC,MAAM,CAAC,CAAC;IACjD,OAAO,CAAC,EAAE,iCAAiC,CAAC,SAAS,CAAC,CAAC;IACvD,cAAc,CAAC,EAAE,iCAAiC,CAAC,gBAAgB,CAAC,CAAC;CACtE;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,8BAA8B,GACnC,SAAS,CAAC,wBAAwB,CAAC,CAiBrC"}

package/dist/src/llm/createDocumentRelevancyEvaluator.js

@@ -12,8 +12,8 @@ var __rest = (this && this.__rest) || function (s, e) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createDocumentRelevancyEvaluator = createDocumentRelevancyEvaluator;
-const createClassifier_1 = require("./createClassifier");
 const DOCUMENT_RELEVANCY_TEMPLATE_1 = require("../default_templates/DOCUMENT_RELEVANCY_TEMPLATE");
+const createClassificationEvaluator_1 = require("./createClassificationEvaluator");
 /**
  * Creates a document relevancy evaluator function.
  *
@@ -33,7 +33,7 @@ const DOCUMENT_RELEVANCY_TEMPLATE_1 = require("../default_templates/DOCUMENT_REL
  * @example
  * ```ts
  * const evaluator = createDocumentRelevancyEvaluator({ model: openai("gpt-4o-mini") });
- * const result = await evaluator({
+ * const result = await evaluator.evaluate({
  *   input: "What is the capital of France?",
  *   documentText: "Paris is the capital and most populous city of France.",
  * });
@@ -41,9 +41,10 @@ const DOCUMENT_RELEVANCY_TEMPLATE_1 = require("../default_templates/DOCUMENT_REL
  * ```
  */
 function createDocumentRelevancyEvaluator(args) {
-    const { choices = DOCUMENT_RELEVANCY_TEMPLATE_1.DOCUMENT_RELEVANCY_CHOICES, promptTemplate = DOCUMENT_RELEVANCY_TEMPLATE_1.DOCUMENT_RELEVANCY_TEMPLATE } = args, rest = __rest(args, ["choices", "promptTemplate"]);
-    const documentRelevancyEvaluatorFn = (0, createClassifier_1.createClassifier)(Object.assign(Object.assign(Object.assign({}, args), { promptTemplate,
-        choices }), rest));
-    return documentRelevancyEvaluatorFn;
+    const { choices = DOCUMENT_RELEVANCY_TEMPLATE_1.DOCUMENT_RELEVANCY_CHOICES, promptTemplate = DOCUMENT_RELEVANCY_TEMPLATE_1.DOCUMENT_RELEVANCY_TEMPLATE, optimizationDirection = "MAXIMIZE", name = "document_relevancy" } = args, rest = __rest(args, ["choices", "promptTemplate", "optimizationDirection", "name"]);
+    return (0, createClassificationEvaluator_1.createClassificationEvaluator)(Object.assign(Object.assign(Object.assign({}, args), { promptTemplate,
+        choices,
+        optimizationDirection,
+        name }), rest));
 }
 //# sourceMappingURL=createDocumentRelevancyEvaluator.js.map

package/dist/src/llm/createDocumentRelevancyEvaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"createDocumentRelevancyEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;AA+CA,4EAgBC;AA/DD,yDAAsD;AAEtD,kGAG0D;AAgB1D;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAgB,gCAAgC,CAC9C,IAAoC;IAEpC,MAAM,EACJ,OAAO,GAAG,wDAA0B,EACpC,cAAc,GAAG,yDAA2B,KAE1C,IAAI,EADH,IAAI,UACL,IAAI,EAJF,6BAIL,CAAO,CAAC;IACT,MAAM,4BAA4B,GAChC,IAAA,mCAAgB,gDACX,IAAI,KACP,cAAc;QACd,OAAO,KACJ,IAAI,EACP,CAAC;IACL,OAAO,4BAA4B,CAAC;AACtC,CAAC"}
+{"version":3,"file":"createDocumentRelevancyEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAoDA,4EAmBC;AAtED,kGAG0D;AAC1D,mFAAgF;AAqBhF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAgB,gCAAgC,CAC9C,IAAoC;IAEpC,MAAM,EACJ,OAAO,GAAG,wDAA0B,EACpC,cAAc,GAAG,yDAA2B,EAC5C,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,oBAAoB,KAEzB,IAAI,EADH,IAAI,UACL,IAAI,EANF,8DAML,CAAO,CAAC;IAET,OAAO,IAAA,6DAA6B,gDAC/B,IAAI,KACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI,KACD,IAAI,EACP,CAAC;AACL,CAAC"}

package/dist/src/llm/createHallucinationEvaluator.d.ts

@@ -1,7 +1,9 @@
-import { CreateClassifierArgs, EvaluatorFn } from "../types/evals";
-export interface HallucinationEvaluatorArgs extends Omit<CreateClassifierArgs, "promptTemplate" | "choices"> {
-    choices?: CreateClassifierArgs["choices"];
-    promptTemplate?: CreateClassifierArgs["promptTemplate"];
+import { CreateClassificationEvaluatorArgs, Evaluator } from "../types/evals";
+export interface HallucinationEvaluatorArgs extends Omit<CreateClassificationEvaluatorArgs, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
+    optimizationDirection?: CreateClassificationEvaluatorArgs["optimizationDirection"];
+    name?: CreateClassificationEvaluatorArgs["name"];
+    choices?: CreateClassificationEvaluatorArgs["choices"];
+    promptTemplate?: CreateClassificationEvaluatorArgs["promptTemplate"];
 }
 /**
  * An example to be evaluated by the hallucination evaluator.
@@ -18,5 +20,5 @@ export type HallucinationExample = {
  * @param args - The arguments for creating the hallucination evaluator.
  * @returns A function that evaluates whether an answer is factual or hallucinated based on a query and reference text.
  */
-export declare function createHallucinationEvaluator(args: HallucinationEvaluatorArgs): EvaluatorFn<HallucinationExample>;
+export declare function createHallucinationEvaluator(args: HallucinationEvaluatorArgs): Evaluator<HallucinationExample>;
 //# sourceMappingURL=createHallucinationEvaluator.d.ts.map

package/dist/src/llm/createHallucinationEvaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createHallucinationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAMnE,MAAM,WAAW,0BACf,SAAQ,IAAI,CAAC,oBAAoB,EAAE,gBAAgB,GAAG,SAAS,CAAC;IAChE,OAAO,CAAC,EAAE,oBAAoB,CAAC,SAAS,CAAC,CAAC;IAC1C,cAAc,CAAC,EAAE,oBAAoB,CAAC,gBAAgB,CAAC,CAAC;CACzD;AAED;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AACF;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,0BAA0B,GAC/B,WAAW,CAAC,oBAAoB,CAAC,CAanC"}
+{"version":3,"file":"createHallucinationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAO9E,MAAM,WAAW,0BACf,SAAQ,IAAI,CACV,iCAAiC,EACjC,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,uBAAuB,CAAC,CAAC;IACnF,IAAI,CAAC,EAAE,iCAAiC,CAAC,MAAM,CAAC,CAAC;IACjD,OAAO,CAAC,EAAE,iCAAiC,CAAC,SAAS,CAAC,CAAC;IACvD,cAAc,CAAC,EAAE,iCAAiC,CAAC,gBAAgB,CAAC,CAAC;CACtE;AAED;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AACF;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,0BAA0B,GAC/B,SAAS,CAAC,oBAAoB,CAAC,CAgBjC"}

package/dist/src/llm/createHallucinationEvaluator.js

@@ -12,8 +12,8 @@ var __rest = (this && this.__rest) || function (s, e) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createHallucinationEvaluator = createHallucinationEvaluator;
-const createClassifier_1 = require("./createClassifier");
 const HALLUCINATION_TEMPLATE_1 = require("../default_templates/HALLUCINATION_TEMPLATE");
+const createClassificationEvaluator_1 = require("./createClassificationEvaluator");
 /**
  * Creates a function that evaluates whether an answer is factual or hallucinated based on a query and reference text.
  *
@@ -21,9 +21,10 @@ const HALLUCINATION_TEMPLATE_1 = require("../default_templates/HALLUCINATION_TEM
  * @returns A function that evaluates whether an answer is factual or hallucinated based on a query and reference text.
  */
 function createHallucinationEvaluator(args) {
-    const { choices = HALLUCINATION_TEMPLATE_1.HALLUCINATION_CHOICES, promptTemplate = HALLUCINATION_TEMPLATE_1.HALLUCINATION_TEMPLATE } = args, rest = __rest(args, ["choices", "promptTemplate"]);
-    const hallucinationEvaluatorFn = (0, createClassifier_1.createClassifier)(Object.assign(Object.assign(Object.assign({}, args), { promptTemplate,
-        choices }), rest));
-    return hallucinationEvaluatorFn;
+    const { choices = HALLUCINATION_TEMPLATE_1.HALLUCINATION_CHOICES, promptTemplate = HALLUCINATION_TEMPLATE_1.HALLUCINATION_TEMPLATE, optimizationDirection = "MINIMIZE", name = "hallucination" } = args, rest = __rest(args, ["choices", "promptTemplate", "optimizationDirection", "name"]);
+    return (0, createClassificationEvaluator_1.createClassificationEvaluator)(Object.assign(Object.assign(Object.assign({}, args), { promptTemplate,
+        choices,
+        optimizationDirection,
+        name }), rest));
 }
 //# sourceMappingURL=createHallucinationEvaluator.js.map

package/dist/src/llm/createHallucinationEvaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"createHallucinationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;AA4BA,oEAeC;AA3CD,yDAAsD;AAEtD,wFAGqD;AAiBrD;;;;;GAKG;AACH,SAAgB,4BAA4B,CAC1C,IAAgC;IAEhC,MAAM,EACJ,OAAO,GAAG,8CAAqB,EAC/B,cAAc,GAAG,+CAAsB,KAErC,IAAI,EADH,IAAI,UACL,IAAI,EAJF,6BAIL,CAAO,CAAC;IACT,MAAM,wBAAwB,GAAG,IAAA,mCAAgB,gDAC5C,IAAI,KACP,cAAc;QACd,OAAO,KACJ,IAAI,EACP,CAAC;IACH,OAAO,wBAAwB,CAAC;AAClC,CAAC"}
+{"version":3,"file":"createHallucinationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAiCA,oEAkBC;AAlDD,wFAGqD;AACrD,mFAAgF;AAsBhF;;;;;GAKG;AACH,SAAgB,4BAA4B,CAC1C,IAAgC;IAEhC,MAAM,EACJ,OAAO,GAAG,8CAAqB,EAC/B,cAAc,GAAG,+CAAsB,EACvC,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,eAAe,KAEpB,IAAI,EADH,IAAI,UACL,IAAI,EANF,8DAML,CAAO,CAAC;IACT,OAAO,IAAA,6DAA6B,gDAC/B,IAAI,KACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI,KACD,IAAI,EACP,CAAC;AACL,CAAC"}

package/dist/src/llm/index.d.ts

@@ -1,5 +1,6 @@
 export * from "./generateClassification";
-export * from "./createClassifier";
+export * from "./createClassifierFn";
+export * from "./createClassificationEvaluator";
 export * from "./createHallucinationEvaluator";
 export * from "./createDocumentRelevancyEvaluator";
 //# sourceMappingURL=index.d.ts.map

package/dist/src/llm/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAC;AACzC,cAAc,oBAAoB,CAAC;AACnC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,oCAAoC,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAC;AACzC,cAAc,sBAAsB,CAAC;AACrC,cAAc,iCAAiC,CAAC;AAChD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,oCAAoC,CAAC"}

package/dist/src/llm/index.js

@@ -15,7 +15,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./generateClassification"), exports);
-__exportStar(require("./createClassifier"), exports);
+__exportStar(require("./createClassifierFn"), exports);
+__exportStar(require("./createClassificationEvaluator"), exports);
 __exportStar(require("./createHallucinationEvaluator"), exports);
 __exportStar(require("./createDocumentRelevancyEvaluator"), exports);
 //# sourceMappingURL=index.js.map

package/dist/src/llm/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2DAAyC;AACzC,qDAAmC;AACnC,iEAA+C;AAC/C,qEAAmD"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2DAAyC;AACzC,uDAAqC;AACrC,kEAAgD;AAChD,iEAA+C;AAC/C,qEAAmD"}

package/dist/src/types/evals.d.ts

@@ -67,5 +67,50 @@ export interface CreateClassifierArgs extends WithTelemetry {
      */
     promptTemplate: string;
 }
+export interface CreateClassificationEvaluatorArgs extends CreateClassifierArgs {
+    /**
+     * The name of the metric that the evaluator produces
+     * E.x. "correctness"
+     */
+    name: string;
+    /**
+     * If present, represents the direction in which you want the metric to be optimized
+     * E.x. "MAXIMIZE" means you want the number to be higher.
+     */
+    optimizationDirection?: OptimizationDirection;
+}
 export type EvaluatorFn<ExampleType extends Record<string, unknown>> = (args: ExampleType) => Promise<EvaluationResult>;
+/**
+ * The source of the evaluation
+ */
+type EvaluationSource = "LLM" | "CODE";
+/**
+ * The direction to optimize the numeric evaluation score
+ * E.x. "MAXIMIZE" means that the higher the score, the better the evaluation
+ */
+type OptimizationDirection = "MAXIMIZE" | "MINIMIZE";
+/**
+ * The Base Evaluator interface
+ * This is the interface that all evaluators must implement
+ */
+export interface Evaluator<ExampleType extends Record<string, unknown>> {
+    /**
+     * The name of the evaluator / the metric that it measures
+     */
+    name: string;
+    /**
+     * The source of the evaluation. Also known as the "kind" of evaluator.
+     */
+    source: EvaluationSource;
+    /**
+     * The direction to optimize the numeric evaluation score
+     * E.x. "MAXIMIZE" means that the higher the score, the better the evaluation
+     */
+    optimizationDirection?: OptimizationDirection;
+    /**
+     * The function that evaluates the example
+     */
+    evaluate: EvaluatorFn<ExampleType>;
+}
+export {};
 //# sourceMappingURL=evals.d.ts.map

package/dist/src/types/evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAEvC;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,aAAa,CAAC;CACtB;AAED,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC"}
+{"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAEvC;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,aAAa,CAAC;CACtB;AAED,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,iCACf,SAAQ,oBAAoB;IAC5B;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B;;GAEG;AACH,KAAK,gBAAgB,GAAG,KAAK,GAAG,MAAM,CAAC;AAEvC;;;GAGG;AACH,KAAK,qBAAqB,GAAG,UAAU,GAAG,UAAU,CAAC;AAErD;;;GAGG;AACH,MAAM,WAAW,SAAS,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IACpE;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,MAAM,EAAE,gBAAgB,CAAC;IACzB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;OAEG;IACH,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;CACpC"}