@arizeai/phoenix-evals 0.2.0 → 0.2.1

package/README.md

@@ -12,7 +12,14 @@
     </div>
 </h1>
 
-![NPM](https://img.shields.io/npm/v/%40arizeai%2Fphoenix-evals)
+<p align="center">
+    <a href="https://www.npmjs.com/package/@arizeai/phoenix-evals">
+        <img src="https://img.shields.io/npm/v/%40arizeai%2Fphoenix-evals" alt="NPM Version">
+    </a>
+    <a href="https://arize-ai.github.io/phoenix/">
+        <img src="https://img.shields.io/badge/docs-blue?logo=typescript&logoColor=white" alt="Documentation">
+    </a>
+</p>
 
 This package provides a TypeScript evaluation library. It is vendor agnostic and can be used in isolation of any framework or platform. This package is still under active development and is subject to change.
 
@@ -176,3 +183,15 @@ runExperiment({
   evaluators: [hallucinationCheck],
 });
 ```
+
+## Community
+
+Join our community to connect with thousands of AI builders:
+
+- 🌍 Join our [Slack community](https://arize-ai.slack.com/join/shared_invite/zt-11t1vbu4x-xkBIHmOREQnYnYDH1GDfCg).
+- 📚 Read the [Phoenix documentation](https://arize.com/docs/phoenix).
+- 💡 Ask questions and provide feedback in the _#phoenix-support_ channel.
+- 🌟 Leave a star on our [GitHub](https://github.com/Arize-ai/phoenix).
+- 🐞 Report bugs with [GitHub Issues](https://github.com/Arize-ai/phoenix/issues).
+- 𝕏 Follow us on [𝕏](https://twitter.com/ArizePhoenix).
+- 🗺️ Check out our [roadmap](https://github.com/orgs/Arize-ai/projects/45) to see where we're heading next.

package/dist/esm/llm/ClassificationEvaluator.d.ts

@@ -0,0 +1,17 @@
+import { CreateClassificationEvaluatorArgs, EvaluatorFn, Template } from "../types/index.js";
+import { LLMEvaluator } from "./LLMEvaluator.js";
+/**
+ * An LLM evaluator that performs evaluation via classification
+ */
+export declare class ClassificationEvaluator<RecordType extends Record<string, unknown>> extends LLMEvaluator<RecordType> {
+    readonly evaluatorFn: EvaluatorFn<RecordType>;
+    readonly promptTemplate: Template;
+    private _promptTemplateVariables;
+    constructor(args: CreateClassificationEvaluatorArgs);
+    evaluate: (example: RecordType) => Promise<import("../types/index.js").EvaluationResult>;
+    /**
+     * List out the prompt template variables needed to perform evaluation
+     */
+    get promptTemplateVariables(): string[];
+}
+//# sourceMappingURL=ClassificationEvaluator.d.ts.map

package/dist/esm/llm/ClassificationEvaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iCAAiC,EACjC,WAAW,EACX,QAAQ,EACT,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAG9C;;GAEG;AACH,qBAAa,uBAAuB,CAClC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC1C,SAAQ,YAAY,CAAC,UAAU,CAAC;IAChC,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC;IAC9C,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC;IAClC,OAAO,CAAC,wBAAwB,CAAuB;gBAC3C,IAAI,EAAE,iCAAiC;IAKnD,QAAQ,GAAI,SAAS,UAAU,kDAE7B;IACF;;OAEG;IACH,IAAI,uBAAuB,IAAI,MAAM,EAAE,CAStC;CACF"}

package/dist/esm/llm/ClassificationEvaluator.js

@@ -0,0 +1,33 @@
+import { createClassifierFn } from "./createClassifierFn.js";
+import { LLMEvaluator } from "./LLMEvaluator.js";
+import { getTemplateVariables } from "../template/index.js";
+/**
+ * An LLM evaluator that performs evaluation via classification
+ */
+export class ClassificationEvaluator extends LLMEvaluator {
+    evaluatorFn;
+    promptTemplate;
+    _promptTemplateVariables;
+    constructor(args) {
+        super(args);
+        this.promptTemplate = args.promptTemplate;
+        this.evaluatorFn = createClassifierFn(args);
+    }
+    evaluate = (example) => {
+        return this.evaluatorFn(example);
+    };
+    /**
+     * List out the prompt template variables needed to perform evaluation
+     */
+    get promptTemplateVariables() {
+        // Use dynamic programming to see if it's computed already
+        if (!Array.isArray(this._promptTemplateVariables)) {
+            this._promptTemplateVariables = getTemplateVariables({
+                template: this.promptTemplate,
+            });
+        }
+        // Give a copy of the variables
+        return [...this._promptTemplateVariables];
+    }
+}
+//# sourceMappingURL=ClassificationEvaluator.js.map

package/dist/esm/llm/ClassificationEvaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD;;GAEG;AACH,MAAM,OAAO,uBAEX,SAAQ,YAAwB;IACvB,WAAW,CAA0B;IACrC,cAAc,CAAW;IAC1B,wBAAwB,CAAuB;IACvD,YAAY,IAAuC;QACjD,KAAK,CAAC,IAAI,CAAC,CAAC;QACZ,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,kBAAkB,CAAa,IAAI,CAAC,CAAC;IAC1D,CAAC;IACD,QAAQ,GAAG,CAAC,OAAmB,EAAE,EAAE;QACjC,OAAO,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACnC,CAAC,CAAC;IACF;;OAEG;IACH,IAAI,uBAAuB;QACzB,0DAA0D;QAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,wBAAwB,GAAG,oBAAoB,CAAC;gBACnD,QAAQ,EAAE,IAAI,CAAC,cAAc;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,+BAA+B;QAC/B,OAAO,CAAC,GAAG,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC5C,CAAC;CACF"}

package/dist/esm/llm/LLMEvaluator.d.ts

@@ -0,0 +1,12 @@
+import { EvaluationResult, Evaluator, OptimizationDirection, CreateEvaluatorArgs } from "../types/index.js";
+/**
+ * Base class for llm evaluation metrics / scores
+ */
+export declare abstract class LLMEvaluator<RecordType extends Record<string, unknown>> implements Evaluator<RecordType> {
+    readonly name: string;
+    readonly source: "LLM";
+    readonly optimizationDirection?: OptimizationDirection;
+    constructor({ name, optimizationDirection }: CreateEvaluatorArgs);
+    abstract evaluate(_example: RecordType): Promise<EvaluationResult>;
+}
+//# sourceMappingURL=LLMEvaluator.d.ts.map

package/dist/esm/llm/LLMEvaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LLMEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/LLMEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gBAAgB,EAChB,SAAS,EACT,qBAAqB,EACrB,mBAAmB,EACpB,MAAM,UAAU,CAAC;AAElB;;GAEG;AACH,8BAAsB,YAAY,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC3E,YAAW,SAAS,CAAC,UAAU,CAAC;IAEhC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAG,KAAK,CAAU;IACjC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;gBAC3C,EAAE,IAAI,EAAE,qBAAqB,EAAE,EAAE,mBAAmB;IAIhE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;CACnE"}

package/dist/esm/llm/LLMEvaluator.js

@@ -0,0 +1,13 @@
+/**
+ * Base class for llm evaluation metrics / scores
+ */
+export class LLMEvaluator {
+    name;
+    source = "LLM";
+    optimizationDirection;
+    constructor({ name, optimizationDirection }) {
+        this.name = name;
+        this.optimizationDirection = optimizationDirection;
+    }
+}
+//# sourceMappingURL=LLMEvaluator.js.map

package/dist/esm/llm/LLMEvaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LLMEvaluator.js","sourceRoot":"","sources":["../../../src/llm/LLMEvaluator.ts"],"names":[],"mappings":"AAOA;;GAEG;AACH,MAAM,OAAgB,YAAY;IAGvB,IAAI,CAAS;IACb,MAAM,GAAG,KAAc,CAAC;IACxB,qBAAqB,CAAyB;IACvD,YAAY,EAAE,IAAI,EAAE,qBAAqB,EAAuB;QAC9D,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;IACrD,CAAC;CAEF"}

package/dist/esm/llm/createClassificationEvaluator.d.ts

@@ -1,3 +1,4 @@
-import { CreateClassificationEvaluatorArgs, Evaluator } from "../types/evals.js";
-export declare function createClassificationEvaluator<ExampleType extends Record<string, unknown>>(args: CreateClassificationEvaluatorArgs): Evaluator<ExampleType>;
+import { CreateClassificationEvaluatorArgs } from "../types/evals.js";
+import { ClassificationEvaluator } from "./ClassificationEvaluator.js";
+export declare function createClassificationEvaluator<RecordType extends Record<string, unknown>>(args: CreateClassificationEvaluatorArgs): ClassificationEvaluator<RecordType>;
 //# sourceMappingURL=createClassificationEvaluator.d.ts.map

package/dist/esm/llm/createClassificationEvaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAG9E,wBAAgB,6BAA6B,CAC3C,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC3C,IAAI,EAAE,iCAAiC,GAAG,SAAS,CAAC,WAAW,CAAC,CAOjE"}
+{"version":3,"file":"createClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,wBAAgB,6BAA6B,CAC3C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAE1C,IAAI,EAAE,iCAAiC,GACtC,uBAAuB,CAAC,UAAU,CAAC,CAErC"}

package/dist/esm/llm/createClassificationEvaluator.js

@@ -1,10 +1,5 @@
-import { createClassifierFn } from "./createClassifierFn.js";
+import { ClassificationEvaluator } from "./ClassificationEvaluator.js";
 export function createClassificationEvaluator(args) {
-    return {
-        name: args.name,
-        source: "LLM",
-        optimizationDirection: args.optimizationDirection,
-        evaluate: createClassifierFn(args),
-    };
+    return new ClassificationEvaluator(args);
 }
 //# sourceMappingURL=createClassificationEvaluator.js.map

package/dist/esm/llm/createClassificationEvaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"createClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1D,MAAM,UAAU,6BAA6B,CAE3C,IAAuC;IACvC,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,MAAM,EAAE,KAAK;QACb,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;QACjD,QAAQ,EAAE,kBAAkB,CAAC,IAAI,CAAC;KACnC,CAAC;AACJ,CAAC"}
+{"version":3,"file":"createClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,MAAM,UAAU,6BAA6B,CAG3C,IAAuC;IAEvC,OAAO,IAAI,uBAAuB,CAAa,IAAI,CAAC,CAAC;AACvD,CAAC"}

package/dist/esm/llm/createClassifierFn.d.ts

@@ -2,5 +2,5 @@ import { CreateClassifierArgs, EvaluatorFn } from "../types/evals.js";
 /**
  * A function that serves as a factory that will output a classification evaluator function
  */
-export declare function createClassifierFn<ExampleType extends Record<string, unknown>>(args: CreateClassifierArgs): EvaluatorFn<ExampleType>;
+export declare function createClassifierFn<RecordToEvaluate extends Record<string, unknown>>(args: CreateClassifierArgs): EvaluatorFn<RecordToEvaluate>;
 //# sourceMappingURL=createClassifierFn.d.ts.map

package/dist/esm/llm/createClassifierFn.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createClassifierFn.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassifierFn.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,oBAAoB,EACpB,WAAW,EACZ,MAAM,gBAAgB,CAAC;AAkBxB;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC5E,IAAI,EAAE,oBAAoB,GACzB,WAAW,CAAC,WAAW,CAAC,CA4B1B"}
+{"version":3,"file":"createClassifierFn.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassifierFn.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,oBAAoB,EACpB,WAAW,EACZ,MAAM,gBAAgB,CAAC;AAkBxB;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,gBAAgB,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChD,IAAI,EAAE,oBAAoB,GAAG,WAAW,CAAC,gBAAgB,CAAC,CA4B3D"}

package/dist/esm/llm/createClassifierFn.js.map

@@ -1 +1 @@
-{"version":3,"file":"createClassifierFn.js","sourceRoot":"","sources":["../../../src/llm/createClassifierFn.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE7C;;;GAGG;AACH,SAAS,eAAe,CACtB,OAAiC;IAEjC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,MAA+B,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAChC,IAA0B;IAE1B,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC;IAEzD,OAAO,KAAK,EAAE,IAAiB,EAA6B,EAAE;QAC5D,MAAM,iBAAiB,GAAG;YACxB,GAAG,IAAI;SACR,CAAC;QAEF,MAAM,MAAM,GAAG,cAAc,CAAC;YAC5B,QAAQ,EAAE,cAAc;YACxB,SAAS,EAAE,iBAAiB;SAC7B,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,MAAM,sBAAsB,CAAC;YAClD,KAAK;YACL,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC;YAChC,MAAM;YACN,GAAG,IAAI;SACR,CAAC,CAAC;QAEH,mEAAmE;QACnE,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAE5C,OAAO;YACL,KAAK;YACL,GAAG,cAAc;SAClB,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}
+{"version":3,"file":"createClassifierFn.js","sourceRoot":"","sources":["../../../src/llm/createClassifierFn.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE7C;;;GAGG;AACH,SAAS,eAAe,CACtB,OAAiC;IAEjC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,MAA+B,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAEhC,IAA0B;IAC1B,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC;IAEzD,OAAO,KAAK,EAAE,IAAsB,EAA6B,EAAE;QACjE,MAAM,iBAAiB,GAAG;YACxB,GAAG,IAAI;SACR,CAAC;QAEF,MAAM,MAAM,GAAG,cAAc,CAAC;YAC5B,QAAQ,EAAE,cAAc;YACxB,SAAS,EAAE,iBAAiB;SAC7B,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,MAAM,sBAAsB,CAAC;YAClD,KAAK;YACL,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC;YAChC,MAAM;YACN,GAAG,IAAI;SACR,CAAC,CAAC;QAEH,mEAAmE;QACnE,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAE5C,OAAO;YACL,KAAK;YACL,GAAG,cAAc;SAClB,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}

package/dist/esm/llm/createDocumentRelevancyEvaluator.d.ts

@@ -6,12 +6,13 @@ export interface DocumentRelevancyEvaluatorArgs extends Omit<CreateClassificatio
     promptTemplate?: CreateClassificationEvaluatorArgs["promptTemplate"];
 }
 /**
- * An example to be evaluated by the document relevancy evaluator.
+ * A record to be evaluated by the document relevancy evaluator.
  */
-export type DocumentRelevancyExample = {
+export interface DocumentRelevancyEvaluationRecord {
     input: string;
     documentText: string;
-};
+    [key: string]: unknown;
+}
 /**
  * Creates a document relevancy evaluator function.
  *
@@ -38,5 +39,5 @@ export type DocumentRelevancyExample = {
  * console.log(result.label); // "relevant" or "unrelated"
  * ```
  */
-export declare function createDocumentRelevancyEvaluator(args: DocumentRelevancyEvaluatorArgs): Evaluator<DocumentRelevancyExample>;
+export declare function createDocumentRelevancyEvaluator<RecordType extends Record<string, unknown> = DocumentRelevancyEvaluationRecord>(args: DocumentRelevancyEvaluatorArgs): Evaluator<RecordType>;
 //# sourceMappingURL=createDocumentRelevancyEvaluator.d.ts.map

package/dist/esm/llm/createDocumentRelevancyEvaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createDocumentRelevancyEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAO9E,MAAM,WAAW,8BACf,SAAQ,IAAI,CACV,iCAAiC,EACjC,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,uBAAuB,CAAC,CAAC;IACnF,IAAI,CAAC,EAAE,iCAAiC,CAAC,MAAM,CAAC,CAAC;IACjD,OAAO,CAAC,EAAE,iCAAiC,CAAC,SAAS,CAAC,CAAC;IACvD,cAAc,CAAC,EAAE,iCAAiC,CAAC,gBAAgB,CAAC,CAAC;CACtE;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,8BAA8B,GACnC,SAAS,CAAC,wBAAwB,CAAC,CAiBrC"}
+{"version":3,"file":"createDocumentRelevancyEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAO9E,MAAM,WAAW,8BACf,SAAQ,IAAI,CACV,iCAAiC,EACjC,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,uBAAuB,CAAC,CAAC;IACnF,IAAI,CAAC,EAAE,iCAAiC,CAAC,MAAM,CAAC,CAAC;IACjD,OAAO,CAAC,EAAE,iCAAiC,CAAC,SAAS,CAAC,CAAC;IACvD,cAAc,CAAC,EAAE,iCAAiC,CAAC,gBAAgB,CAAC,CAAC;CACtE;AAED;;GAEG;AACH,MAAM,WAAW,iCAAiC;IAChD,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,gCAAgC,CAC9C,UAAU,SAAS,MAAM,CACvB,MAAM,EACN,OAAO,CACR,GAAG,iCAAiC,EACrC,IAAI,EAAE,8BAA8B,GAAG,SAAS,CAAC,UAAU,CAAC,CAiB7D"}

package/dist/esm/llm/createDocumentRelevancyEvaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"createDocumentRelevancyEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,EACL,2BAA2B,EAC3B,0BAA0B,GAC3B,MAAM,kDAAkD,CAAC;AAC1D,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAqBhF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,UAAU,gCAAgC,CAC9C,IAAoC;IAEpC,MAAM,EACJ,OAAO,GAAG,0BAA0B,EACpC,cAAc,GAAG,2BAA2B,EAC5C,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,oBAAoB,EAC3B,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IAET,OAAO,6BAA6B,CAA2B;QAC7D,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;QACJ,GAAG,IAAI;KACR,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"createDocumentRelevancyEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,EACL,2BAA2B,EAC3B,0BAA0B,GAC3B,MAAM,kDAAkD,CAAC;AAC1D,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAsBhF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,UAAU,gCAAgC,CAK9C,IAAoC;IACpC,MAAM,EACJ,OAAO,GAAG,0BAA0B,EACpC,cAAc,GAAG,2BAA2B,EAC5C,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,oBAAoB,EAC3B,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IAET,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;QACJ,GAAG,IAAI;KACR,CAAC,CAAC;AACL,CAAC"}

package/dist/esm/llm/createHallucinationEvaluator.d.ts

@@ -1,4 +1,5 @@
-import { CreateClassificationEvaluatorArgs, Evaluator } from "../types/evals.js";
+import { CreateClassificationEvaluatorArgs } from "../types/evals.js";
+import { ClassificationEvaluator } from "./ClassificationEvaluator.js";
 export interface HallucinationEvaluatorArgs extends Omit<CreateClassificationEvaluatorArgs, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
     optimizationDirection?: CreateClassificationEvaluatorArgs["optimizationDirection"];
     name?: CreateClassificationEvaluatorArgs["name"];
@@ -6,9 +7,9 @@ export interface HallucinationEvaluatorArgs extends Omit<CreateClassificationEva
     promptTemplate?: CreateClassificationEvaluatorArgs["promptTemplate"];
 }
 /**
- * An example to be evaluated by the hallucination evaluator.
+ * A record to be evaluated by the hallucination evaluator.
  */
-export type HallucinationExample = {
+export type HallucinationEvaluationRecord = {
     input: string;
     output: string;
     reference?: string;
@@ -20,5 +21,5 @@ export type HallucinationExample = {
  * @param args - The arguments for creating the hallucination evaluator.
  * @returns A function that evaluates whether an answer is factual or hallucinated based on a query and reference text.
  */
-export declare function createHallucinationEvaluator(args: HallucinationEvaluatorArgs): Evaluator<HallucinationExample>;
+export declare function createHallucinationEvaluator<RecordType extends Record<string, unknown> = HallucinationEvaluationRecord>(args: HallucinationEvaluatorArgs): ClassificationEvaluator<RecordType>;
 //# sourceMappingURL=createHallucinationEvaluator.d.ts.map

package/dist/esm/llm/createHallucinationEvaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createHallucinationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAO9E,MAAM,WAAW,0BACf,SAAQ,IAAI,CACV,iCAAiC,EACjC,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,uBAAuB,CAAC,CAAC;IACnF,IAAI,CAAC,EAAE,iCAAiC,CAAC,MAAM,CAAC,CAAC;IACjD,OAAO,CAAC,EAAE,iCAAiC,CAAC,SAAS,CAAC,CAAC;IACvD,cAAc,CAAC,EAAE,iCAAiC,CAAC,gBAAgB,CAAC,CAAC;CACtE;AAED;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AACF;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,0BAA0B,GAC/B,SAAS,CAAC,oBAAoB,CAAC,CAgBjC"}
+{"version":3,"file":"createHallucinationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AAMnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,MAAM,WAAW,0BACf,SAAQ,IAAI,CACV,iCAAiC,EACjC,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,uBAAuB,CAAC,CAAC;IACnF,IAAI,CAAC,EAAE,iCAAiC,CAAC,MAAM,CAAC,CAAC;IACjD,OAAO,CAAC,EAAE,iCAAiC,CAAC,SAAS,CAAC,CAAC;IACvD,cAAc,CAAC,EAAE,iCAAiC,CAAC,gBAAgB,CAAC,CAAC;CACtE;AAED;;GAEG;AACH,MAAM,MAAM,6BAA6B,GAAG;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AACF;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAC1C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,6BAA6B,EAC1E,IAAI,EAAE,0BAA0B,GAAG,uBAAuB,CAAC,UAAU,CAAC,CAgBvE"}

package/dist/esm/llm/createHallucinationEvaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"createHallucinationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,EACL,sBAAsB,EACtB,qBAAqB,GACtB,MAAM,6CAA6C,CAAC;AACrD,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAsBhF;;;;;GAKG;AACH,MAAM,UAAU,4BAA4B,CAC1C,IAAgC;IAEhC,MAAM,EACJ,OAAO,GAAG,qBAAqB,EAC/B,cAAc,GAAG,sBAAsB,EACvC,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,eAAe,EACtB,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAuB;QACzD,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;QACJ,GAAG,IAAI;KACR,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"createHallucinationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,EACL,sBAAsB,EACtB,qBAAqB,GACtB,MAAM,6CAA6C,CAAC;AACrD,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAuBhF;;;;;GAKG;AACH,MAAM,UAAU,4BAA4B,CAE1C,IAAgC;IAChC,MAAM,EACJ,OAAO,GAAG,qBAAqB,EAC/B,cAAc,GAAG,sBAAsB,EACvC,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,eAAe,EACtB,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;QACJ,GAAG,IAAI;KACR,CAAC,CAAC;AACL,CAAC"}

package/dist/esm/llm/generateClassification.d.ts

@@ -1,7 +1,7 @@
 import { ClassificationResult, WithLLM } from "../types/evals.js";
 import { WithTelemetry } from "../types/otel.js";
 import type { WithPrompt } from "../types/prompts.js";
-export interface ClassifyArgs extends WithLLM, WithPrompt, WithTelemetry {
+export type ClassifyArgs = WithLLM & WithTelemetry & WithPrompt & {
     /**
      * The labels to classify the example into. E.x. ["correct", "incorrect"]
      */
@@ -14,7 +14,7 @@ export interface ClassifyArgs extends WithLLM, WithPrompt, WithTelemetry {
      * The description of the schema for generating the label and explanation.
      */
     schemaDescription?: string;
-}
+};
 /**
  * A function that leverages an llm to perform a classification
  */

package/dist/esm/llm/generateClassification.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"generateClassification.d.ts","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAInD,MAAM,WAAW,YAAa,SAAQ,OAAO,EAAE,UAAU,EAAE,aAAa;IACtE;;OAEG;IACH,MAAM,EAAE,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IAC9B;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AACD;;GAEG;AACH,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,YAAY,GACjB,OAAO,CAAC,oBAAoB,CAAC,CAyB/B"}
+{"version":3,"file":"generateClassification.d.ts","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAInD,MAAM,MAAM,YAAY,GAAG,OAAO,GAChC,aAAa,GACb,UAAU,GAAG;IACX;;OAEG;IACH,MAAM,EAAE,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IAC9B;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,CAAC;AACJ;;GAEG;AACH,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,YAAY,GACjB,OAAO,CAAC,oBAAoB,CAAC,CAyB/B"}

package/dist/esm/llm/generateClassification.js.map

@@ -1 +1 @@
-{"version":3,"file":"generateClassification.js","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,cAAc,EAAE,MAAM,IAAI,CAAC;AACpC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAetC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,IAAkB;IAElB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,GAC1E,IAAI,CAAC;IAEP,MAAM,sBAAsB,GAAG;QAC7B,SAAS,EAAE,SAAS,EAAE,SAAS,IAAI,IAAI;QACvC,UAAU,EAAE,wBAAwB;QACpC,MAAM,EAAE,SAAS,EAAE,MAAM,IAAI,MAAM;KACpC,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC;QAClC,KAAK;QACL,UAAU;QACV,iBAAiB;QACjB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;YACf,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,4EAA4E;YACrG,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SACtB,CAAC;QACF,sBAAsB;QACtB,GAAG,MAAM;KACV,CAAC,CAAC;IACH,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;QAC1B,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;KACvC,CAAC;AACJ,CAAC"}
+{"version":3,"file":"generateClassification.js","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,cAAc,EAAE,MAAM,IAAI,CAAC;AACpC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAiBtC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,IAAkB;IAElB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,GAC1E,IAAI,CAAC;IAEP,MAAM,sBAAsB,GAAG;QAC7B,SAAS,EAAE,SAAS,EAAE,SAAS,IAAI,IAAI;QACvC,UAAU,EAAE,wBAAwB;QACpC,MAAM,EAAE,SAAS,EAAE,MAAM,IAAI,MAAM;KACpC,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC;QAClC,KAAK;QACL,UAAU;QACV,iBAAiB;QACjB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;YACf,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,4EAA4E;YACrG,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SACtB,CAAC;QACF,sBAAsB;QACtB,GAAG,MAAM;KACV,CAAC,CAAC;IACH,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;QAC1B,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;KACvC,CAAC;AACJ,CAAC"}

package/dist/esm/template/getTemplateVariables.d.ts

@@ -0,0 +1,12 @@
+import { Template } from "../types/templating.js";
+type GetTemplateVariableArgs = {
+    template: Template;
+};
+/**
+ * Parse out the template variables of a prompt
+ * @param {GetTemplateVariableArgs} args
+ * @returns {string[]} a list of prompt template variables
+ */
+export declare function getTemplateVariables(args: GetTemplateVariableArgs): string[];
+export {};
+//# sourceMappingURL=getTemplateVariables.d.ts.map

package/dist/esm/template/getTemplateVariables.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"getTemplateVariables.d.ts","sourceRoot":"","sources":["../../../src/template/getTemplateVariables.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAG/C,KAAK,uBAAuB,GAAG;IAC7B,QAAQ,EAAE,QAAQ,CAAC;CACpB,CAAC;AACF;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,uBAAuB,GAAG,MAAM,EAAE,CAU5E"}

package/dist/esm/template/getTemplateVariables.js

@@ -0,0 +1,18 @@
+import Mustache from "mustache";
+/**
+ * Parse out the template variables of a prompt
+ * @param {GetTemplateVariableArgs} args
+ * @returns {string[]} a list of prompt template variables
+ */
+export function getTemplateVariables(args) {
+    const { template } = args;
+    const templateSpans = Mustache.parse(template);
+    return templateSpans.reduce((acc, templateSpan) => {
+        const [spanType, value] = templateSpan;
+        if (spanType === "name" && typeof value === "string") {
+            acc = [...acc, value];
+        }
+        return acc;
+    }, []);
+}
+//# sourceMappingURL=getTemplateVariables.js.map

package/dist/esm/template/getTemplateVariables.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getTemplateVariables.js","sourceRoot":"","sources":["../../../src/template/getTemplateVariables.ts"],"names":[],"mappings":"AACA,OAAO,QAAQ,MAAM,UAAU,CAAC;AAKhC;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAA6B;IAChE,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAC1B,MAAM,aAAa,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,YAAY,EAAE,EAAE;QAChD,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,GAAG,YAAY,CAAC;QACvC,IAAI,QAAQ,KAAK,MAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACrD,GAAG,GAAG,CAAC,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;QACxB,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EAAE,EAAc,CAAC,CAAC;AACrB,CAAC"}

package/dist/esm/template/index.d.ts

@@ -1,2 +1,3 @@
 export * from "./applyTemplate.js";
+export * from "./getTemplateVariables.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/esm/template/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/template/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/template/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,wBAAwB,CAAC"}

package/dist/esm/template/index.js

@@ -1,2 +1,3 @@
 export * from "./applyTemplate.js";
+export * from "./getTemplateVariables.js";
 //# sourceMappingURL=index.js.map

package/dist/esm/template/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/template/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/template/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,wBAAwB,CAAC"}