@arizeai/phoenix-evals 0.2.0 → 0.2.1

package/dist/esm/types/evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAEvC;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,aAAa,CAAC;CACtB;AAED,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,iCACf,SAAQ,oBAAoB;IAC5B;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B;;GAEG;AACH,KAAK,gBAAgB,GAAG,KAAK,GAAG,MAAM,CAAC;AAEvC;;;GAGG;AACH,KAAK,qBAAqB,GAAG,UAAU,GAAG,UAAU,CAAC;AAErD;;;GAGG;AACH,MAAM,WAAW,SAAS,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IACpE;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,MAAM,EAAE,gBAAgB,CAAC;IACzB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;OAEG;IACH,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;CACpC"}
+{"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAEvC;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,aAAa,CAAC;CACtB;AAGD,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,mBAAmB;IAClC;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED,MAAM,WAAW,iCACf,SAAQ,oBAAoB,EAC1B,mBAAmB;CAAG;AAE1B,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,KAAK,GAAG,MAAM,CAAC;AAE9C;;;GAGG;AACH,MAAM,MAAM,qBAAqB,GAAG,UAAU,GAAG,UAAU,CAAC;AAE5D;;GAEG;AACH,UAAU,oBAAoB;IAC5B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,MAAM,EAAE,gBAAgB,CAAC;IACzB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CACpE,SAAQ,oBAAoB;IAC5B;;OAEG;IACH,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;CACpC"}

package/dist/esm/types/otel.d.ts

@@ -1,5 +1,5 @@
 import { Tracer } from "@opentelemetry/api";
-export interface WithTelemetry {
+export type WithTelemetry = {
     telemetry?: {
         /**
          * Whether OpenTelemetry is enabled on the call.
@@ -13,5 +13,5 @@ export interface WithTelemetry {
          */
         tracer?: Tracer;
     };
-}
+};
 //# sourceMappingURL=otel.d.ts.map

package/dist/esm/types/otel.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"otel.d.ts","sourceRoot":"","sources":["../../../src/types/otel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,MAAM,WAAW,aAAa;IAC5B,SAAS,CAAC,EAAE;QACV;;;;WAIG;QACH,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB;;;WAGG;QACH,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACH"}
+{"version":3,"file":"otel.d.ts","sourceRoot":"","sources":["../../../src/types/otel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,CAAC,EAAE;QACV;;;;WAIG;QACH,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB;;;WAGG;QACH,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACH,CAAC"}

package/dist/esm/types/prompts.d.ts

@@ -1,22 +1,8 @@
-import { ModelMessage } from "ai";
+import type { Prompt } from "ai";
 /**
  * Prompt part of the AI function options for model generation.
  * It contains a system message, a simple text prompt, or a list of model messages.
  * Uses ModelMessage format compatible with AI SDK v5 generateObject function.
  */
-export interface WithPrompt {
-    /**
-     * System message to include in the prompt. Can be used with `prompt` or `messages`.
-     */
-    system?: string;
-    /**
-     * A simple text prompt. You can either use `prompt` or `messages` but not both.
-     */
-    prompt?: string;
-    /**
-     * A list of model messages. You can either use `prompt` or `messages` but not both.
-     * Uses ModelMessage format for compatibility with AI SDK v5.
-     */
-    messages?: Array<ModelMessage>;
-}
+export type WithPrompt = Prompt;
 //# sourceMappingURL=prompts.d.ts.map

package/dist/esm/types/prompts.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../../../src/types/prompts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAElC;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,QAAQ,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;CAChC"}
+{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../../../src/types/prompts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AACjC;;;;GAIG;AACH,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC"}

package/dist/src/llm/ClassificationEvaluator.d.ts

@@ -0,0 +1,17 @@
+import { CreateClassificationEvaluatorArgs, EvaluatorFn, Template } from "../types";
+import { LLMEvaluator } from "./LLMEvaluator";
+/**
+ * An LLM evaluator that performs evaluation via classification
+ */
+export declare class ClassificationEvaluator<RecordType extends Record<string, unknown>> extends LLMEvaluator<RecordType> {
+    readonly evaluatorFn: EvaluatorFn<RecordType>;
+    readonly promptTemplate: Template;
+    private _promptTemplateVariables;
+    constructor(args: CreateClassificationEvaluatorArgs);
+    evaluate: (example: RecordType) => Promise<import("../types").EvaluationResult>;
+    /**
+     * List out the prompt template variables needed to perform evaluation
+     */
+    get promptTemplateVariables(): string[];
+}
+//# sourceMappingURL=ClassificationEvaluator.d.ts.map

package/dist/src/llm/ClassificationEvaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iCAAiC,EACjC,WAAW,EACX,QAAQ,EACT,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAG9C;;GAEG;AACH,qBAAa,uBAAuB,CAClC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC1C,SAAQ,YAAY,CAAC,UAAU,CAAC;IAChC,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC;IAC9C,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC;IAClC,OAAO,CAAC,wBAAwB,CAAuB;gBAC3C,IAAI,EAAE,iCAAiC;IAKnD,QAAQ,GAAI,SAAS,UAAU,kDAE7B;IACF;;OAEG;IACH,IAAI,uBAAuB,IAAI,MAAM,EAAE,CAStC;CACF"}

package/dist/src/llm/ClassificationEvaluator.js

@@ -0,0 +1,34 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ClassificationEvaluator = void 0;
+const createClassifierFn_1 = require("./createClassifierFn");
+const LLMEvaluator_1 = require("./LLMEvaluator");
+const template_1 = require("../template");
+/**
+ * An LLM evaluator that performs evaluation via classification
+ */
+class ClassificationEvaluator extends LLMEvaluator_1.LLMEvaluator {
+    constructor(args) {
+        super(args);
+        this.evaluate = (example) => {
+            return this.evaluatorFn(example);
+        };
+        this.promptTemplate = args.promptTemplate;
+        this.evaluatorFn = (0, createClassifierFn_1.createClassifierFn)(args);
+    }
+    /**
+     * List out the prompt template variables needed to perform evaluation
+     */
+    get promptTemplateVariables() {
+        // Use dynamic programming to see if it's computed already
+        if (!Array.isArray(this._promptTemplateVariables)) {
+            this._promptTemplateVariables = (0, template_1.getTemplateVariables)({
+                template: this.promptTemplate,
+            });
+        }
+        // Give a copy of the variables
+        return [...this._promptTemplateVariables];
+    }
+}
+exports.ClassificationEvaluator = ClassificationEvaluator;
+//# sourceMappingURL=ClassificationEvaluator.js.map

package/dist/src/llm/ClassificationEvaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":";;;AAKA,6DAA0D;AAC1D,iDAA8C;AAC9C,0CAAmD;AAEnD;;GAEG;AACH,MAAa,uBAEX,SAAQ,2BAAwB;IAIhC,YAAY,IAAuC;QACjD,KAAK,CAAC,IAAI,CAAC,CAAC;QAId,aAAQ,GAAG,CAAC,OAAmB,EAAE,EAAE;YACjC,OAAO,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QACnC,CAAC,CAAC;QALA,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,IAAA,uCAAkB,EAAa,IAAI,CAAC,CAAC;IAC1D,CAAC;IAID;;OAEG;IACH,IAAI,uBAAuB;QACzB,0DAA0D;QAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,wBAAwB,GAAG,IAAA,+BAAoB,EAAC;gBACnD,QAAQ,EAAE,IAAI,CAAC,cAAc;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,+BAA+B;QAC/B,OAAO,CAAC,GAAG,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC5C,CAAC;CACF;AA3BD,0DA2BC"}

package/dist/src/llm/LLMEvaluator.d.ts

@@ -0,0 +1,12 @@
+import { EvaluationResult, Evaluator, OptimizationDirection, CreateEvaluatorArgs } from "../types";
+/**
+ * Base class for llm evaluation metrics / scores
+ */
+export declare abstract class LLMEvaluator<RecordType extends Record<string, unknown>> implements Evaluator<RecordType> {
+    readonly name: string;
+    readonly source: "LLM";
+    readonly optimizationDirection?: OptimizationDirection;
+    constructor({ name, optimizationDirection }: CreateEvaluatorArgs);
+    abstract evaluate(_example: RecordType): Promise<EvaluationResult>;
+}
+//# sourceMappingURL=LLMEvaluator.d.ts.map

package/dist/src/llm/LLMEvaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LLMEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/LLMEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gBAAgB,EAChB,SAAS,EACT,qBAAqB,EACrB,mBAAmB,EACpB,MAAM,UAAU,CAAC;AAElB;;GAEG;AACH,8BAAsB,YAAY,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC3E,YAAW,SAAS,CAAC,UAAU,CAAC;IAEhC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAG,KAAK,CAAU;IACjC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;gBAC3C,EAAE,IAAI,EAAE,qBAAqB,EAAE,EAAE,mBAAmB;IAIhE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;CACnE"}

package/dist/src/llm/LLMEvaluator.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LLMEvaluator = void 0;
+/**
+ * Base class for llm evaluation metrics / scores
+ */
+class LLMEvaluator {
+    constructor({ name, optimizationDirection }) {
+        this.source = "LLM";
+        this.name = name;
+        this.optimizationDirection = optimizationDirection;
+    }
+}
+exports.LLMEvaluator = LLMEvaluator;
+//# sourceMappingURL=LLMEvaluator.js.map

package/dist/src/llm/LLMEvaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LLMEvaluator.js","sourceRoot":"","sources":["../../../src/llm/LLMEvaluator.ts"],"names":[],"mappings":";;;AAOA;;GAEG;AACH,MAAsB,YAAY;IAMhC,YAAY,EAAE,IAAI,EAAE,qBAAqB,EAAuB;QAFvD,WAAM,GAAG,KAAc,CAAC;QAG/B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;IACrD,CAAC;CAEF;AAXD,oCAWC"}

package/dist/src/llm/createClassificationEvaluator.d.ts

@@ -1,3 +1,4 @@
-import { CreateClassificationEvaluatorArgs, Evaluator } from "../types/evals";
-export declare function createClassificationEvaluator<ExampleType extends Record<string, unknown>>(args: CreateClassificationEvaluatorArgs): Evaluator<ExampleType>;
+import { CreateClassificationEvaluatorArgs } from "../types/evals";
+import { ClassificationEvaluator } from "./ClassificationEvaluator";
+export declare function createClassificationEvaluator<RecordType extends Record<string, unknown>>(args: CreateClassificationEvaluatorArgs): ClassificationEvaluator<RecordType>;
 //# sourceMappingURL=createClassificationEvaluator.d.ts.map

package/dist/src/llm/createClassificationEvaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAG9E,wBAAgB,6BAA6B,CAC3C,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC3C,IAAI,EAAE,iCAAiC,GAAG,SAAS,CAAC,WAAW,CAAC,CAOjE"}
+{"version":3,"file":"createClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,wBAAgB,6BAA6B,CAC3C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAE1C,IAAI,EAAE,iCAAiC,GACtC,uBAAuB,CAAC,UAAU,CAAC,CAErC"}

package/dist/src/llm/createClassificationEvaluator.js

@@ -1,13 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createClassificationEvaluator = createClassificationEvaluator;
-const createClassifierFn_1 = require("./createClassifierFn");
+const ClassificationEvaluator_1 = require("./ClassificationEvaluator");
 function createClassificationEvaluator(args) {
-    return {
-        name: args.name,
-        source: "LLM",
-        optimizationDirection: args.optimizationDirection,
-        evaluate: (0, createClassifierFn_1.createClassifierFn)(args),
-    };
+    return new ClassificationEvaluator_1.ClassificationEvaluator(args);
 }
 //# sourceMappingURL=createClassificationEvaluator.js.map

package/dist/src/llm/createClassificationEvaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"createClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":";;AAGA,sEASC;AAXD,6DAA0D;AAE1D,SAAgB,6BAA6B,CAE3C,IAAuC;IACvC,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,MAAM,EAAE,KAAK;QACb,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;QACjD,QAAQ,EAAE,IAAA,uCAAkB,EAAC,IAAI,CAAC;KACnC,CAAC;AACJ,CAAC"}
+{"version":3,"file":"createClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":";;AAGA,sEAMC;AARD,uEAAoE;AAEpE,SAAgB,6BAA6B,CAG3C,IAAuC;IAEvC,OAAO,IAAI,iDAAuB,CAAa,IAAI,CAAC,CAAC;AACvD,CAAC"}

package/dist/src/llm/createClassifierFn.d.ts

@@ -2,5 +2,5 @@ import { CreateClassifierArgs, EvaluatorFn } from "../types/evals";
 /**
  * A function that serves as a factory that will output a classification evaluator function
  */
-export declare function createClassifierFn<ExampleType extends Record<string, unknown>>(args: CreateClassifierArgs): EvaluatorFn<ExampleType>;
+export declare function createClassifierFn<RecordToEvaluate extends Record<string, unknown>>(args: CreateClassifierArgs): EvaluatorFn<RecordToEvaluate>;
 //# sourceMappingURL=createClassifierFn.d.ts.map

package/dist/src/llm/createClassifierFn.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createClassifierFn.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassifierFn.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,oBAAoB,EACpB,WAAW,EACZ,MAAM,gBAAgB,CAAC;AAkBxB;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC5E,IAAI,EAAE,oBAAoB,GACzB,WAAW,CAAC,WAAW,CAAC,CA4B1B"}
+{"version":3,"file":"createClassifierFn.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassifierFn.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,oBAAoB,EACpB,WAAW,EACZ,MAAM,gBAAgB,CAAC;AAkBxB;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,gBAAgB,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChD,IAAI,EAAE,oBAAoB,GAAG,WAAW,CAAC,gBAAgB,CAAC,CA4B3D"}

package/dist/src/llm/createClassifierFn.js.map

@@ -1 +1 @@
-{"version":3,"file":"createClassifierFn.js","sourceRoot":"","sources":["../../../src/llm/createClassifierFn.ts"],"names":[],"mappings":";;;;;;;;;;;;;AA0BA,gDA8BC;AAlDD,qEAAkE;AAClE,0CAA6C;AAE7C;;;GAGG;AACH,SAAS,eAAe,CACtB,OAAiC;IAEjC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,MAA+B,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,SAAgB,kBAAkB,CAChC,IAA0B;IAE1B,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,KAAc,IAAI,EAAb,IAAI,UAAK,IAAI,EAAlD,sCAA2C,CAAO,CAAC;IAEzD,OAAO,KAAK,EAAE,IAAiB,EAA6B,EAAE;QAC5D,MAAM,iBAAiB,qBAClB,IAAI,CACR,CAAC;QAEF,MAAM,MAAM,GAAG,IAAA,yBAAc,EAAC;YAC5B,QAAQ,EAAE,cAAc;YACxB,SAAS,EAAE,iBAAiB;SAC7B,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,MAAM,IAAA,+CAAsB,kBACjD,KAAK,EACL,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,EAChC,MAAM,IACH,IAAI,EACP,CAAC;QAEH,mEAAmE;QACnE,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAE5C,uBACE,KAAK,IACF,cAAc,EACjB;IACJ,CAAC,CAAC;AACJ,CAAC"}
+{"version":3,"file":"createClassifierFn.js","sourceRoot":"","sources":["../../../src/llm/createClassifierFn.ts"],"names":[],"mappings":";;;;;;;;;;;;;AA0BA,gDA8BC;AAlDD,qEAAkE;AAClE,0CAA6C;AAE7C;;;GAGG;AACH,SAAS,eAAe,CACtB,OAAiC;IAEjC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,MAA+B,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,SAAgB,kBAAkB,CAEhC,IAA0B;IAC1B,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,KAAc,IAAI,EAAb,IAAI,UAAK,IAAI,EAAlD,sCAA2C,CAAO,CAAC;IAEzD,OAAO,KAAK,EAAE,IAAsB,EAA6B,EAAE;QACjE,MAAM,iBAAiB,qBAClB,IAAI,CACR,CAAC;QAEF,MAAM,MAAM,GAAG,IAAA,yBAAc,EAAC;YAC5B,QAAQ,EAAE,cAAc;YACxB,SAAS,EAAE,iBAAiB;SAC7B,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,MAAM,IAAA,+CAAsB,kBACjD,KAAK,EACL,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,EAChC,MAAM,IACH,IAAI,EACP,CAAC;QAEH,mEAAmE;QACnE,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAE5C,uBACE,KAAK,IACF,cAAc,EACjB;IACJ,CAAC,CAAC;AACJ,CAAC"}

package/dist/src/llm/createDocumentRelevancyEvaluator.d.ts

@@ -6,12 +6,13 @@ export interface DocumentRelevancyEvaluatorArgs extends Omit<CreateClassificatio
     promptTemplate?: CreateClassificationEvaluatorArgs["promptTemplate"];
 }
 /**
- * An example to be evaluated by the document relevancy evaluator.
+ * A record to be evaluated by the document relevancy evaluator.
  */
-export type DocumentRelevancyExample = {
+export interface DocumentRelevancyEvaluationRecord {
     input: string;
     documentText: string;
-};
+    [key: string]: unknown;
+}
 /**
  * Creates a document relevancy evaluator function.
  *
@@ -38,5 +39,5 @@ export type DocumentRelevancyExample = {
  * console.log(result.label); // "relevant" or "unrelated"
  * ```
  */
-export declare function createDocumentRelevancyEvaluator(args: DocumentRelevancyEvaluatorArgs): Evaluator<DocumentRelevancyExample>;
+export declare function createDocumentRelevancyEvaluator<RecordType extends Record<string, unknown> = DocumentRelevancyEvaluationRecord>(args: DocumentRelevancyEvaluatorArgs): Evaluator<RecordType>;
 //# sourceMappingURL=createDocumentRelevancyEvaluator.d.ts.map

package/dist/src/llm/createDocumentRelevancyEvaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createDocumentRelevancyEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAO9E,MAAM,WAAW,8BACf,SAAQ,IAAI,CACV,iCAAiC,EACjC,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,uBAAuB,CAAC,CAAC;IACnF,IAAI,CAAC,EAAE,iCAAiC,CAAC,MAAM,CAAC,CAAC;IACjD,OAAO,CAAC,EAAE,iCAAiC,CAAC,SAAS,CAAC,CAAC;IACvD,cAAc,CAAC,EAAE,iCAAiC,CAAC,gBAAgB,CAAC,CAAC;CACtE;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,8BAA8B,GACnC,SAAS,CAAC,wBAAwB,CAAC,CAiBrC"}
+{"version":3,"file":"createDocumentRelevancyEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAO9E,MAAM,WAAW,8BACf,SAAQ,IAAI,CACV,iCAAiC,EACjC,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,uBAAuB,CAAC,CAAC;IACnF,IAAI,CAAC,EAAE,iCAAiC,CAAC,MAAM,CAAC,CAAC;IACjD,OAAO,CAAC,EAAE,iCAAiC,CAAC,SAAS,CAAC,CAAC;IACvD,cAAc,CAAC,EAAE,iCAAiC,CAAC,gBAAgB,CAAC,CAAC;CACtE;AAED;;GAEG;AACH,MAAM,WAAW,iCAAiC;IAChD,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,gCAAgC,CAC9C,UAAU,SAAS,MAAM,CACvB,MAAM,EACN,OAAO,CACR,GAAG,iCAAiC,EACrC,IAAI,EAAE,8BAA8B,GAAG,SAAS,CAAC,UAAU,CAAC,CAiB7D"}

package/dist/src/llm/createDocumentRelevancyEvaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"createDocumentRelevancyEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAoDA,4EAmBC;AAtED,kGAG0D;AAC1D,mFAAgF;AAqBhF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAgB,gCAAgC,CAC9C,IAAoC;IAEpC,MAAM,EACJ,OAAO,GAAG,wDAA0B,EACpC,cAAc,GAAG,yDAA2B,EAC5C,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,oBAAoB,KAEzB,IAAI,EADH,IAAI,UACL,IAAI,EANF,8DAML,CAAO,CAAC;IAET,OAAO,IAAA,6DAA6B,gDAC/B,IAAI,KACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI,KACD,IAAI,EACP,CAAC;AACL,CAAC"}
+{"version":3,"file":"createDocumentRelevancyEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAqDA,4EAsBC;AA1ED,kGAG0D;AAC1D,mFAAgF;AAsBhF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAgB,gCAAgC,CAK9C,IAAoC;IACpC,MAAM,EACJ,OAAO,GAAG,wDAA0B,EACpC,cAAc,GAAG,yDAA2B,EAC5C,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,oBAAoB,KAEzB,IAAI,EADH,IAAI,UACL,IAAI,EANF,8DAML,CAAO,CAAC;IAET,OAAO,IAAA,6DAA6B,gDAC/B,IAAI,KACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI,KACD,IAAI,EACP,CAAC;AACL,CAAC"}

package/dist/src/llm/createHallucinationEvaluator.d.ts

@@ -1,4 +1,5 @@
-import { CreateClassificationEvaluatorArgs, Evaluator } from "../types/evals";
+import { CreateClassificationEvaluatorArgs } from "../types/evals";
+import { ClassificationEvaluator } from "./ClassificationEvaluator";
 export interface HallucinationEvaluatorArgs extends Omit<CreateClassificationEvaluatorArgs, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
     optimizationDirection?: CreateClassificationEvaluatorArgs["optimizationDirection"];
     name?: CreateClassificationEvaluatorArgs["name"];
@@ -6,9 +7,9 @@ export interface HallucinationEvaluatorArgs extends Omit<CreateClassificationEva
     promptTemplate?: CreateClassificationEvaluatorArgs["promptTemplate"];
 }
 /**
- * An example to be evaluated by the hallucination evaluator.
+ * A record to be evaluated by the hallucination evaluator.
  */
-export type HallucinationExample = {
+export type HallucinationEvaluationRecord = {
     input: string;
     output: string;
     reference?: string;
@@ -20,5 +21,5 @@ export type HallucinationExample = {
  * @param args - The arguments for creating the hallucination evaluator.
  * @returns A function that evaluates whether an answer is factual or hallucinated based on a query and reference text.
  */
-export declare function createHallucinationEvaluator(args: HallucinationEvaluatorArgs): Evaluator<HallucinationExample>;
+export declare function createHallucinationEvaluator<RecordType extends Record<string, unknown> = HallucinationEvaluationRecord>(args: HallucinationEvaluatorArgs): ClassificationEvaluator<RecordType>;
 //# sourceMappingURL=createHallucinationEvaluator.d.ts.map

package/dist/src/llm/createHallucinationEvaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createHallucinationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAO9E,MAAM,WAAW,0BACf,SAAQ,IAAI,CACV,iCAAiC,EACjC,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,uBAAuB,CAAC,CAAC;IACnF,IAAI,CAAC,EAAE,iCAAiC,CAAC,MAAM,CAAC,CAAC;IACjD,OAAO,CAAC,EAAE,iCAAiC,CAAC,SAAS,CAAC,CAAC;IACvD,cAAc,CAAC,EAAE,iCAAiC,CAAC,gBAAgB,CAAC,CAAC;CACtE;AAED;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AACF;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,0BAA0B,GAC/B,SAAS,CAAC,oBAAoB,CAAC,CAgBjC"}
+{"version":3,"file":"createHallucinationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AAMnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,MAAM,WAAW,0BACf,SAAQ,IAAI,CACV,iCAAiC,EACjC,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,uBAAuB,CAAC,CAAC;IACnF,IAAI,CAAC,EAAE,iCAAiC,CAAC,MAAM,CAAC,CAAC;IACjD,OAAO,CAAC,EAAE,iCAAiC,CAAC,SAAS,CAAC,CAAC;IACvD,cAAc,CAAC,EAAE,iCAAiC,CAAC,gBAAgB,CAAC,CAAC;CACtE;AAED;;GAEG;AACH,MAAM,MAAM,6BAA6B,GAAG;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AACF;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAC1C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,6BAA6B,EAC1E,IAAI,EAAE,0BAA0B,GAAG,uBAAuB,CAAC,UAAU,CAAC,CAgBvE"}

package/dist/src/llm/createHallucinationEvaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"createHallucinationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAiCA,oEAkBC;AAlDD,wFAGqD;AACrD,mFAAgF;AAsBhF;;;;;GAKG;AACH,SAAgB,4BAA4B,CAC1C,IAAgC;IAEhC,MAAM,EACJ,OAAO,GAAG,8CAAqB,EAC/B,cAAc,GAAG,+CAAsB,EACvC,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,eAAe,KAEpB,IAAI,EADH,IAAI,UACL,IAAI,EANF,8DAML,CAAO,CAAC;IACT,OAAO,IAAA,6DAA6B,gDAC/B,IAAI,KACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI,KACD,IAAI,EACP,CAAC;AACL,CAAC"}
+{"version":3,"file":"createHallucinationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAkCA,oEAkBC;AAnDD,wFAGqD;AACrD,mFAAgF;AAuBhF;;;;;GAKG;AACH,SAAgB,4BAA4B,CAE1C,IAAgC;IAChC,MAAM,EACJ,OAAO,GAAG,8CAAqB,EAC/B,cAAc,GAAG,+CAAsB,EACvC,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,eAAe,KAEpB,IAAI,EADH,IAAI,UACL,IAAI,EANF,8DAML,CAAO,CAAC;IACT,OAAO,IAAA,6DAA6B,gDAC/B,IAAI,KACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI,KACD,IAAI,EACP,CAAC;AACL,CAAC"}

package/dist/src/llm/generateClassification.d.ts

@@ -1,7 +1,7 @@
 import { ClassificationResult, WithLLM } from "../types/evals";
 import { WithTelemetry } from "../types/otel";
 import type { WithPrompt } from "../types/prompts";
-export interface ClassifyArgs extends WithLLM, WithPrompt, WithTelemetry {
+export type ClassifyArgs = WithLLM & WithTelemetry & WithPrompt & {
     /**
      * The labels to classify the example into. E.x. ["correct", "incorrect"]
      */
@@ -14,7 +14,7 @@ export interface ClassifyArgs extends WithLLM, WithPrompt, WithTelemetry {
      * The description of the schema for generating the label and explanation.
      */
     schemaDescription?: string;
-}
+};
 /**
  * A function that leverages an llm to perform a classification
  */

package/dist/src/llm/generateClassification.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"generateClassification.d.ts","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAInD,MAAM,WAAW,YAAa,SAAQ,OAAO,EAAE,UAAU,EAAE,aAAa;IACtE;;OAEG;IACH,MAAM,EAAE,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IAC9B;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AACD;;GAEG;AACH,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,YAAY,GACjB,OAAO,CAAC,oBAAoB,CAAC,CAyB/B"}
+{"version":3,"file":"generateClassification.d.ts","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAInD,MAAM,MAAM,YAAY,GAAG,OAAO,GAChC,aAAa,GACb,UAAU,GAAG;IACX;;OAEG;IACH,MAAM,EAAE,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IAC9B;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,CAAC;AACJ;;GAEG;AACH,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,YAAY,GACjB,OAAO,CAAC,oBAAoB,CAAC,CAyB/B"}

package/dist/src/llm/generateClassification.js.map

@@ -1 +1 @@
-{"version":3,"file":"generateClassification.js","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAuBA,wDA2BC;AA/CD,2BAAoC;AACpC,6BAAwB;AACxB,4CAAsC;AAetC;;GAEG;AACI,KAAK,UAAU,sBAAsB,CAC1C,IAAkB;;IAElB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,KAC7D,IAAI,EAD8D,MAAM,UACxE,IAAI,EADA,mEAAsE,CACtE,CAAC;IAEP,MAAM,sBAAsB,GAAG;QAC7B,SAAS,EAAE,MAAA,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,SAAS,mCAAI,IAAI;QACvC,UAAU,EAAE,wBAAwB;QACpC,MAAM,EAAE,MAAA,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,MAAM,mCAAI,kBAAM;KACpC,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,IAAA,mBAAc,kBACjC,KAAK;QACL,UAAU;QACV,iBAAiB,EACjB,MAAM,EAAE,OAAC,CAAC,MAAM,CAAC;YACf,WAAW,EAAE,OAAC,CAAC,MAAM,EAAE,EAAE,4EAA4E;YACrG,KAAK,EAAE,OAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SACtB,CAAC,EACF,sBAAsB,IACnB,MAAM,EACT,CAAC;IACH,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;QAC1B,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;KACvC,CAAC;AACJ,CAAC"}
+{"version":3,"file":"generateClassification.js","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAyBA,wDA2BC;AAjDD,2BAAoC;AACpC,6BAAwB;AACxB,4CAAsC;AAiBtC;;GAEG;AACI,KAAK,UAAU,sBAAsB,CAC1C,IAAkB;;IAElB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,KAC7D,IAAI,EAD8D,MAAM,UACxE,IAAI,EADA,mEAAsE,CACtE,CAAC;IAEP,MAAM,sBAAsB,GAAG;QAC7B,SAAS,EAAE,MAAA,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,SAAS,mCAAI,IAAI;QACvC,UAAU,EAAE,wBAAwB;QACpC,MAAM,EAAE,MAAA,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,MAAM,mCAAI,kBAAM;KACpC,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,IAAA,mBAAc,kBACjC,KAAK;QACL,UAAU;QACV,iBAAiB,EACjB,MAAM,EAAE,OAAC,CAAC,MAAM,CAAC;YACf,WAAW,EAAE,OAAC,CAAC,MAAM,EAAE,EAAE,4EAA4E;YACrG,KAAK,EAAE,OAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SACtB,CAAC,EACF,sBAAsB,IACnB,MAAM,EACT,CAAC;IACH,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;QAC1B,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;KACvC,CAAC;AACJ,CAAC"}

package/dist/src/template/getTemplateVariables.d.ts

@@ -0,0 +1,12 @@
+import { Template } from "../types/templating";
+type GetTemplateVariableArgs = {
+    template: Template;
+};
+/**
+ * Parse out the template variables of a prompt
+ * @param {GetTemplateVariableArgs} args
+ * @returns {string[]} a list of prompt template variables
+ */
+export declare function getTemplateVariables(args: GetTemplateVariableArgs): string[];
+export {};
+//# sourceMappingURL=getTemplateVariables.d.ts.map

package/dist/src/template/getTemplateVariables.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"getTemplateVariables.d.ts","sourceRoot":"","sources":["../../../src/template/getTemplateVariables.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAG/C,KAAK,uBAAuB,GAAG;IAC7B,QAAQ,EAAE,QAAQ,CAAC;CACpB,CAAC;AACF;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,uBAAuB,GAAG,MAAM,EAAE,CAU5E"}

package/dist/src/template/getTemplateVariables.js

@@ -0,0 +1,24 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getTemplateVariables = getTemplateVariables;
+const mustache_1 = __importDefault(require("mustache"));
+/**
+ * Parse out the template variables of a prompt
+ * @param {GetTemplateVariableArgs} args
+ * @returns {string[]} a list of prompt template variables
+ */
+function getTemplateVariables(args) {
+    const { template } = args;
+    const templateSpans = mustache_1.default.parse(template);
+    return templateSpans.reduce((acc, templateSpan) => {
+        const [spanType, value] = templateSpan;
+        if (spanType === "name" && typeof value === "string") {
+            acc = [...acc, value];
+        }
+        return acc;
+    }, []);
+}
+//# sourceMappingURL=getTemplateVariables.js.map

package/dist/src/template/getTemplateVariables.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getTemplateVariables.js","sourceRoot":"","sources":["../../../src/template/getTemplateVariables.ts"],"names":[],"mappings":";;;;;AAWA,oDAUC;AApBD,wDAAgC;AAKhC;;;;GAIG;AACH,SAAgB,oBAAoB,CAAC,IAA6B;IAChE,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAC1B,MAAM,aAAa,GAAG,kBAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,YAAY,EAAE,EAAE;QAChD,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,GAAG,YAAY,CAAC;QACvC,IAAI,QAAQ,KAAK,MAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACrD,GAAG,GAAG,CAAC,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;QACxB,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EAAE,EAAc,CAAC,CAAC;AACrB,CAAC"}

package/dist/src/template/index.d.ts

@@ -1,2 +1,3 @@
 export * from "./applyTemplate";
+export * from "./getTemplateVariables";
 //# sourceMappingURL=index.d.ts.map

package/dist/src/template/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/template/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/template/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,wBAAwB,CAAC"}

package/dist/src/template/index.js

@@ -15,4 +15,5 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./applyTemplate"), exports);
+__exportStar(require("./getTemplateVariables"), exports);
 //# sourceMappingURL=index.js.map

package/dist/src/template/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/template/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,kDAAgC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/template/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,kDAAgC;AAChC,yDAAuC"}

package/dist/src/types/evals.d.ts

@@ -67,7 +67,7 @@ export interface CreateClassifierArgs extends WithTelemetry {
      */
     promptTemplate: string;
 }
-export interface CreateClassificationEvaluatorArgs extends CreateClassifierArgs {
+export interface CreateEvaluatorArgs {
     /**
      * The name of the metric that the evaluator produces
      * E.x. "correctness"
@@ -79,21 +79,22 @@ export interface CreateClassificationEvaluatorArgs extends CreateClassifierArgs
      */
     optimizationDirection?: OptimizationDirection;
 }
+export interface CreateClassificationEvaluatorArgs extends CreateClassifierArgs, CreateEvaluatorArgs {
+}
 export type EvaluatorFn<ExampleType extends Record<string, unknown>> = (args: ExampleType) => Promise<EvaluationResult>;
 /**
  * The source of the evaluation
  */
-type EvaluationSource = "LLM" | "CODE";
+export type EvaluationSource = "LLM" | "CODE";
 /**
  * The direction to optimize the numeric evaluation score
  * E.x. "MAXIMIZE" means that the higher the score, the better the evaluation
  */
-type OptimizationDirection = "MAXIMIZE" | "MINIMIZE";
+export type OptimizationDirection = "MAXIMIZE" | "MINIMIZE";
 /**
- * The Base Evaluator interface
- * This is the interface that all evaluators must implement
+ * The description of an evaluator
  */
-export interface Evaluator<ExampleType extends Record<string, unknown>> {
+interface EvaluatorDescription {
     /**
      * The name of the evaluator / the metric that it measures
      */
@@ -107,6 +108,12 @@ export interface Evaluator<ExampleType extends Record<string, unknown>> {
      * E.x. "MAXIMIZE" means that the higher the score, the better the evaluation
      */
     optimizationDirection?: OptimizationDirection;
+}
+/**
+ * The Base Evaluator interface
+ * This is the interface that all evaluators must implement
+ */
+export interface Evaluator<ExampleType extends Record<string, unknown>> extends EvaluatorDescription {
     /**
      * The function that evaluates the example
      */

package/dist/src/types/evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAEvC;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,aAAa,CAAC;CACtB;AAED,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,iCACf,SAAQ,oBAAoB;IAC5B;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B;;GAEG;AACH,KAAK,gBAAgB,GAAG,KAAK,GAAG,MAAM,CAAC;AAEvC;;;GAGG;AACH,KAAK,qBAAqB,GAAG,UAAU,GAAG,UAAU,CAAC;AAErD;;;GAGG;AACH,MAAM,WAAW,SAAS,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IACpE;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,MAAM,EAAE,gBAAgB,CAAC;IACzB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;OAEG;IACH,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;CACpC"}
+{"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAEvC;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,aAAa,CAAC;CACtB;AAGD,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,mBAAmB;IAClC;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED,MAAM,WAAW,iCACf,SAAQ,oBAAoB,EAC1B,mBAAmB;CAAG;AAE1B,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,KAAK,GAAG,MAAM,CAAC;AAE9C;;;GAGG;AACH,MAAM,MAAM,qBAAqB,GAAG,UAAU,GAAG,UAAU,CAAC;AAE5D;;GAEG;AACH,UAAU,oBAAoB;IAC5B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,MAAM,EAAE,gBAAgB,CAAC;IACzB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CACpE,SAAQ,oBAAoB;IAC5B;;OAEG;IACH,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;CACpC"}

package/dist/src/types/otel.d.ts

@@ -1,5 +1,5 @@
 import { Tracer } from "@opentelemetry/api";
-export interface WithTelemetry {
+export type WithTelemetry = {
     telemetry?: {
         /**
          * Whether OpenTelemetry is enabled on the call.
@@ -13,5 +13,5 @@ export interface WithTelemetry {
          */
         tracer?: Tracer;
     };
-}
+};
 //# sourceMappingURL=otel.d.ts.map

package/dist/src/types/otel.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"otel.d.ts","sourceRoot":"","sources":["../../../src/types/otel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,MAAM,WAAW,aAAa;IAC5B,SAAS,CAAC,EAAE;QACV;;;;WAIG;QACH,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB;;;WAGG;QACH,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACH"}
+{"version":3,"file":"otel.d.ts","sourceRoot":"","sources":["../../../src/types/otel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,CAAC,EAAE;QACV;;;;WAIG;QACH,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB;;;WAGG;QACH,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACH,CAAC"}

package/dist/src/types/prompts.d.ts

@@ -1,22 +1,8 @@
-import { ModelMessage } from "ai";
+import type { Prompt } from "ai";
 /**
  * Prompt part of the AI function options for model generation.
  * It contains a system message, a simple text prompt, or a list of model messages.
  * Uses ModelMessage format compatible with AI SDK v5 generateObject function.
  */
-export interface WithPrompt {
-    /**
-     * System message to include in the prompt. Can be used with `prompt` or `messages`.
-     */
-    system?: string;
-    /**
-     * A simple text prompt. You can either use `prompt` or `messages` but not both.
-     */
-    prompt?: string;
-    /**
-     * A list of model messages. You can either use `prompt` or `messages` but not both.
-     * Uses ModelMessage format for compatibility with AI SDK v5.
-     */
-    messages?: Array<ModelMessage>;
-}
+export type WithPrompt = Prompt;
 //# sourceMappingURL=prompts.d.ts.map

package/dist/src/types/prompts.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../../../src/types/prompts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAElC;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,QAAQ,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;CAChC"}
+{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../../../src/types/prompts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AACjC;;;;GAIG;AACH,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC"}