@arizeai/phoenix-evals 0.6.0 → 0.6.2

package/dist/esm/utils/typeUtils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"typeUtils.d.ts","sourceRoot":"","sources":["../../../src/utils/typeUtils.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,CAAC,GAAG,OAAO,EAAE,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,OAAO,CAAC,CAAC,CAAC,CAQ1E"}

package/dist/esm/utils/typeUtils.js

@@ -0,0 +1,13 @@
+/**
+ * Type guard for if a function is a Promise
+ * @param value
+ * @returns true if it is a Promise
+ */
+export function isPromise(value) {
+    return (!!value &&
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        typeof value?.then === "function" &&
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        typeof value?.catch === "function");
+}
+//# sourceMappingURL=typeUtils.js.map

package/dist/esm/utils/typeUtils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"typeUtils.js","sourceRoot":"","sources":["../../../src/utils/typeUtils.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,UAAU,SAAS,CAAc,KAAc;IACnD,OAAO,CACL,CAAC,CAAC,KAAK;QACP,8DAA8D;QAC9D,OAAQ,KAAa,EAAE,IAAI,KAAK,UAAU;QAC1C,8DAA8D;QAC9D,OAAQ,KAAa,EAAE,KAAK,KAAK,UAAU,CAC5C,CAAC;AACJ,CAAC"}

package/dist/src/__generated__/default_templates/DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts

@@ -0,0 +1,3 @@
+import type { ClassificationEvaluatorConfig } from "../types";
+export declare const DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG: ClassificationEvaluatorConfig;
+//# sourceMappingURL=DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map

package/dist/src/__generated__/default_templates/DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,kDAAkD,EAAE,6BAkChE,CAAC"}

package/dist/src/__generated__/default_templates/DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js

@@ -0,0 +1,40 @@
+"use strict";
+// This file is generated. Do not edit by hand.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG = void 0;
+exports.DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG = {
+    name: "document_relevance",
+    description: "A specialized evaluator for determining document relevance to a given question.",
+    optimizationDirection: "MAXIMIZE",
+    template: [
+        {
+            role: "user",
+            content: `
+You are comparing a document to a question and trying to determine
+if the document text contains information relevant to answering the
+question. Here is the data:
+
+[BEGIN DATA]
+************
+[Question]: {{input}}
+************
+[Document text]: {{documentText}}
+************
+[END DATA]
+
+Compare the question above to the document text. You must determine
+whether the document text contains information that can answer the
+question. Please focus on whether the very specific question can be
+answered by the information in the document text. Your response must be
+either "relevant" or "unrelated". "unrelated" means that the document
+text does not contain an answer to the question. "relevant" means the
+document text contains an answer to the question.
+`,
+        },
+    ],
+    choices: {
+        "relevant": 1,
+        "unrelated": 0
+    },
+};
+//# sourceMappingURL=DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js.map

package/dist/src/__generated__/default_templates/DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAIlC,QAAA,kDAAkD,GAAkC;IAC/F,IAAI,EAAE,oBAAoB;IAC1B,WAAW,EAAE,iFAAiF;IAC9F,qBAAqB,EAAE,UAAU;IACjC,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;CAoBd;SACI;KACF;IACD,OAAO,EAAE;QACT,UAAU,EAAE,CAAC;QACb,WAAW,EAAE,CAAC;KACf;CACA,CAAC"}

package/dist/src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts

@@ -0,0 +1,3 @@
+import type { ClassificationEvaluatorConfig } from "../types";
+export declare const HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG: ClassificationEvaluatorConfig;
+//# sourceMappingURL=HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map

package/dist/src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,6CAA6C,EAAE,6BAwC3D,CAAC"}

package/dist/src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js

@@ -0,0 +1,46 @@
+"use strict";
+// This file is generated. Do not edit by hand.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG = void 0;
+exports.HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG = {
+    name: "hallucination",
+    description: "A specialized evaluator for detecting hallucinations in grounded LLM responses.",
+    optimizationDirection: "MINIMIZE",
+    template: [
+        {
+            role: "user",
+            content: `
+In this task, you will be presented with a query, some context and a response. The response
+is generated to the question based on the context. The response may contain false
+information. You must use the context to determine if the response to the question
+contains false information, if the response is a hallucination of facts. Your objective is
+to determine whether the response text contains factual information and is not a
+hallucination. A 'hallucination' refers to a response that is not based on the context or
+assumes information that is not available in the context. Your response should be a single
+word: either 'factual' or 'hallucinated', and it should not include any other text or
+characters. 'hallucinated' indicates that the response provides factually inaccurate
+information to the query based on the context. 'factual' indicates that the response to
+the question is correct relative to the context, and does not contain made up
+information. Please read the query and context carefully before determining your
+response.
+
+[BEGIN DATA]
+************
+[Query]: {{input}}
+************
+[Context]: {{context}}
+************
+[Response]: {{output}}
+************
+[END DATA]
+
+Is the response above factual or hallucinated based on the query and context?
+`,
+        },
+    ],
+    choices: {
+        "hallucinated": 1,
+        "factual": 0
+    },
+};
+//# sourceMappingURL=HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js.map

package/dist/src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAIlC,QAAA,6CAA6C,GAAkC;IAC1F,IAAI,EAAE,eAAe;IACrB,WAAW,EAAE,iFAAiF;IAC9F,qBAAqB,EAAE,UAAU;IACjC,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Bd;SACI;KACF;IACD,OAAO,EAAE;QACT,cAAc,EAAE,CAAC;QACjB,SAAS,EAAE,CAAC;KACb;CACA,CAAC"}

package/dist/src/__generated__/default_templates/index.d.ts

@@ -0,0 +1,3 @@
+export { DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG } from "./DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG";
+export { HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG } from "./HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG";
+//# sourceMappingURL=index.d.ts.map

package/dist/src/__generated__/default_templates/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,kDAAkD,EAAE,MAAM,sDAAsD,CAAC;AAC1H,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC"}

package/dist/src/__generated__/default_templates/index.js

@@ -0,0 +1,9 @@
+"use strict";
+// This file is generated. Do not edit by hand.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG = exports.DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG = void 0;
+var DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG_1 = require("./DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG");
+Object.defineProperty(exports, "DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG", { enumerable: true, get: function () { return DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG_1.DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG; } });
+var HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG_1 = require("./HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG");
+Object.defineProperty(exports, "HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG", { enumerable: true, get: function () { return HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG_1.HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG; } });
+//# sourceMappingURL=index.js.map

package/dist/src/__generated__/default_templates/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/index.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAE/C,2HAA0H;AAAjH,wLAAA,kDAAkD,OAAA;AAC3D,iHAAgH;AAAvG,8KAAA,6CAA6C,OAAA"}

package/dist/src/__generated__/types.d.ts

@@ -0,0 +1,9 @@
+import type { PromptTemplate } from "../types/templating";
+export type ClassificationEvaluatorConfig = {
+    name: string;
+    description: string;
+    optimizationDirection: "MINIMIZE" | "MAXIMIZE";
+    template: PromptTemplate;
+    choices: Record<string, number>;
+};
+//# sourceMappingURL=types.d.ts.map

package/dist/src/__generated__/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/__generated__/types.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAE1D,MAAM,MAAM,6BAA6B,GAAG;IAC1C,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,qBAAqB,EAAE,UAAU,GAAG,UAAU,CAAC;IAC/C,QAAQ,EAAE,cAAc,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC,CAAC"}

package/dist/src/__generated__/types.js

@@ -0,0 +1,4 @@
+"use strict";
+// This file is generated. Do not edit by hand.
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/src/__generated__/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/__generated__/types.ts"],"names":[],"mappings":";AAAA,+CAA+C"}

package/dist/src/core/FunctionEvaluator.d.ts

@@ -0,0 +1,16 @@
+import { CreateEvaluatorArgs, EvaluationResult, ObjectMapping } from "../types";
+import { EvaluatorBase } from "./EvaluatorBase";
+type FunctionEvaluatorArgs<RecordType extends Record<string, unknown>> = CreateEvaluatorArgs<RecordType> & {
+    evaluateFn: (args: RecordType) => Promise<EvaluationResult>;
+};
+/**
+ * A class that constructs an evaluator based on an evaluate function.
+ */
+export declare class FunctionEvaluator<RecordType extends Record<string, unknown>> extends EvaluatorBase<RecordType> {
+    readonly evaluateFn: (args: RecordType) => Promise<EvaluationResult>;
+    constructor({ evaluateFn, ...args }: FunctionEvaluatorArgs<RecordType>);
+    evaluate(args: RecordType): Promise<EvaluationResult>;
+    bindInputMapping(inputMapping: ObjectMapping<RecordType>): FunctionEvaluator<RecordType>;
+}
+export {};
+//# sourceMappingURL=FunctionEvaluator.d.ts.map

package/dist/src/core/FunctionEvaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"FunctionEvaluator.d.ts","sourceRoot":"","sources":["../../../src/core/FunctionEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEhF,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD,KAAK,qBAAqB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IACnE,mBAAmB,CAAC,UAAU,CAAC,GAAG;IAChC,UAAU,EAAE,CAAC,IAAI,EAAE,UAAU,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;CAC7D,CAAC;AACJ;;GAEG;AACH,qBAAa,iBAAiB,CAC5B,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC1C,SAAQ,aAAa,CAAC,UAAU,CAAC;IACjC,QAAQ,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,UAAU,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;gBACzD,EAAE,UAAU,EAAE,GAAG,IAAI,EAAE,EAAE,qBAAqB,CAAC,UAAU,CAAC;IAIhE,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAG3D,gBAAgB,CACd,YAAY,EAAE,aAAa,CAAC,UAAU,CAAC,GACtC,iBAAiB,CAAC,UAAU,CAAC;CAGjC"}

package/dist/src/core/FunctionEvaluator.js

@@ -0,0 +1,33 @@
+"use strict";
+var __rest = (this && this.__rest) || function (s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
+            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
+                t[p[i]] = s[p[i]];
+        }
+    return t;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FunctionEvaluator = void 0;
+const EvaluatorBase_1 = require("./EvaluatorBase");
+/**
+ * A class that constructs an evaluator based on an evaluate function.
+ */
+class FunctionEvaluator extends EvaluatorBase_1.EvaluatorBase {
+    constructor(_a) {
+        var { evaluateFn } = _a, args = __rest(_a, ["evaluateFn"]);
+        super(Object.assign({}, args));
+        this.evaluateFn = evaluateFn;
+    }
+    async evaluate(args) {
+        return this.evaluateFn(args);
+    }
+    bindInputMapping(inputMapping) {
+        return new FunctionEvaluator(Object.assign(Object.assign({}, this), { inputMapping }));
+    }
+}
+exports.FunctionEvaluator = FunctionEvaluator;
+//# sourceMappingURL=FunctionEvaluator.js.map

package/dist/src/core/FunctionEvaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"FunctionEvaluator.js","sourceRoot":"","sources":["../../../src/core/FunctionEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAEA,mDAAgD;AAMhD;;GAEG;AACH,MAAa,iBAEX,SAAQ,6BAAyB;IAEjC,YAAY,EAA0D;YAA1D,EAAE,UAAU,OAA8C,EAAzC,IAAI,cAArB,cAAuB,CAAF;QAC/B,KAAK,mBAAM,IAAI,EAAG,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IACD,KAAK,CAAC,QAAQ,CAAC,IAAgB;QAC7B,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IACD,gBAAgB,CACd,YAAuC;QAEvC,OAAO,IAAI,iBAAiB,iCAAM,IAAI,KAAE,YAAY,IAAG,CAAC;IAC1D,CAAC;CACF;AAhBD,8CAgBC"}

package/dist/src/helpers/asEvaluatorFn.d.ts

@@ -0,0 +1,6 @@
+import { AnyFn, EvaluatorFn } from "../types";
+/**
+ * A function that converts a generic function into an evaluator function
+ */
+export declare function asEvaluatorFn<RecordType extends Record<string, unknown>>(fn: AnyFn): EvaluatorFn<RecordType>;
+//# sourceMappingURL=asEvaluatorFn.d.ts.map

package/dist/src/helpers/asEvaluatorFn.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"asEvaluatorFn.d.ts","sourceRoot":"","sources":["../../../src/helpers/asEvaluatorFn.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAK9C;;GAEG;AACH,wBAAgB,aAAa,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACtE,EAAE,EAAE,KAAK,GACR,WAAW,CAAC,UAAU,CAAC,CAQzB"}

package/dist/src/helpers/asEvaluatorFn.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.asEvaluatorFn = asEvaluatorFn;
+const typeUtils_1 = require("../utils/typeUtils");
+const toEvaluationResult_1 = require("./toEvaluationResult");
+/**
+ * A function that converts a generic function into an evaluator function
+ */
+function asEvaluatorFn(fn) {
+    return async (...args) => {
+        let result = fn(...args);
+        if ((0, typeUtils_1.isPromise)(result)) {
+            result = await result;
+        }
+        return (0, toEvaluationResult_1.toEvaluationResult)(result);
+    };
+}
+//# sourceMappingURL=asEvaluatorFn.js.map

package/dist/src/helpers/asEvaluatorFn.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"asEvaluatorFn.js","sourceRoot":"","sources":["../../../src/helpers/asEvaluatorFn.ts"],"names":[],"mappings":";;AAQA,sCAUC;AAjBD,kDAA+C;AAE/C,6DAA0D;AAE1D;;GAEG;AACH,SAAgB,aAAa,CAC3B,EAAS;IAET,OAAO,KAAK,EAAE,GAAG,IAAI,EAAE,EAAE;QACvB,IAAI,MAAM,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QACzB,IAAI,IAAA,qBAAS,EAAC,MAAM,CAAC,EAAE,CAAC;YACtB,MAAM,GAAG,MAAM,MAAM,CAAC;QACxB,CAAC;QACD,OAAO,IAAA,uCAAkB,EAAC,MAAM,CAAC,CAAC;IACpC,CAAC,CAAC;AACJ,CAAC"}

package/dist/src/helpers/createEvaluator.d.ts

@@ -0,0 +1,141 @@
+import { EvaluatorBase } from "../core/EvaluatorBase";
+import { EvaluationKind, OptimizationDirection, TelemetryConfig } from "../types";
+type AnyFn = (...args: any[]) => any;
+/**
+ * Options for creating a custom evaluator using {@link CreateEvaluator}.
+ *
+ * @public
+ */
+export type CreateEvaluatorOptions = {
+    /**
+     * The name of the evaluator / metric that it measures.
+     *
+     * If not provided, the function will attempt to infer the name from the function's `name` property.
+     * If the function has no name, a unique name will be generated.
+     *
+     * @example
+     * ```typescript
+     * const evaluator = CreateEvaluator(myFunction, { name: "custom-metric" });
+     * ```
+     */
+    name?: string;
+    /**
+     * The kind of the evaluation.
+     *
+     * - `"CODE"`: Code-based evaluator that runs custom logic
+     * - `"LLM"`: LLM-based evaluator that uses a language model
+     *
+     * @defaultValue `"CODE"`
+     *
+     * @example
+     * ```typescript
+     * const evaluator = CreateEvaluator(myFunction, { kind: "CODE" });
+     * ```
+     */
+    kind?: EvaluationKind;
+    /**
+     * The direction to optimize the numeric evaluation score.
+     *
+     * - `"MAXIMIZE"`: Higher scores are better (e.g., accuracy, F1 score)
+     * - `"MINIMIZE"`: Lower scores are better (e.g., error rate, latency)
+     *
+     * @defaultValue `"MAXIMIZE"`
+     *
+     * @example
+     * ```typescript
+     * const evaluator = CreateEvaluator(myFunction, {
+     *   optimizationDirection: "MAXIMIZE"
+     * });
+     * ```
+     */
+    optimizationDirection?: OptimizationDirection;
+    /**
+     * The telemetry configuration for the evaluator.
+     *
+     * When enabled, the evaluator will automatically create OpenTelemetry spans
+     * for tracing and observability. This allows you to track evaluator performance
+     * and debug issues in distributed systems.
+     *
+     * @defaultValue `{ isEnabled: true }`
+     *
+     * @example
+     * ```typescript
+     * const evaluator = CreateEvaluator(myFunction, {
+     *   telemetry: { isEnabled: true, tracer: myTracer }
+     * });
+     * ```
+     */
+    telemetry?: TelemetryConfig;
+};
+/**
+ * A factory function for creating a custom evaluator from any function.
+ *
+ * This function wraps a user-provided function into an evaluator that can be used
+ * with Phoenix experiments and evaluations. The function can be synchronous or
+ * asynchronous, and can return a number, an {@link EvaluationResult} object, or
+ * a value that will be automatically converted to an evaluation result.
+ *
+ * The evaluator will automatically:
+ * - Convert the function's return value to an {@link EvaluationResult}
+ * - Handle both sync and async functions
+ * - Wrap the function with OpenTelemetry spans if telemetry is enabled
+ * - Infer the evaluator name from the function name if not provided
+ *
+ * @typeParam RecordType - The type of the input record that the evaluator expects.
+ *   Must extend `Record<string, unknown>`.
+ * @typeParam Fn - The type of the function being wrapped. Must be a function that
+ *   accepts the record type and returns a value compatible with {@link EvaluationResult}.
+ *
+ * @param fn - The function to wrap as an evaluator. Can be synchronous or asynchronous.
+ *   The function should accept a record of type `RecordType` and return either:
+ *   - A number (will be converted to `{ score: number }`)
+ *   - An {@link EvaluationResult} object
+ *   - Any value that can be converted to an evaluation result
+ *
+ * @param options - Optional configuration for the evaluator. See {@link CreateEvaluatorOptions}
+ *   for details on available options.
+ *
+ * @returns An {@link EvaluatorInterface} that can be used with Phoenix experiments
+ *   and evaluation workflows.
+ *
+ * @example
+ * Basic usage with a simple scoring function:
+ * ```typescript
+ * const accuracyEvaluator = CreateEvaluator(
+ *   ({ output, expected }) => {
+ *     return output === expected ? 1 : 0;
+ *   },
+ *   {
+ *     name: "accuracy",
+ *     kind: "CODE",
+ *     optimizationDirection: "MAXIMIZE"
+ *   }
+ * );
+ *
+ * const result = await accuracyEvaluator.evaluate({
+ *   output: "correct answer",
+ *   expected: "correct answer"
+ * });
+ * // result: { score: 1 }
+ * ```
+ *
+ *
+ * @example
+ * Returning a full EvaluationResult:
+ * ```typescript
+ * const qualityEvaluator = CreateEvaluator(
+ *   ({ output }) => {
+ *     const score = calculateQuality(output);
+ *     return {
+ *       score,
+ *       label: score > 0.8 ? "high" : "low",
+ *       explanation: `Quality score: ${score}`
+ *     };
+ *   },
+ *   { name: "quality" }
+ * );
+ * ```
+ */
+export declare function createEvaluator<RecordType extends Record<string, unknown> = Record<string, unknown>, Fn extends AnyFn = AnyFn>(fn: Fn, options?: CreateEvaluatorOptions): EvaluatorBase<RecordType>;
+export {};
+//# sourceMappingURL=createEvaluator.d.ts.map

package/dist/src/helpers/createEvaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"createEvaluator.d.ts","sourceRoot":"","sources":["../../../src/helpers/createEvaluator.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAEtD,OAAO,EACL,cAAc,EACd,qBAAqB,EACrB,eAAe,EAChB,MAAM,UAAU,CAAC;AAKlB,KAAK,KAAK,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC;AAMrC;;;;GAIG;AACH,MAAM,MAAM,sBAAsB,GAAG;IACnC;;;;;;;;;;OAUG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;;;;;;;;;OAYG;IACH,IAAI,CAAC,EAAE,cAAc,CAAC;IACtB;;;;;;;;;;;;;;OAcG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;;;;;;;;;;;;;;OAeG;IACH,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AACH,wBAAgB,eAAe,CAC7B,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACpE,EAAE,SAAS,KAAK,GAAG,KAAK,EACxB,EAAE,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,aAAa,CAAC,UAAU,CAAC,CAyBrE"}

package/dist/src/helpers/createEvaluator.js

@@ -0,0 +1,99 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createEvaluator = createEvaluator;
+const openinference_core_1 = require("@arizeai/openinference-core");
+const FunctionEvaluator_1 = require("../core/FunctionEvaluator");
+const asEvaluatorFn_1 = require("./asEvaluatorFn");
+function generateUniqueName() {
+    return `evaluator-${Math.random().toString(36).substring(2, 15)}`;
+}
+/**
+ * A factory function for creating a custom evaluator from any function.
+ *
+ * This function wraps a user-provided function into an evaluator that can be used
+ * with Phoenix experiments and evaluations. The function can be synchronous or
+ * asynchronous, and can return a number, an {@link EvaluationResult} object, or
+ * a value that will be automatically converted to an evaluation result.
+ *
+ * The evaluator will automatically:
+ * - Convert the function's return value to an {@link EvaluationResult}
+ * - Handle both sync and async functions
+ * - Wrap the function with OpenTelemetry spans if telemetry is enabled
+ * - Infer the evaluator name from the function name if not provided
+ *
+ * @typeParam RecordType - The type of the input record that the evaluator expects.
+ *   Must extend `Record<string, unknown>`.
+ * @typeParam Fn - The type of the function being wrapped. Must be a function that
+ *   accepts the record type and returns a value compatible with {@link EvaluationResult}.
+ *
+ * @param fn - The function to wrap as an evaluator. Can be synchronous or asynchronous.
+ *   The function should accept a record of type `RecordType` and return either:
+ *   - A number (will be converted to `{ score: number }`)
+ *   - An {@link EvaluationResult} object
+ *   - Any value that can be converted to an evaluation result
+ *
+ * @param options - Optional configuration for the evaluator. See {@link CreateEvaluatorOptions}
+ *   for details on available options.
+ *
+ * @returns An {@link EvaluatorInterface} that can be used with Phoenix experiments
+ *   and evaluation workflows.
+ *
+ * @example
+ * Basic usage with a simple scoring function:
+ * ```typescript
+ * const accuracyEvaluator = CreateEvaluator(
+ *   ({ output, expected }) => {
+ *     return output === expected ? 1 : 0;
+ *   },
+ *   {
+ *     name: "accuracy",
+ *     kind: "CODE",
+ *     optimizationDirection: "MAXIMIZE"
+ *   }
+ * );
+ *
+ * const result = await accuracyEvaluator.evaluate({
+ *   output: "correct answer",
+ *   expected: "correct answer"
+ * });
+ * // result: { score: 1 }
+ * ```
+ *
+ *
+ * @example
+ * Returning a full EvaluationResult:
+ * ```typescript
+ * const qualityEvaluator = CreateEvaluator(
+ *   ({ output }) => {
+ *     const score = calculateQuality(output);
+ *     return {
+ *       score,
+ *       label: score > 0.8 ? "high" : "low",
+ *       explanation: `Quality score: ${score}`
+ *     };
+ *   },
+ *   { name: "quality" }
+ * );
+ * ```
+ */
+function createEvaluator(fn, options) {
+    const { name, kind, optimizationDirection, telemetry = { isEnabled: true }, } = options || {};
+    const evaluatorName = name || fn.name || generateUniqueName();
+    let evaluateFn = (0, asEvaluatorFn_1.asEvaluatorFn)(fn);
+    // Add OpenTelemetry span wrapping if telemetry is enabled
+    if (telemetry && telemetry.isEnabled) {
+        evaluateFn = (0, openinference_core_1.withSpan)(evaluateFn, {
+            tracer: telemetry.tracer,
+            name: evaluatorName,
+            kind: "EVALUATOR",
+        });
+    }
+    return new FunctionEvaluator_1.FunctionEvaluator({
+        evaluateFn,
+        name: evaluatorName,
+        kind: kind || "CODE",
+        optimizationDirection: optimizationDirection || "MAXIMIZE",
+        telemetry,
+    });
+}
+//# sourceMappingURL=createEvaluator.js.map

package/dist/src/helpers/createEvaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createEvaluator.js","sourceRoot":"","sources":["../../../src/helpers/createEvaluator.ts"],"names":[],"mappings":";;AA2JA,0CA4BC;AAvLD,oEAAuD;AAGvD,iEAA8D;AAO9D,mDAAgD;AAKhD,SAAS,kBAAkB;IACzB,OAAO,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;AACpE,CAAC;AAqED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AACH,SAAgB,eAAe,CAG7B,EAAM,EAAE,OAAgC;IACxC,MAAM,EACJ,IAAI,EACJ,IAAI,EACJ,qBAAqB,EACrB,SAAS,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,GAChC,GAAG,OAAO,IAAI,EAAE,CAAC;IAClB,MAAM,aAAa,GAAG,IAAI,IAAI,EAAE,CAAC,IAAI,IAAI,kBAAkB,EAAE,CAAC;IAC9D,IAAI,UAAU,GAAG,IAAA,6BAAa,EAAa,EAAE,CAAC,CAAC;IAE/C,0DAA0D;IAC1D,IAAI,SAAS,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;QACrC,UAAU,GAAG,IAAA,6BAAQ,EAAC,UAAU,EAAE;YAChC,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,IAAI,EAAE,aAAa;YACnB,IAAI,EAAE,WAAW;SAClB,CAAC,CAAC;IACL,CAAC;IACD,OAAO,IAAI,qCAAiB,CAAa;QACvC,UAAU;QACV,IAAI,EAAE,aAAa;QACnB,IAAI,EAAE,IAAI,IAAI,MAAM;QACpB,qBAAqB,EAAE,qBAAqB,IAAI,UAAU;QAC1D,SAAS;KACV,CAAC,CAAC;AACL,CAAC"}

package/dist/src/helpers/index.d.ts

@@ -0,0 +1,2 @@
+export * from "./createEvaluator";
+//# sourceMappingURL=index.d.ts.map

package/dist/src/helpers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/helpers/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC"}

package/dist/src/{default_templates → helpers}/index.js

@@ -14,6 +14,5 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./HALLUCINATION_TEMPLATE"), exports);
-__exportStar(require("./DOCUMENT_RELEVANCY_TEMPLATE"), exports);
+__exportStar(require("./createEvaluator"), exports);
 //# sourceMappingURL=index.js.map

package/dist/src/helpers/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/helpers/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,oDAAkC"}

package/dist/src/helpers/toEvaluationResult.d.ts

@@ -0,0 +1,67 @@
+import { EvaluationResult } from "../types";
+/**
+ * Converts an unknown value to an {@link EvaluationResult}.
+ *
+ * This function provides a flexible way to normalize various return types from
+ * evaluator functions into a standardized `EvaluationResult` format. It handles
+ * multiple input types:
+ *
+ * - **Numbers**: Converted to `{ score: number }`
+ * - **Strings**: Converted to `{ label: string }`
+ * - **Objects**: Extracts `score`, `label`, and `explanation` properties if present
+ * - **Other types**: Returns an empty `EvaluationResult` object
+ *
+ * This is particularly useful when creating evaluators from functions that may
+ * return different types, ensuring consistent evaluation result formatting.
+ *
+ * @param result - The value to convert to an EvaluationResult. Can be:
+ *   - A number (converted to score)
+ *   - A string (converted to label)
+ *   - An object with optional `score`, `label`, and/or `explanation` properties
+ *   - Any other value (returns empty object)
+ *
+ * @returns An {@link EvaluationResult} object with extracted properties
+ *
+ * @example
+ * Convert a number to an EvaluationResult:
+ * ```typescript
+ * const result = toEvaluationResult(0.95);
+ * // Returns: { score: 0.95 }
+ * ```
+ *
+ * @example
+ * Convert a string to an EvaluationResult:
+ * ```typescript
+ * const result = toEvaluationResult("correct");
+ * // Returns: { label: "correct" }
+ * ```
+ *
+ * @example
+ * Convert an object with all properties:
+ * ```typescript
+ * const result = toEvaluationResult({
+ *   score: 0.9,
+ *   label: "high",
+ *   explanation: "High quality output"
+ * });
+ * // Returns: { score: 0.9, label: "high", explanation: "High quality output" }
+ * ```
+ *
+ * @example
+ * Convert an object with partial properties:
+ * ```typescript
+ * const result = toEvaluationResult({ score: 0.8 });
+ * // Returns: { score: 0.8 }
+ * ```
+ *
+ * @example
+ * Handle null or undefined:
+ * ```typescript
+ * const result = toEvaluationResult(null);
+ * // Returns: {}
+ * ```
+ *
+ * @public
+ */
+export declare function toEvaluationResult(result: unknown): EvaluationResult;
+//# sourceMappingURL=toEvaluationResult.d.ts.map

package/dist/src/helpers/toEvaluationResult.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"toEvaluationResult.d.ts","sourceRoot":"","sources":["../../../src/helpers/toEvaluationResult.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAuD5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,OAAO,GAAG,gBAAgB,CAyBpE"}