@arizeai/phoenix-evals 0.6.1 → 0.6.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arizeai/phoenix-evals",
-  "version": "0.6.1",
+  "version": "0.6.2",
   "description": "A library for running evaluations for AI use cases",
   "main": "dist/src/index.js",
   "module": "dist/esm/index.js",
@@ -25,6 +25,10 @@
     "./types/*": {
       "import": "./dist/esm/types/*.d.ts",
       "require": "./dist/src/types/*.d.ts"
+    },
+    "./helpers/*": {
+      "import": "./dist/esm/helpers/*.js",
+      "require": "./dist/src/helpers/*.js"
     }
   },
   "files": [
@@ -45,6 +49,7 @@
     "@ai-sdk/google": "^2.0.40",
     "@ai-sdk/openai": "^2.0.0",
     "@arizeai/openinference-instrumentation-openai": "^2.3.0",
+    "@opentelemetry/sdk-trace-node": "^1.30.1",
     "@types/mustache": "^4.2.6",
     "@types/node": "^24.0.10",
     "msw": "^2.11.6",
@@ -58,6 +63,7 @@
     "node": ">=18"
   },
   "dependencies": {
+    "@arizeai/openinference-core": "^2.0.0",
     "@opentelemetry/api": "^1.9.0",
     "ai": "^5.0.0",
     "jsonpath-plus": "^10.3.0",

package/src/core/FunctionEvaluator.ts

@@ -0,0 +1,28 @@
+import { CreateEvaluatorArgs, EvaluationResult, ObjectMapping } from "../types";
+
+import { EvaluatorBase } from "./EvaluatorBase";
+
+type FunctionEvaluatorArgs<RecordType extends Record<string, unknown>> =
+  CreateEvaluatorArgs<RecordType> & {
+    evaluateFn: (args: RecordType) => Promise<EvaluationResult>;
+  };
+/**
+ * A class that constructs an evaluator based on an evaluate function.
+ */
+export class FunctionEvaluator<
+  RecordType extends Record<string, unknown>,
+> extends EvaluatorBase<RecordType> {
+  readonly evaluateFn: (args: RecordType) => Promise<EvaluationResult>;
+  constructor({ evaluateFn, ...args }: FunctionEvaluatorArgs<RecordType>) {
+    super({ ...args });
+    this.evaluateFn = evaluateFn;
+  }
+  async evaluate(args: RecordType): Promise<EvaluationResult> {
+    return this.evaluateFn(args);
+  }
+  bindInputMapping(
+    inputMapping: ObjectMapping<RecordType>
+  ): FunctionEvaluator<RecordType> {
+    return new FunctionEvaluator({ ...this, inputMapping });
+  }
+}

package/src/helpers/asEvaluatorFn.ts

@@ -0,0 +1,19 @@
+import { AnyFn, EvaluatorFn } from "../types";
+import { isPromise } from "../utils/typeUtils";
+
+import { toEvaluationResult } from "./toEvaluationResult";
+
+/**
+ * A function that converts a generic function into an evaluator function
+ */
+export function asEvaluatorFn<RecordType extends Record<string, unknown>>(
+  fn: AnyFn
+): EvaluatorFn<RecordType> {
+  return async (...args) => {
+    let result = fn(...args);
+    if (isPromise(result)) {
+      result = await result;
+    }
+    return toEvaluationResult(result);
+  };
+}

package/src/helpers/createEvaluator.ts

@@ -0,0 +1,184 @@
+import { withSpan } from "@arizeai/openinference-core";
+
+import { EvaluatorBase } from "../core/EvaluatorBase";
+import { FunctionEvaluator } from "../core/FunctionEvaluator";
+import {
+  EvaluationKind,
+  OptimizationDirection,
+  TelemetryConfig,
+} from "../types";
+
+import { asEvaluatorFn } from "./asEvaluatorFn";
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+type AnyFn = (...args: any[]) => any;
+
+function generateUniqueName(): string {
+  return `evaluator-${Math.random().toString(36).substring(2, 15)}`;
+}
+
+/**
+ * Options for creating a custom evaluator using {@link CreateEvaluator}.
+ *
+ * @public
+ */
+export type CreateEvaluatorOptions = {
+  /**
+   * The name of the evaluator / metric that it measures.
+   *
+   * If not provided, the function will attempt to infer the name from the function's `name` property.
+   * If the function has no name, a unique name will be generated.
+   *
+   * @example
+   * ```typescript
+   * const evaluator = CreateEvaluator(myFunction, { name: "custom-metric" });
+   * ```
+   */
+  name?: string;
+  /**
+   * The kind of the evaluation.
+   *
+   * - `"CODE"`: Code-based evaluator that runs custom logic
+   * - `"LLM"`: LLM-based evaluator that uses a language model
+   *
+   * @defaultValue `"CODE"`
+   *
+   * @example
+   * ```typescript
+   * const evaluator = CreateEvaluator(myFunction, { kind: "CODE" });
+   * ```
+   */
+  kind?: EvaluationKind;
+  /**
+   * The direction to optimize the numeric evaluation score.
+   *
+   * - `"MAXIMIZE"`: Higher scores are better (e.g., accuracy, F1 score)
+   * - `"MINIMIZE"`: Lower scores are better (e.g., error rate, latency)
+   *
+   * @defaultValue `"MAXIMIZE"`
+   *
+   * @example
+   * ```typescript
+   * const evaluator = CreateEvaluator(myFunction, {
+   *   optimizationDirection: "MAXIMIZE"
+   * });
+   * ```
+   */
+  optimizationDirection?: OptimizationDirection;
+  /**
+   * The telemetry configuration for the evaluator.
+   *
+   * When enabled, the evaluator will automatically create OpenTelemetry spans
+   * for tracing and observability. This allows you to track evaluator performance
+   * and debug issues in distributed systems.
+   *
+   * @defaultValue `{ isEnabled: true }`
+   *
+   * @example
+   * ```typescript
+   * const evaluator = CreateEvaluator(myFunction, {
+   *   telemetry: { isEnabled: true, tracer: myTracer }
+   * });
+   * ```
+   */
+  telemetry?: TelemetryConfig;
+};
+
+/**
+ * A factory function for creating a custom evaluator from any function.
+ *
+ * This function wraps a user-provided function into an evaluator that can be used
+ * with Phoenix experiments and evaluations. The function can be synchronous or
+ * asynchronous, and can return a number, an {@link EvaluationResult} object, or
+ * a value that will be automatically converted to an evaluation result.
+ *
+ * The evaluator will automatically:
+ * - Convert the function's return value to an {@link EvaluationResult}
+ * - Handle both sync and async functions
+ * - Wrap the function with OpenTelemetry spans if telemetry is enabled
+ * - Infer the evaluator name from the function name if not provided
+ *
+ * @typeParam RecordType - The type of the input record that the evaluator expects.
+ *   Must extend `Record<string, unknown>`.
+ * @typeParam Fn - The type of the function being wrapped. Must be a function that
+ *   accepts the record type and returns a value compatible with {@link EvaluationResult}.
+ *
+ * @param fn - The function to wrap as an evaluator. Can be synchronous or asynchronous.
+ *   The function should accept a record of type `RecordType` and return either:
+ *   - A number (will be converted to `{ score: number }`)
+ *   - An {@link EvaluationResult} object
+ *   - Any value that can be converted to an evaluation result
+ *
+ * @param options - Optional configuration for the evaluator. See {@link CreateEvaluatorOptions}
+ *   for details on available options.
+ *
+ * @returns An {@link EvaluatorInterface} that can be used with Phoenix experiments
+ *   and evaluation workflows.
+ *
+ * @example
+ * Basic usage with a simple scoring function:
+ * ```typescript
+ * const accuracyEvaluator = CreateEvaluator(
+ *   ({ output, expected }) => {
+ *     return output === expected ? 1 : 0;
+ *   },
+ *   {
+ *     name: "accuracy",
+ *     kind: "CODE",
+ *     optimizationDirection: "MAXIMIZE"
+ *   }
+ * );
+ *
+ * const result = await accuracyEvaluator.evaluate({
+ *   output: "correct answer",
+ *   expected: "correct answer"
+ * });
+ * // result: { score: 1 }
+ * ```
+ *
+ *
+ * @example
+ * Returning a full EvaluationResult:
+ * ```typescript
+ * const qualityEvaluator = CreateEvaluator(
+ *   ({ output }) => {
+ *     const score = calculateQuality(output);
+ *     return {
+ *       score,
+ *       label: score > 0.8 ? "high" : "low",
+ *       explanation: `Quality score: ${score}`
+ *     };
+ *   },
+ *   { name: "quality" }
+ * );
+ * ```
+ */
+export function createEvaluator<
+  RecordType extends Record<string, unknown> = Record<string, unknown>,
+  Fn extends AnyFn = AnyFn,
+>(fn: Fn, options?: CreateEvaluatorOptions): EvaluatorBase<RecordType> {
+  const {
+    name,
+    kind,
+    optimizationDirection,
+    telemetry = { isEnabled: true },
+  } = options || {};
+  const evaluatorName = name || fn.name || generateUniqueName();
+  let evaluateFn = asEvaluatorFn<RecordType>(fn);
+
+  // Add OpenTelemetry span wrapping if telemetry is enabled
+  if (telemetry && telemetry.isEnabled) {
+    evaluateFn = withSpan(evaluateFn, {
+      tracer: telemetry.tracer,
+      name: evaluatorName,
+      kind: "EVALUATOR",
+    });
+  }
+  return new FunctionEvaluator<RecordType>({
+    evaluateFn,
+    name: evaluatorName,
+    kind: kind || "CODE",
+    optimizationDirection: optimizationDirection || "MAXIMIZE",
+    telemetry,
+  });
+}

package/src/helpers/index.ts

@@ -0,0 +1 @@
+export * from "./createEvaluator";

package/src/helpers/toEvaluationResult.ts

@@ -0,0 +1,145 @@
+import { EvaluationResult } from "../types";
+
+/**
+ * Type guard to check if a value has a score property.
+ *
+ * @param result - The value to check
+ * @returns True if the value is an object with a numeric score property
+ *
+ * @internal
+ */
+function resultHasScore(result: unknown): result is { score: number } {
+  return (
+    typeof result === "object" &&
+    result !== null &&
+    "score" in result &&
+    typeof result.score === "number"
+  );
+}
+
+/**
+ * Type guard to check if a value has a label property.
+ *
+ * @param result - The value to check
+ * @returns True if the value is an object with a string label property
+ *
+ * @internal
+ */
+function resultHasLabel(result: unknown): result is { label: string } {
+  return (
+    typeof result === "object" &&
+    result !== null &&
+    "label" in result &&
+    typeof result.label === "string"
+  );
+}
+
+/**
+ * Type guard to check if a value has an explanation property.
+ *
+ * @param result - The value to check
+ * @returns True if the value is an object with a string explanation property
+ *
+ * @internal
+ */
+function resultHasExplanation(
+  result: unknown
+): result is { explanation: string } {
+  return (
+    typeof result === "object" &&
+    result !== null &&
+    "explanation" in result &&
+    typeof result.explanation === "string"
+  );
+}
+
+/**
+ * Converts an unknown value to an {@link EvaluationResult}.
+ *
+ * This function provides a flexible way to normalize various return types from
+ * evaluator functions into a standardized `EvaluationResult` format. It handles
+ * multiple input types:
+ *
+ * - **Numbers**: Converted to `{ score: number }`
+ * - **Strings**: Converted to `{ label: string }`
+ * - **Objects**: Extracts `score`, `label`, and `explanation` properties if present
+ * - **Other types**: Returns an empty `EvaluationResult` object
+ *
+ * This is particularly useful when creating evaluators from functions that may
+ * return different types, ensuring consistent evaluation result formatting.
+ *
+ * @param result - The value to convert to an EvaluationResult. Can be:
+ *   - A number (converted to score)
+ *   - A string (converted to label)
+ *   - An object with optional `score`, `label`, and/or `explanation` properties
+ *   - Any other value (returns empty object)
+ *
+ * @returns An {@link EvaluationResult} object with extracted properties
+ *
+ * @example
+ * Convert a number to an EvaluationResult:
+ * ```typescript
+ * const result = toEvaluationResult(0.95);
+ * // Returns: { score: 0.95 }
+ * ```
+ *
+ * @example
+ * Convert a string to an EvaluationResult:
+ * ```typescript
+ * const result = toEvaluationResult("correct");
+ * // Returns: { label: "correct" }
+ * ```
+ *
+ * @example
+ * Convert an object with all properties:
+ * ```typescript
+ * const result = toEvaluationResult({
+ *   score: 0.9,
+ *   label: "high",
+ *   explanation: "High quality output"
+ * });
+ * // Returns: { score: 0.9, label: "high", explanation: "High quality output" }
+ * ```
+ *
+ * @example
+ * Convert an object with partial properties:
+ * ```typescript
+ * const result = toEvaluationResult({ score: 0.8 });
+ * // Returns: { score: 0.8 }
+ * ```
+ *
+ * @example
+ * Handle null or undefined:
+ * ```typescript
+ * const result = toEvaluationResult(null);
+ * // Returns: {}
+ * ```
+ *
+ * @public
+ */
+export function toEvaluationResult(result: unknown): EvaluationResult {
+  if (typeof result === "number") {
+    return {
+      score: result,
+    };
+  }
+  if (typeof result === "string") {
+    return {
+      label: result,
+    };
+  }
+  if (typeof result === "object" && result !== null) {
+    const evaluationResult: EvaluationResult = {};
+    if (resultHasScore(result)) {
+      evaluationResult.score = result.score;
+    }
+    if (resultHasLabel(result)) {
+      evaluationResult.label = result.label;
+    }
+    if (resultHasExplanation(result)) {
+      evaluationResult.explanation = result.explanation;
+    }
+    return evaluationResult;
+  }
+  return {};
+}

package/src/index.ts

@@ -2,3 +2,4 @@ export * from "./llm";
 export * from "./template";
 export * from "./types";
 export * from "./utils";
+export * from "./helpers";

package/src/types/base.ts

@@ -0,0 +1,2 @@
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type AnyFn = (...args: any[]) => any;

package/src/types/index.ts

@@ -2,3 +2,4 @@ export * from "./evals";
 export * from "./templating";
 export * from "./otel";
 export * from "./data";
+export * from "./base";

package/src/utils/typeUtils.ts

@@ -0,0 +1,14 @@
+/**
+ * Type guard for if a function is a Promise
+ * @param value
+ * @returns true if it is a Promise
+ */
+export function isPromise<T = unknown>(value: unknown): value is Promise<T> {
+  return (
+    !!value &&
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    typeof (value as any)?.then === "function" &&
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    typeof (value as any)?.catch === "function"
+  );
+}