@arizeai/phoenix-client 5.3.0 → 5.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arizeai/phoenix-client",
-  "version": "5.3.0",
+  "version": "5.4.0",
   "description": "A client for the Phoenix API",
   "main": "dist/src/index.js",
   "module": "dist/esm/index.js",
@@ -66,8 +66,8 @@
     "openapi-typescript": "^7.6.1",
     "tsx": "^4.19.3",
     "typescript": "^5.8.2",
-    "vitest": "^2.1.9",
-    "@arizeai/phoenix-evals": "0.3.0"
+    "vitest": "^4.0.10",
+    "@arizeai/phoenix-evals": "0.4.0"
   },
   "dependencies": {
     "@arizeai/openinference-semantic-conventions": "^1.1.0",
@@ -75,7 +75,7 @@
     "async": "^3.2.6",
     "openapi-fetch": "^0.12.5",
     "tiny-invariant": "^1.3.3",
-    "zod": "^3.24.2",
+    "zod": "^3.24.3",
     "zod-to-json-schema": "^3.24.3",
     "@arizeai/phoenix-otel": "0.3.0"
   },
@@ -94,6 +94,7 @@
     "build": "tsc --build tsconfig.json tsconfig.esm.json && tsc-alias -p tsconfig.esm.json",
     "postbuild": "echo '{\"type\": \"module\"}' > ./dist/esm/package.json",
     "type:check": "tsc --noEmit",
-    "test": "vitest --typecheck"
+    "test": "vitest run",
+    "test:watch": "vitest watch"
   }
 }

package/src/experiments/helpers/asExperimentEvaluator.ts

@@ -0,0 +1,29 @@
+import { AnnotatorKind } from "../../types/annotations";
+import { Evaluator } from "../../types/experiments";
+
+/**
+ * Wrap an evaluator function in an object with a name property.
+ *
+ * @experimental This feature is not complete, and will change in the future.
+ *
+ * @param params - The parameters for creating the evaluator
+ * @param params.name - The name of the evaluator.
+ * @param params.kind - The kind of evaluator (e.g., "CODE", "LLM")
+ * @param params.evaluate - The evaluator function.
+ * @returns The evaluator object.
+ */
+export function asExperimentEvaluator({
+  name,
+  kind,
+  evaluate,
+}: {
+  name: string;
+  kind: AnnotatorKind;
+  evaluate: Evaluator["evaluate"];
+}): Evaluator {
+  return {
+    name,
+    kind,
+    evaluate,
+  };
+}

package/src/experiments/helpers/fromPhoenixLLMEvaluator.ts

@@ -0,0 +1,24 @@
+import type { LLMEvaluator } from "@arizeai/phoenix-evals";
+
+import { Evaluator } from "../../types/experiments";
+
+import { asExperimentEvaluator } from "./asExperimentEvaluator";
+
+/**
+ * A function that acts as a bridge, converting phoenix-evals to be experiment evaluator compatible
+ * @param phoenixEvaluator
+ * @returns an experiment compatible Evaluator
+ */
+export function fromPhoenixLLMEvaluator<
+  RecordType extends Record<string, unknown>,
+>(phoenixLLMEvaluator: LLMEvaluator<RecordType>): Evaluator {
+  return asExperimentEvaluator({
+    name: phoenixLLMEvaluator.name,
+    kind: "LLM",
+    evaluate: (example) => {
+      // For now blindly coerce the types
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      return phoenixLLMEvaluator.evaluate(example as any);
+    },
+  });
+}

package/src/experiments/helpers/getExperimentEvaluators.ts

@@ -0,0 +1,74 @@
+import type { LLMEvaluator } from "@arizeai/phoenix-evals";
+
+import { Evaluator } from "../../types/experiments";
+
+import { fromPhoenixLLMEvaluator } from "./fromPhoenixLLMEvaluator";
+
+/**
+ * A type guard for LLMEvaluator classes.
+ * Note: this is not fool proof, and may need to be updated as phoenix-evals evolves.
+ */
+function isPhoenixLLMEvaluator(
+  evaluator: unknown
+): evaluator is LLMEvaluator<Record<string, unknown>> {
+  if (
+    typeof evaluator !== "object" ||
+    evaluator === null ||
+    !("evaluate" in evaluator) ||
+    typeof evaluator.evaluate !== "function" ||
+    !("name" in evaluator) ||
+    typeof evaluator.name !== "string" ||
+    !("kind" in evaluator) ||
+    typeof evaluator.kind !== "string" ||
+    evaluator.kind !== "LLM"
+  ) {
+    return false;
+  }
+
+  // Check if it's a class instance (not a plain object)
+  // Phoenix evaluators are class instances, plain evaluators are objects
+  const isClassInstance =
+    evaluator.constructor !== Object && evaluator.constructor !== undefined;
+
+  // If it's a class instance, it's definitely a phoenix evaluator
+  if (isClassInstance) {
+    return true;
+  }
+
+  // Otherwise, it's a plain Evaluator object, not a phoenix evaluator
+  return false;
+}
+
+/**
+ * A type guard for Evaluator objects.
+ * Note: this is not fool proof, and may need to be updated as the package evolves
+ */
+function isExperimentEvaluator(evaluator: unknown): evaluator is Evaluator {
+  return (
+    typeof evaluator === "object" &&
+    evaluator !== null &&
+    "evaluate" in evaluator &&
+    typeof evaluator.evaluate === "function" &&
+    "name" in evaluator &&
+    typeof evaluator.name === "string" &&
+    "kind" in evaluator &&
+    typeof evaluator.kind === "string" &&
+    (evaluator.kind === "CODE" || evaluator.kind === "LLM")
+  );
+}
+
+/**
+ * A function that normalizes evaluators to be runnable by experiments. This is a best effort to support a variety of evaluator types.
+ */
+export function getExperimentEvaluators(evaluators: unknown[]): Evaluator[] {
+  return evaluators.map((evaluator) => {
+    // Check phoenix evaluators first, as they are more specific
+    if (isPhoenixLLMEvaluator(evaluator)) {
+      return fromPhoenixLLMEvaluator(evaluator);
+    }
+    if (isExperimentEvaluator(evaluator)) {
+      return evaluator;
+    }
+    throw new Error(`Unsupported evaluator: ${JSON.stringify(evaluator)}`);
+  });
+}

package/src/experiments/helpers/index.ts

@@ -0,0 +1,3 @@
+export * from "./asExperimentEvaluator";
+export * from "./getExperimentEvaluators";
+export * from "./fromPhoenixLLMEvaluator";

package/src/experiments/index.ts

@@ -7,3 +7,4 @@ export * from "./listExperiments";
 export * from "./deleteExperiment";
 export * from "./resumeExperiment";
 export * from "./resumeEvaluation";
+export * from "./helpers";

package/src/experiments/resumeEvaluation.ts

@@ -18,6 +18,7 @@ import { ClientFn } from "../types/core";
 import type {
   EvaluationResult,
   Evaluator,
+  ExperimentEvaluatorLike,
   IncompleteEvaluation,
   TaskOutput,
 } from "../types/experiments";
@@ -27,6 +28,7 @@ import { ensureString } from "../utils/ensureString";
 import { toObjectHeaders } from "../utils/toObjectHeaders";
 
 import { getExperimentInfo } from "./getExperimentInfo.js";
+import { getExperimentEvaluators } from "./helpers";
 
 import invariant from "tiny-invariant";
 
@@ -64,7 +66,9 @@ export type ResumeEvaluationParams = ClientFn & {
   /**
    * A single evaluator or list of evaluators to run on incomplete evaluations
    */
-  readonly evaluators: Evaluator | readonly Evaluator[];
+  readonly evaluators:
+    | ExperimentEvaluatorLike
+    | readonly ExperimentEvaluatorLike[];
   /**
    * The logger to use
    * @default console
@@ -321,8 +325,9 @@ export async function resumeEvaluation({
   const pageSize = DEFAULT_PAGE_SIZE;
 
   // Normalize evaluators to array
-  const evaluators = Array.isArray(_evaluators) ? _evaluators : [_evaluators];
-
+  const evaluators = getExperimentEvaluators(
+    Array.isArray(_evaluators) ? _evaluators : [_evaluators]
+  );
   // Validate inputs
   invariant(evaluators.length > 0, "Must specify at least one evaluator");
 

package/src/experiments/resumeExperiment.ts

@@ -16,7 +16,10 @@ import { components } from "../__generated__/api/v1";
 import { createClient, type PhoenixClient } from "../client";
 import { ClientFn } from "../types/core";
 import { ExampleWithId } from "../types/datasets";
-import type { Evaluator, ExperimentTask } from "../types/experiments";
+import type {
+  ExperimentEvaluatorLike,
+  ExperimentTask,
+} from "../types/experiments";
 import { type Logger } from "../types/logger";
 import { Channel, ChannelError } from "../utils/channel";
 import { ensureString } from "../utils/ensureString";
@@ -68,7 +71,7 @@ export type ResumeExperimentParams = ClientFn & {
    * Optional evaluators to run on completed task runs
    * @default undefined
    */
-  readonly evaluators?: readonly Evaluator[];
+  readonly evaluators?: readonly ExperimentEvaluatorLike[];
   /**
    * The logger to use
    * @default console

package/src/experiments/runExperiment.ts

@@ -27,6 +27,7 @@ import {
 import type {
   Evaluator,
   ExperimentEvaluationRun,
+  ExperimentEvaluatorLike,
   ExperimentInfo,
   ExperimentRun,
   ExperimentRunID,
@@ -45,6 +46,7 @@ import {
 } from "../utils/urlUtils";
 
 import { getExperimentInfo } from "./getExperimentInfo";
+import { getExperimentEvaluators } from "./helpers";
 
 import assert from "assert";
 import { queue } from "async";
@@ -87,7 +89,7 @@ export type RunExperimentParams = ClientFn & {
   /**
    * The evaluators to use
    */
-  evaluators?: Evaluator[];
+  evaluators?: ExperimentEvaluatorLike[];
   /**
    * The logger to use
    */
@@ -534,7 +536,7 @@ export async function evaluateExperiment({
    **/
   experiment: RanExperiment;
   /** The evaluators to use */
-  evaluators: Evaluator[];
+  evaluators: ExperimentEvaluatorLike[];
   /** The client to use */
   client?: PhoenixClient;
   /** The logger to use */
@@ -652,7 +654,8 @@ export async function evaluateExperiment({
 
   // Run evaluators against all runs
   // Flat list of evaluator + run tuples
-  const evaluatorsAndRuns = evaluators.flatMap((evaluator) =>
+  const normalizedEvaluators = getExperimentEvaluators(evaluators);
+  const evaluatorsAndRuns = normalizedEvaluators.flatMap((evaluator) =>
     runsToEvaluate.map((run) => ({
       evaluator,
       run,
@@ -825,6 +828,7 @@ async function runEvaluator({
  * @param params.kind - The kind of evaluator (e.g., "CODE", "LLM")
  * @param params.evaluate - The evaluator function.
  * @returns The evaluator object.
+ * @deprecated use asExperimentEvaluator instead
  */
 export function asEvaluator({
   name,

package/src/types/experiments.ts

@@ -1,3 +1,5 @@
+import type { LLMEvaluator } from "@arizeai/phoenix-evals";
+
 import { AnnotatorKind } from "./annotations";
 import { Node } from "./core";
 import { Example, ExampleWithId } from "./datasets";
@@ -176,3 +178,11 @@ export interface ExperimentParameters {
    */
   nExamples: number;
 }
+
+/**
+ * A type that represents any type of evaluator that can be used in an experiment.
+ * Unknown is used to capture evaluators from an external library such as phoenix-evals.
+ */
+export type ExperimentEvaluatorLike =
+  | Evaluator
+  | LLMEvaluator<Record<string, unknown>>;