@arizeai/phoenix-client 5.2.1 → 5.4.0

package/src/experiments/helpers/fromPhoenixLLMEvaluator.ts

@@ -0,0 +1,24 @@
+import type { LLMEvaluator } from "@arizeai/phoenix-evals";
+
+import { Evaluator } from "../../types/experiments";
+
+import { asExperimentEvaluator } from "./asExperimentEvaluator";
+
+/**
+ * A function that acts as a bridge, converting phoenix-evals to be experiment evaluator compatible
+ * @param phoenixEvaluator
+ * @returns an experiment compatible Evaluator
+ */
+export function fromPhoenixLLMEvaluator<
+  RecordType extends Record<string, unknown>,
+>(phoenixLLMEvaluator: LLMEvaluator<RecordType>): Evaluator {
+  return asExperimentEvaluator({
+    name: phoenixLLMEvaluator.name,
+    kind: "LLM",
+    evaluate: (example) => {
+      // For now blindly coerce the types
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      return phoenixLLMEvaluator.evaluate(example as any);
+    },
+  });
+}

package/src/experiments/helpers/getExperimentEvaluators.ts

@@ -0,0 +1,74 @@
+import type { LLMEvaluator } from "@arizeai/phoenix-evals";
+
+import { Evaluator } from "../../types/experiments";
+
+import { fromPhoenixLLMEvaluator } from "./fromPhoenixLLMEvaluator";
+
+/**
+ * A type guard for LLMEvaluator classes.
+ * Note: this is not fool proof, and may need to be updated as phoenix-evals evolves.
+ */
+function isPhoenixLLMEvaluator(
+  evaluator: unknown
+): evaluator is LLMEvaluator<Record<string, unknown>> {
+  if (
+    typeof evaluator !== "object" ||
+    evaluator === null ||
+    !("evaluate" in evaluator) ||
+    typeof evaluator.evaluate !== "function" ||
+    !("name" in evaluator) ||
+    typeof evaluator.name !== "string" ||
+    !("kind" in evaluator) ||
+    typeof evaluator.kind !== "string" ||
+    evaluator.kind !== "LLM"
+  ) {
+    return false;
+  }
+
+  // Check if it's a class instance (not a plain object)
+  // Phoenix evaluators are class instances, plain evaluators are objects
+  const isClassInstance =
+    evaluator.constructor !== Object && evaluator.constructor !== undefined;
+
+  // If it's a class instance, it's definitely a phoenix evaluator
+  if (isClassInstance) {
+    return true;
+  }
+
+  // Otherwise, it's a plain Evaluator object, not a phoenix evaluator
+  return false;
+}
+
+/**
+ * A type guard for Evaluator objects.
+ * Note: this is not fool proof, and may need to be updated as the package evolves
+ */
+function isExperimentEvaluator(evaluator: unknown): evaluator is Evaluator {
+  return (
+    typeof evaluator === "object" &&
+    evaluator !== null &&
+    "evaluate" in evaluator &&
+    typeof evaluator.evaluate === "function" &&
+    "name" in evaluator &&
+    typeof evaluator.name === "string" &&
+    "kind" in evaluator &&
+    typeof evaluator.kind === "string" &&
+    (evaluator.kind === "CODE" || evaluator.kind === "LLM")
+  );
+}
+
+/**
+ * A function that normalizes evaluators to be runnable by experiments. This is a best effort to support a variety of evaluator types.
+ */
+export function getExperimentEvaluators(evaluators: unknown[]): Evaluator[] {
+  return evaluators.map((evaluator) => {
+    // Check phoenix evaluators first, as they are more specific
+    if (isPhoenixLLMEvaluator(evaluator)) {
+      return fromPhoenixLLMEvaluator(evaluator);
+    }
+    if (isExperimentEvaluator(evaluator)) {
+      return evaluator;
+    }
+    throw new Error(`Unsupported evaluator: ${JSON.stringify(evaluator)}`);
+  });
+}

package/src/experiments/helpers/index.ts

@@ -0,0 +1,3 @@
+export * from "./asExperimentEvaluator";
+export * from "./getExperimentEvaluators";
+export * from "./fromPhoenixLLMEvaluator";

package/src/experiments/index.ts

@@ -2,3 +2,9 @@ export * from "./getExperimentInfo";
 export * from "./getExperiment";
 export * from "./getExperimentRuns";
 export * from "./runExperiment";
+export * from "./createExperiment";
+export * from "./listExperiments";
+export * from "./deleteExperiment";
+export * from "./resumeExperiment";
+export * from "./resumeEvaluation";
+export * from "./helpers";

package/src/experiments/listExperiments.ts

@@ -0,0 +1,83 @@
+import { components } from "../__generated__/api/v1";
+import { createClient } from "../client";
+import { ClientFn } from "../types/core";
+import { ExperimentInfo } from "../types/experiments";
+
+import invariant from "tiny-invariant";
+
+export type ListExperimentsParams = ClientFn & {
+  /**
+   * The dataset ID to list experiments for
+   */
+  datasetId: string;
+};
+
+const DEFAULT_PAGE_SIZE = 50;
+
+/**
+ * List all experiments for a dataset with automatic pagination handling.
+ *
+ * This function automatically handles pagination behind the scenes and returns
+ * a simple list of experiments.
+ *
+ * @example
+ * ```ts
+ * import { listExperiments } from "@arizeai/phoenix-client/experiments";
+ *
+ * const experiments = await listExperiments({
+ *   datasetId: "dataset_123",
+ * });
+ *
+ * for (const experiment of experiments) {
+ *   console.log(`Experiment: ${experiment.id}, Runs: ${experiment.successfulRunCount}`);
+ * }
+ * ```
+ */
+export async function listExperiments({
+  client: _client,
+  datasetId,
+}: ListExperimentsParams): Promise<ExperimentInfo[]> {
+  const client = _client || createClient();
+
+  const experiments: ExperimentInfo[] = [];
+  let cursor: string | null = null;
+
+  do {
+    const res: {
+      data?: components["schemas"]["ListExperimentsResponseBody"];
+    } = await client.GET("/v1/datasets/{dataset_id}/experiments", {
+      params: {
+        path: {
+          dataset_id: datasetId,
+        },
+        query: {
+          cursor,
+          limit: DEFAULT_PAGE_SIZE,
+        },
+      },
+    });
+
+    cursor = res.data?.next_cursor || null;
+    const data = res.data?.data;
+    invariant(data, "Failed to list experiments");
+
+    experiments.push(
+      ...data.map((exp) => ({
+        id: exp.id,
+        datasetId: exp.dataset_id,
+        datasetVersionId: exp.dataset_version_id,
+        repetitions: exp.repetitions,
+        metadata: exp.metadata || {},
+        projectName: exp.project_name || null,
+        createdAt: exp.created_at,
+        updatedAt: exp.updated_at,
+        exampleCount: exp.example_count,
+        successfulRunCount: exp.successful_run_count,
+        failedRunCount: exp.failed_run_count,
+        missingRunCount: exp.missing_run_count,
+      }))
+    );
+  } while (cursor != null);
+
+  return experiments;
+}