@arizeai/phoenix-client 4.1.0 → 5.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arizeai/phoenix-client",
-  "version": "4.1.0",
+  "version": "5.0.0",
   "description": "A client for the Phoenix API",
   "main": "dist/src/index.js",
   "module": "dist/esm/index.js",
@@ -67,7 +67,7 @@
     "tsx": "^4.19.3",
     "typescript": "^5.8.2",
     "vitest": "^2.1.9",
-    "@arizeai/phoenix-evals": "0.2.1"
+    "@arizeai/phoenix-evals": "0.2.2"
   },
   "dependencies": {
     "@arizeai/openinference-semantic-conventions": "^1.1.0",
@@ -90,7 +90,7 @@
   },
   "optionalDependencies": {
     "@anthropic-ai/sdk": "^0.35.0",
-    "ai": "^5.0.38",
+    "ai": "^5.0.76",
     "openai": "^5.12.1"
   },
   "scripts": {
@@ -98,7 +98,7 @@
     "prebuild": "pnpm run clean && pnpm run generate",
     "generate": "openapi-typescript --empty-objects-unknown=true --default-non-nullable=false ../../../schemas/openapi.json -o ./src/__generated__/api/v1.ts",
     "build": "tsc --build tsconfig.json tsconfig.esm.json && tsc-alias -p tsconfig.esm.json",
-    "postbuild": "echo '{\"type\": \"module\"}' > ./dist/esm/package.json && rimraf dist/test dist/examples",
+    "postbuild": "echo '{\"type\": \"module\"}' > ./dist/esm/package.json",
     "type:check": "tsc --noEmit",
     "test": "vitest --typecheck"
   }

package/src/__generated__/api/v1.ts

@@ -939,6 +939,11 @@ export interface components {
              * @description ID of the dataset version over which the experiment will be run (if omitted, the latest version will be used)
              */
             version_id?: string | null;
+            /**
+             * Splits
+             * @description List of dataset split identifiers (GlobalIDs or names) to filter by
+             */
+            splits?: string[] | null;
             /**
              * Repetitions
              * @description Number of times the experiment should be repeated for each example
@@ -1397,6 +1402,8 @@ export interface components {
             dataset_id: string;
             /** Version Id */
             version_id: string;
+            /** Filtered Splits */
+            filtered_splits?: string[];
             /** Examples */
             examples: components["schemas"]["DatasetExample"][];
         };
@@ -3437,6 +3444,8 @@ export interface operations {
             query?: {
                 /** @description The ID of the dataset version (if omitted, returns data from the latest version) */
                 version_id?: string | null;
+                /** @description List of dataset split identifiers (GlobalIDs or names) to filter by */
+                split?: string[] | null;
             };
             header?: never;
             path: {

package/src/datasets/createOrGetDataset.ts

@@ -0,0 +1,39 @@
+import { createClient } from "../client";
+import { CreateDatasetParams, createDataset } from "./createDataset";
+import { getDatasetInfoByName } from "./getDatasetInfoByName";
+
+export type CreateOrGetDatasetParams = CreateDatasetParams;
+
+export type CreateOrGetDatasetResponse = {
+  datasetId: string;
+};
+
+/**
+ * Given the parameters to create a dataset, this function will either
+ * retrieve an existing dataset by name or create a new one with the provided parameters.
+ *
+ * This is useful in cases where you would like to re-run a pipeline like:
+ * - ensure dataset exists
+ * - create a task
+ * - run experiment
+ * - evaluate experiment
+ * without having to create a new dataset each time.
+ */
+export async function createOrGetDataset({
+  name,
+  description,
+  examples,
+  client: _client,
+}: CreateOrGetDatasetParams): Promise<CreateOrGetDatasetResponse> {
+  const client = _client || createClient();
+  // start by fetching an existing dataset by name, catching any errors that occur
+  try {
+    const dataset = await getDatasetInfoByName({ datasetName: name, client });
+    return {
+      datasetId: dataset.id,
+    };
+  } catch {
+    // If the dataset doesn't exist, create it, falling back to the error handling inside createDataset
+    return await createDataset({ name, description, examples, client });
+  }
+}

package/src/datasets/getDataset.ts

@@ -5,24 +5,22 @@ import { getDatasetExamples } from "./getDatasetExamples";
 import { getDatasetInfo } from "./getDatasetInfo";
 
 export type GetDatasetParams = ClientFn & {
+  /** Dataset selector (ID or name) */
   dataset: DatasetSelector;
-  versionId?: string;
 };
 
 /**
  * Get dataset info and examples from the dataset
  * @param dataset - Dataset selector (ID or name)
- * @param versionId - Optional specific version ID (if omitted, returns data from the latest version)
  */
 export async function getDataset({
   client: _client,
   dataset,
-  versionId,
 }: GetDatasetParams): Promise<Dataset> {
   const client = _client || createClient();
   const [datasetInfo, datasetExamples] = await Promise.all([
     getDatasetInfo({ client, dataset }),
-    getDatasetExamples({ client, dataset, versionId }),
+    getDatasetExamples({ client, dataset }),
   ]);
   return {
     ...datasetInfo,

package/src/datasets/getDatasetExamples.ts

@@ -5,44 +5,44 @@ import { DatasetSelector, DatasetExamples } from "../types/datasets";
 import { getDatasetInfoByName } from "./getDatasetInfoByName";
 
 export type GetDatasetExamplesParams = ClientFn & {
+  /** Dataset selector (ID, name, or version ID) */
   dataset: DatasetSelector;
-  versionId?: string;
 };
 
 /**
  * Get examples from a dataset
- * @param dataset - Dataset selector (ID, name, or version ID)
- * @param versionId - Optional specific version ID (ignored if dataset selector is datasetVersionId)
+ * @param dataset - Dataset selector (ID, name, version ID, or splits)
+ * @returns Dataset examples
  */
 export async function getDatasetExamples({
   client: _client,
-  dataset,
-  versionId,
+  dataset: datasetSelector,
 }: GetDatasetExamplesParams): Promise<DatasetExamples> {
   const client = _client || createClient();
 
   let datasetId: string;
 
-  if ("datasetName" in dataset) {
+  if ("datasetName" in datasetSelector) {
     const datasetInfo = await getDatasetInfoByName({
       client,
-      datasetName: dataset.datasetName,
+      datasetName: datasetSelector.datasetName,
     });
     datasetId = datasetInfo.id;
   } else {
-    datasetId = dataset.datasetId;
+    datasetId = datasetSelector.datasetId;
   }
 
+  const { versionId, splits } = datasetSelector;
+
   const response = await client.GET("/v1/datasets/{id}/examples", {
     params: {
       path: {
         id: datasetId,
       },
-      query: versionId
-        ? {
-            version_id: versionId,
-          }
-        : undefined,
+      query: {
+        ...(versionId ? { version_id: versionId } : {}),
+        ...(splits ? { split: splits } : {}),
+      },
     },
   });
 

package/src/datasets/index.ts

@@ -3,3 +3,4 @@ export * from "./getDataset";
 export * from "./getDatasetExamples";
 export * from "./appendDatasetExamples";
 export * from "./getDatasetInfo";
+export * from "./createOrGetDataset";

package/src/experiments/getExperimentInfo.ts

@@ -18,16 +18,14 @@ export async function getExperimentInfo({
   experimentId: experiment_id,
 }: GetExperimentParams): Promise<ExperimentInfo> {
   const client = _client || createClient();
-  const { data: { data: experimentData } = {} } = await client.GET(
-    "/v1/experiments/{experiment_id}",
-    {
+  const { data: { data: experimentData } = { data: undefined } } =
+    await client.GET("/v1/experiments/{experiment_id}", {
       params: {
         path: {
           experiment_id,
         },
       },
-    }
-  );
+    });
   invariant(experimentData, "Failed to get experiment");
   return {
     id: experimentData.id,

package/src/experiments/instrumentation.ts

@@ -20,6 +20,7 @@ export function createProvider({
   baseUrl,
   headers,
   useBatchSpanProcessor = true,
+  diagLogLevel,
 }: {
   projectName: string;
   headers: HeadersOptions;
@@ -32,8 +33,15 @@ export function createProvider({
    * The base URL of the Phoenix. Doesn't include the /v1/traces path.
    */
   baseUrl: string;
+  /**
+   * The diag log level to set for the built in DiagConsoleLogger instance.
+   * Omit to disable built in logging.
+   */
+  diagLogLevel?: DiagLogLevel;
 }) {
-  diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.ERROR);
+  if (diagLogLevel) {
+    diag.setLogger(new DiagConsoleLogger(), diagLogLevel);
+  }
 
   const exporter = new OTLPTraceExporter({
     url: `${baseUrl}/v1/traces`,

package/src/experiments/runExperiment.ts

@@ -23,7 +23,12 @@ import { pluralize } from "../utils/pluralize";
 import { promisifyResult } from "../utils/promisifyResult";
 import { AnnotatorKind } from "../types/annotations";
 import { createProvider, createNoOpProvider } from "./instrumentation";
-import { SpanStatusCode, Tracer, trace } from "@opentelemetry/api";
+import {
+  type DiagLogLevel,
+  SpanStatusCode,
+  Tracer,
+  trace,
+} from "@opentelemetry/api";
 import {
   MimeType,
   OpenInferenceSpanKind,
@@ -111,6 +116,11 @@ export type RunExperimentParams = ClientFn & {
    * @default true
    */
   useBatchSpanProcessor?: boolean;
+  /**
+   * Log level to set for the default DiagConsoleLogger when tracing.
+   * Omit to disable default diag logging, or to bring your own.
+   */
+  diagLogLevel?: DiagLogLevel;
 };
 
 /**
@@ -150,7 +160,7 @@ export async function runExperiment({
   experimentDescription,
   experimentMetadata = {},
   client: _client,
-  dataset: DatasetSelector,
+  dataset: datasetSelector,
   task,
   evaluators,
   logger = console,
@@ -160,6 +170,7 @@ export async function runExperiment({
   setGlobalTracerProvider = true,
   repetitions = 1,
   useBatchSpanProcessor = true,
+  diagLogLevel,
 }: RunExperimentParams): Promise<RanExperiment> {
   // Validation
   assert(
@@ -169,7 +180,10 @@ export async function runExperiment({
   let provider: NodeTracerProvider | undefined;
   const isDryRun = typeof dryRun === "number" || dryRun === true;
   const client = _client ?? createClient();
-  const dataset = await getDataset({ dataset: DatasetSelector, client });
+  const dataset = await getDataset({
+    dataset: datasetSelector,
+    client,
+  });
   invariant(dataset, `Dataset not found`);
   invariant(dataset.examples.length > 0, `Dataset has no examples`);
   const nExamples =
@@ -186,6 +200,8 @@ export async function runExperiment({
       id: localId(),
       datasetId: dataset.id,
       datasetVersionId: dataset.versionId,
+      // @todo: the dataset should return splits in response body
+      datasetSplits: datasetSelector?.splits ?? [],
       projectName,
       metadata: experimentMetadata,
     };
@@ -204,6 +220,11 @@ export async function runExperiment({
           metadata: experimentMetadata,
           project_name: projectName,
           repetitions,
+          // @todo: the dataset should return splits in response body
+          ...(datasetSelector?.splits
+            ? { splits: datasetSelector.splits }
+            : {}),
+          ...(dataset?.versionId ? { version_id: dataset.versionId } : {}),
         },
       })
       .then((res) => res.data?.data);
@@ -213,6 +234,8 @@ export async function runExperiment({
       id: experimentResponse.id,
       datasetId: experimentResponse.dataset_id,
       datasetVersionId: experimentResponse.dataset_version_id,
+      // @todo: the dataset should return splits in response body
+      datasetSplits: datasetSelector?.splits ?? [],
       projectName,
       metadata: experimentResponse.metadata,
     };
@@ -227,6 +250,7 @@ export async function runExperiment({
       baseUrl,
       headers: client.config.headers ?? {},
       useBatchSpanProcessor,
+      diagLogLevel,
     });
     // Register the provider
     if (setGlobalTracerProvider) {
@@ -298,6 +322,8 @@ export async function runExperiment({
     concurrency,
     dryRun,
     tracerProvider: provider,
+    diagLogLevel,
+    useBatchSpanProcessor,
   });
   ranExperiment.evaluationRuns = evaluationRuns;
 
@@ -468,6 +494,7 @@ export async function evaluateExperiment({
   setGlobalTracerProvider = true,
   useBatchSpanProcessor = true,
   tracerProvider: paramsTracerProvider,
+  diagLogLevel,
 }: {
   /**
    * The experiment to evaluate
@@ -502,6 +529,11 @@ export async function evaluateExperiment({
    * Intended as a pass-through from runExperiment
    */
   tracerProvider?: NodeTracerProvider | null;
+  /**
+   * Log level to set for the default DiagConsoleLogger when tracing.
+   * Omit to disable default diag logging, or to bring your own.
+   */
+  diagLogLevel?: DiagLogLevel;
 }): Promise<RanExperiment> {
   const isDryRun = typeof dryRun === "number" || dryRun === true;
   const client = _client ?? createClient();
@@ -521,6 +553,7 @@ export async function evaluateExperiment({
       baseUrl,
       headers: client.config.headers ?? {},
       useBatchSpanProcessor,
+      diagLogLevel,
     });
     if (setGlobalTracerProvider) {
       provider.register();
@@ -536,7 +569,11 @@ export async function evaluateExperiment({
       ? Math.min(dryRun, Object.keys(experiment.runs).length)
       : Object.keys(experiment.runs).length;
   const dataset = await getDataset({
-    dataset: { datasetId: experiment.datasetId },
+    dataset: {
+      datasetId: experiment.datasetId,
+      versionId: experiment.datasetVersionId,
+      splits: experiment.datasetSplits,
+    },
     client,
   });
   invariant(dataset, `Dataset "${experiment.datasetId}" not found`);

package/src/types/datasets.ts

@@ -1,17 +1,13 @@
 import { Node } from "./core";
 
-/**
- * A dataset can be identified by its datasetId, datasetName, or datasetVersionId
- */
-export type DatasetSelector = { datasetId: string } | { datasetName: string };
+type DatasetSelectorBase = { versionId?: string; splits?: string[] };
 
 /**
- * Parameters for selecting a specific version of a dataset
+ * A dataset can be identified by its datasetId, datasetName, or datasetVersionId
  */
-export interface DatasetVersionSelector {
-  dataset: DatasetSelector;
-  versionId?: string;
-}
+export type DatasetSelector =
+  | (DatasetSelectorBase & { datasetId: string })
+  | (DatasetSelectorBase & { datasetName: string });
 
 /**
  * Overview information about a dataset

package/src/types/experiments.ts

@@ -8,6 +8,8 @@ import { Example } from "./datasets";
 export interface ExperimentInfo extends Node {
   datasetId: string;
   datasetVersionId: string;
+  // @todo: mark this as required when experiment API returns it
+  datasetSplits?: string[];
   /**
    * The project under which the experiment task traces are recorded
    */