@arizeai/phoenix-client 4.0.0 → 4.0.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arizeai/phoenix-client",
-  "version": "4.0.0",
+  "version": "4.0.1",
   "description": "A client for the Phoenix API",
   "main": "dist/src/index.js",
   "module": "dist/esm/index.js",

package/src/client.ts

@@ -1,4 +1,7 @@
-import createOpenApiClient, { type ClientOptions } from "openapi-fetch";
+import createOpenApiClient, {
+  type Middleware,
+  type ClientOptions,
+} from "openapi-fetch";
 import type {
   paths as oapiPathsV1,
   components as oapiComponentsV1,
@@ -49,6 +52,20 @@ export const getMergedOptions = ({
   };
 };
 
+/**
+ * Middleware to take non-successful API calls throw instead of being swallowed
+ */
+const middleware: Middleware = {
+  onResponse({ response }) {
+    if (!response.ok) {
+      // Will produce error messages like "https://example.org/api/v1/example: 404 Not Found".
+      throw new Error(
+        `${response.url}: ${response.status} ${response.statusText}`
+      );
+    }
+  },
+};
+
 /**
  * Create a Phoenix client.
  *
@@ -79,8 +96,10 @@ export const createClient = (
   } = {}
 ) => {
   const mergedOptions = getMergedOptions(config);
+  const openApiClient = createOpenApiClient<pathsV1>(mergedOptions);
+  openApiClient.use(middleware);
   return {
-    ...createOpenApiClient<pathsV1>(mergedOptions),
+    ...openApiClient,
     config: mergedOptions,
   };
 };

package/src/datasets/appendDatasetExamples.ts

@@ -33,7 +33,7 @@ export async function appendDatasetExamples({
   const client = _client || createClient();
   const inputs = examples.map((example) => example.input);
   const outputs = examples.map((example) => example.output ?? {}); // Treat null as an empty object
-  const metadata = examples.map((example) => example.metadata);
+  const metadata = examples.map((example) => example.metadata ?? {});
   let datasetName: string;
   if ("datasetName" in dataset) {
     datasetName = dataset.datasetName;

package/src/datasets/createDataset.ts

@@ -34,8 +34,8 @@ export async function createDataset({
 }: CreateDatasetParams): Promise<CreateDatasetResponse> {
   const client = _client || createClient();
   const inputs = examples.map((example) => example.input);
-  const outputs = examples.map((example) => example.output ?? {}); // Treat null as an empty object
-  const metadata = examples.map((example) => example.metadata);
+  const outputs = examples.map((example) => example?.output ?? {}); // Treat null as an empty object
+  const metadata = examples.map((example) => example?.metadata ?? {});
   const createDatasetResponse = await client.POST("/v1/datasets/upload", {
     params: {
       query: {

package/src/experiments/runExperiment.ts

@@ -95,6 +95,11 @@ export type RunExperimentParams = ClientFn & {
    */
   setGlobalTracerProvider?: boolean;
   /**
+   * Number of times to repeat each dataset example
+   * @default 1
+   */
+  repetitions?: number;
+  /*
    * Whether to use batching for the span processor.
    * @default true
    */
@@ -146,6 +151,7 @@ export async function runExperiment({
   concurrency = 5,
   dryRun = false,
   setGlobalTracerProvider = true,
+  repetitions = 1,
   useBatchSpanProcessor = true,
 }: RunExperimentParams): Promise<RanExperiment> {
   let provider: NodeTracerProvider | undefined;
@@ -185,6 +191,7 @@ export async function runExperiment({
           description: experimentDescription,
           metadata: experimentMetadata,
           project_name: projectName,
+          repetitions,
         },
       })
       .then((res) => res.data?.data);
@@ -262,6 +269,7 @@ export async function runExperiment({
     isDryRun,
     nExamples,
     tracer: taskTracer,
+    repetitions,
   });
   logger.info(`✅ Task runs completed`);
 
@@ -315,6 +323,7 @@ function runTaskWithExamples({
   isDryRun,
   nExamples,
   tracer,
+  repetitions = 1,
 }: {
   /** The client to use */
   client: PhoenixClient;
@@ -336,9 +345,17 @@ function runTaskWithExamples({
   nExamples: number;
   /** TraceProvider instance that will be used to create spans from task calls */
   tracer: Tracer;
+  /** Number of repetitions per example */
+  repetitions?: number;
 }): Promise<void> {
   logger.info(`🔧 Running task "${task.name}" on dataset "${dataset.id}"`);
-  const run = async (example: ExampleWithId) => {
+  const run = async ({
+    example,
+    repetitionNumber,
+  }: {
+    example: ExampleWithId;
+    repetitionNumber: number;
+  }) => {
     return tracer.startActiveSpan(`Task: ${task.name}`, async (span) => {
       logger.info(
         `🔧 Running task "${task.name}" on example "${example.id} of dataset "${dataset.id}"`
@@ -374,7 +391,7 @@ function runTaskWithExamples({
           body: {
             dataset_example_id: example.id,
             output: thisRun.output,
-            repetition_number: 0,
+            repetition_number: repetitionNumber,
             start_time: thisRun.startTime.toISOString(),
             end_time: thisRun.endTime.toISOString(),
             trace_id: thisRun.traceId,
@@ -404,15 +421,23 @@ function runTaskWithExamples({
   };
   const q = queue(run, concurrency);
   const examplesToUse = dataset.examples.slice(0, nExamples);
-  examplesToUse.forEach((example) =>
-    q.push(example, (err) => {
-      if (err) {
-        logger.error(
-          `Error running task "${task.name}" on example "${example.id}": ${err}`
-        );
-      }
-    })
-  );
+
+  examplesToUse
+    .flatMap((example) =>
+      Array.from({ length: repetitions }, (_, index) => ({
+        example,
+        repetitionNumber: index,
+      }))
+    )
+    .forEach((exampleWithRepetition) =>
+      q.push(exampleWithRepetition, (err) => {
+        if (err) {
+          logger.error(
+            `Error running task "${task.name}" on example "${exampleWithRepetition.example.id}" repetition ${exampleWithRepetition.repetitionNumber}: ${err}`
+          );
+        }
+      })
+    );
   return q.drain();
 }
 
@@ -674,7 +699,7 @@ async function runEvaluator({
         input: example.input,
         output: run.output ?? null,
         expected: example.output,
-        metadata: example.metadata,
+        metadata: example?.metadata,
       });
       thisEval.result = result;
       logger.info(

package/src/types/datasets.ts

@@ -47,8 +47,8 @@ export interface DatasetExamples {
  */
 export interface Example {
   input: Record<string, unknown>;
-  output: Record<string, unknown> | null;
-  metadata: Record<string, unknown>;
+  output?: Record<string, unknown> | null;
+  metadata?: Record<string, unknown> | null;
 }
 
 /**

package/src/types/experiments.ts

@@ -67,7 +67,7 @@ export type EvaluatorParams = {
   /**
    * Metadata associated with the Dataset Example
    */
-  metadata?: Record<string, unknown>;
+  metadata?: Example["metadata"];
 };
 
 export type Evaluator = {