@arizeai/phoenix-client 4.0.0 → 4.0.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arizeai/phoenix-client",
-  "version": "4.0.0",
+  "version": "4.0.2",
   "description": "A client for the Phoenix API",
   "main": "dist/src/index.js",
   "module": "dist/esm/index.js",

package/src/client.ts

@@ -1,4 +1,7 @@
-import createOpenApiClient, { type ClientOptions } from "openapi-fetch";
+import createOpenApiClient, {
+  type Middleware,
+  type ClientOptions,
+} from "openapi-fetch";
 import type {
   paths as oapiPathsV1,
   components as oapiComponentsV1,
@@ -49,6 +52,20 @@ export const getMergedOptions = ({
   };
 };
 
+/**
+ * Middleware to take non-successful API calls throw instead of being swallowed
+ */
+const middleware: Middleware = {
+  onResponse({ response }) {
+    if (!response.ok) {
+      // Will produce error messages like "https://example.org/api/v1/example: 404 Not Found".
+      throw new Error(
+        `${response.url}: ${response.status} ${response.statusText}`
+      );
+    }
+  },
+};
+
 /**
  * Create a Phoenix client.
  *
@@ -79,8 +96,10 @@ export const createClient = (
   } = {}
 ) => {
   const mergedOptions = getMergedOptions(config);
+  const openApiClient = createOpenApiClient<pathsV1>(mergedOptions);
+  openApiClient.use(middleware);
   return {
-    ...createOpenApiClient<pathsV1>(mergedOptions),
+    ...openApiClient,
     config: mergedOptions,
   };
 };

package/src/datasets/appendDatasetExamples.ts

@@ -33,7 +33,7 @@ export async function appendDatasetExamples({
   const client = _client || createClient();
   const inputs = examples.map((example) => example.input);
   const outputs = examples.map((example) => example.output ?? {}); // Treat null as an empty object
-  const metadata = examples.map((example) => example.metadata);
+  const metadata = examples.map((example) => example.metadata ?? {});
   let datasetName: string;
   if ("datasetName" in dataset) {
     datasetName = dataset.datasetName;

package/src/datasets/createDataset.ts

@@ -34,8 +34,8 @@ export async function createDataset({
 }: CreateDatasetParams): Promise<CreateDatasetResponse> {
   const client = _client || createClient();
   const inputs = examples.map((example) => example.input);
-  const outputs = examples.map((example) => example.output ?? {}); // Treat null as an empty object
-  const metadata = examples.map((example) => example.metadata);
+  const outputs = examples.map((example) => example?.output ?? {}); // Treat null as an empty object
+  const metadata = examples.map((example) => example?.metadata ?? {});
   const createDatasetResponse = await client.POST("/v1/datasets/upload", {
     params: {
       query: {

package/src/experiments/runExperiment.ts

@@ -22,7 +22,7 @@ import { getDataset } from "../datasets/getDataset";
 import { pluralize } from "../utils/pluralize";
 import { promisifyResult } from "../utils/promisifyResult";
 import { AnnotatorKind } from "../types/annotations";
-import { createProvider, createNoOpProvider } from "./instrumention";
+import { createProvider, createNoOpProvider } from "./instrumentation";
 import { SpanStatusCode, Tracer } from "@opentelemetry/api";
 import {
   MimeType,
@@ -37,7 +37,14 @@ import {
   getDatasetExperimentsUrl,
   getExperimentUrl,
 } from "../utils/urlUtils";
+import assert from "assert";
 
+/**
+ * Validate that a repetition is valid
+ */
+function isValidRepetitionParam(repetitions: number) {
+  return Number.isInteger(repetitions) && repetitions > 0;
+}
 /**
  * Parameters for running an experiment.
  *
@@ -95,6 +102,11 @@ export type RunExperimentParams = ClientFn & {
    */
   setGlobalTracerProvider?: boolean;
   /**
+   * Number of times to repeat each dataset example
+   * @default 1
+   */
+  repetitions?: number;
+  /*
    * Whether to use batching for the span processor.
    * @default true
    */
@@ -146,8 +158,14 @@ export async function runExperiment({
   concurrency = 5,
   dryRun = false,
   setGlobalTracerProvider = true,
+  repetitions = 1,
   useBatchSpanProcessor = true,
 }: RunExperimentParams): Promise<RanExperiment> {
+  // Validation
+  assert(
+    isValidRepetitionParam(repetitions),
+    "repetitions must be an integer greater than 0"
+  );
   let provider: NodeTracerProvider | undefined;
   const isDryRun = typeof dryRun === "number" || dryRun === true;
   const client = _client ?? createClient();
@@ -185,6 +203,7 @@ export async function runExperiment({
           description: experimentDescription,
           metadata: experimentMetadata,
           project_name: projectName,
+          repetitions,
         },
       })
       .then((res) => res.data?.data);
@@ -262,6 +281,7 @@ export async function runExperiment({
     isDryRun,
     nExamples,
     tracer: taskTracer,
+    repetitions,
   });
   logger.info(`✅ Task runs completed`);
 
@@ -315,6 +335,7 @@ function runTaskWithExamples({
   isDryRun,
   nExamples,
   tracer,
+  repetitions = 1,
 }: {
   /** The client to use */
   client: PhoenixClient;
@@ -336,9 +357,23 @@ function runTaskWithExamples({
   nExamples: number;
   /** TraceProvider instance that will be used to create spans from task calls */
   tracer: Tracer;
+  /** Number of repetitions per example */
+  repetitions?: number;
 }): Promise<void> {
+  // Validate the input
+  assert(
+    isValidRepetitionParam(repetitions),
+    "repetitions must be an integer greater than 0"
+  );
+
   logger.info(`🔧 Running task "${task.name}" on dataset "${dataset.id}"`);
-  const run = async (example: ExampleWithId) => {
+  const run = async ({
+    example,
+    repetitionNumber,
+  }: {
+    example: ExampleWithId;
+    repetitionNumber: number;
+  }) => {
     return tracer.startActiveSpan(`Task: ${task.name}`, async (span) => {
       logger.info(
         `🔧 Running task "${task.name}" on example "${example.id} of dataset "${dataset.id}"`
@@ -374,7 +409,7 @@ function runTaskWithExamples({
           body: {
             dataset_example_id: example.id,
             output: thisRun.output,
-            repetition_number: 0,
+            repetition_number: repetitionNumber,
             start_time: thisRun.startTime.toISOString(),
             end_time: thisRun.endTime.toISOString(),
             trace_id: thisRun.traceId,
@@ -404,15 +439,23 @@ function runTaskWithExamples({
   };
   const q = queue(run, concurrency);
   const examplesToUse = dataset.examples.slice(0, nExamples);
-  examplesToUse.forEach((example) =>
-    q.push(example, (err) => {
-      if (err) {
-        logger.error(
-          `Error running task "${task.name}" on example "${example.id}": ${err}`
-        );
-      }
-    })
-  );
+
+  examplesToUse
+    .flatMap((example) =>
+      Array.from({ length: repetitions }, (_, index) => ({
+        example,
+        repetitionNumber: index + 1, // Repetitions start at 1
+      }))
+    )
+    .forEach((exampleWithRepetition) =>
+      q.push(exampleWithRepetition, (err) => {
+        if (err) {
+          logger.error(
+            `Error running task "${task.name}" on example "${exampleWithRepetition.example.id}" repetition ${exampleWithRepetition.repetitionNumber}: ${err}`
+          );
+        }
+      })
+    );
   return q.drain();
 }
 
@@ -674,7 +717,7 @@ async function runEvaluator({
         input: example.input,
         output: run.output ?? null,
         expected: example.output,
-        metadata: example.metadata,
+        metadata: example?.metadata,
       });
       thisEval.result = result;
       logger.info(

package/src/types/datasets.ts

@@ -47,8 +47,8 @@ export interface DatasetExamples {
  */
 export interface Example {
   input: Record<string, unknown>;
-  output: Record<string, unknown> | null;
-  metadata: Record<string, unknown>;
+  output?: Record<string, unknown> | null;
+  metadata?: Record<string, unknown> | null;
 }
 
 /**

package/src/types/experiments.ts

@@ -67,7 +67,7 @@ export type EvaluatorParams = {
   /**
    * Metadata associated with the Dataset Example
    */
-  metadata?: Record<string, unknown>;
+  metadata?: Example["metadata"];
 };
 
 export type Evaluator = {

package/dist/esm/experiments/instrumention.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"instrumention.d.ts","sourceRoot":"","sources":["../../../src/experiments/instrumention.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,kBAAkB,EAEnB,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAM/C;;GAEG;AACH,wBAAgB,cAAc,CAAC,EAC7B,WAAW,EACX,OAAO,EACP,OAAO,EACP,qBAA4B,GAC7B,EAAE;IACD,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,cAAc,CAAC;IACxB;;;OAGG;IACH,qBAAqB,EAAE,OAAO,CAAC;IAC/B;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;CACjB,sBAuBA;AAED;;GAEG;AACH,wBAAgB,kBAAkB,uBAIjC"}

package/dist/esm/experiments/instrumention.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"instrumention.js","sourceRoot":"","sources":["../../../src/experiments/instrumention.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAC3E,OAAO,EAAE,iBAAiB,EAAE,MAAM,0CAA0C,CAAC;AAC7E,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EACL,kBAAkB,GAEnB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,wBAAwB,EAAE,MAAM,6CAA6C,CAAC;AAEvF,OAAO,EACL,+BAA+B,EAC/B,gCAAgC,GACjC,MAAM,+BAA+B,CAAC;AAEvC;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,EAC7B,WAAW,EACX,OAAO,EACP,OAAO,EACP,qBAAqB,GAAG,IAAI,GAa7B;IACC,IAAI,CAAC,SAAS,CAAC,IAAI,iBAAiB,EAAE,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC;IAE5D,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC;QACrC,GAAG,EAAE,GAAG,OAAO,YAAY;QAC3B,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO;KACxE,CAAC,CAAC;IAEH,IAAI,aAA4B,CAAC;IACjC,IAAI,qBAAqB,EAAE,CAAC;QAC1B,aAAa,GAAG,IAAI,+BAA+B,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IACpE,CAAC;SAAM,CAAC;QACN,aAAa,GAAG,IAAI,gCAAgC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,kBAAkB,CAAC;QACtC,QAAQ,EAAE,sBAAsB,CAAC;YAC/B,CAAC,wBAAwB,CAAC,EAAE,WAAW;SACxC,CAAC;QACF,cAAc,EAAE,CAAC,aAAa,CAAC;KAChC,CAAC,CAAC;IAEH,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB;IAChC,MAAM,QAAQ,GAAG,IAAI,kBAAkB,CAAC,EAAE,CAAC,CAAC;IAE5C,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/src/experiments/instrumention.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"instrumention.d.ts","sourceRoot":"","sources":["../../../src/experiments/instrumention.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,kBAAkB,EAEnB,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAM/C;;GAEG;AACH,wBAAgB,cAAc,CAAC,EAC7B,WAAW,EACX,OAAO,EACP,OAAO,EACP,qBAA4B,GAC7B,EAAE;IACD,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,cAAc,CAAC;IACxB;;;OAGG;IACH,qBAAqB,EAAE,OAAO,CAAC;IAC/B;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;CACjB,sBAuBA;AAED;;GAEG;AACH,wBAAgB,kBAAkB,uBAIjC"}

package/dist/src/experiments/instrumention.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"instrumention.js","sourceRoot":"","sources":["../../../src/experiments/instrumention.ts"],"names":[],"mappings":";;AAiBA,wCAwCC;AAKD,gDAIC;AAlED,4CAA2E;AAC3E,wFAA6E;AAC7E,wDAAkE;AAClE,kEAGuC;AACvC,oGAAuF;AAEvF,wEAGuC;AAEvC;;GAEG;AACH,SAAgB,cAAc,CAAC,EAC7B,WAAW,EACX,OAAO,EACP,OAAO,EACP,qBAAqB,GAAG,IAAI,GAa7B;IACC,UAAI,CAAC,SAAS,CAAC,IAAI,uBAAiB,EAAE,EAAE,kBAAY,CAAC,KAAK,CAAC,CAAC;IAE5D,MAAM,QAAQ,GAAG,IAAI,6CAAiB,CAAC;QACrC,GAAG,EAAE,GAAG,OAAO,YAAY;QAC3B,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO;KACxE,CAAC,CAAC;IAEH,IAAI,aAA4B,CAAC;IACjC,IAAI,qBAAqB,EAAE,CAAC;QAC1B,aAAa,GAAG,IAAI,sDAA+B,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IACpE,CAAC;SAAM,CAAC;QACN,aAAa,GAAG,IAAI,uDAAgC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,mCAAkB,CAAC;QACtC,QAAQ,EAAE,IAAA,kCAAsB,EAAC;YAC/B,CAAC,6DAAwB,CAAC,EAAE,WAAW;SACxC,CAAC;QACF,cAAc,EAAE,CAAC,aAAa,CAAC;KAChC,CAAC,CAAC;IAEH,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,SAAgB,kBAAkB;IAChC,MAAM,QAAQ,GAAG,IAAI,mCAAkB,CAAC,EAAE,CAAC,CAAC;IAE5C,OAAO,QAAQ,CAAC;AAClB,CAAC"}