@arizeai/phoenix-client 4.0.1 → 4.0.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arizeai/phoenix-client",
-  "version": "4.0.1",
+  "version": "4.0.3",
   "description": "A client for the Phoenix API",
   "main": "dist/src/index.js",
   "module": "dist/esm/index.js",
@@ -61,7 +61,8 @@
     "openai": "^4.77.0",
     "openapi-typescript": "^7.6.1",
     "tsx": "^4.19.3",
-    "vitest": "^2.1.9"
+    "vitest": "^2.1.9",
+    "@arizeai/phoenix-evals": "0.2.1"
   },
   "dependencies": {
     "@arizeai/openinference-semantic-conventions": "^1.1.0",

package/src/experiments/runExperiment.ts

@@ -22,8 +22,8 @@ import { getDataset } from "../datasets/getDataset";
 import { pluralize } from "../utils/pluralize";
 import { promisifyResult } from "../utils/promisifyResult";
 import { AnnotatorKind } from "../types/annotations";
-import { createProvider, createNoOpProvider } from "./instrumention";
-import { SpanStatusCode, Tracer } from "@opentelemetry/api";
+import { createProvider, createNoOpProvider } from "./instrumentation";
+import { SpanStatusCode, Tracer, trace } from "@opentelemetry/api";
 import {
   MimeType,
   OpenInferenceSpanKind,
@@ -37,7 +37,14 @@ import {
   getDatasetExperimentsUrl,
   getExperimentUrl,
 } from "../utils/urlUtils";
+import assert from "assert";
 
+/**
+ * Validate that a repetition is valid
+ */
+function isValidRepetitionParam(repetitions: number) {
+  return Number.isInteger(repetitions) && repetitions > 0;
+}
 /**
  * Parameters for running an experiment.
  *
@@ -154,6 +161,11 @@ export async function runExperiment({
   repetitions = 1,
   useBatchSpanProcessor = true,
 }: RunExperimentParams): Promise<RanExperiment> {
+  // Validation
+  assert(
+    isValidRepetitionParam(repetitions),
+    "repetitions must be an integer greater than 0"
+  );
   let provider: NodeTracerProvider | undefined;
   const isDryRun = typeof dryRun === "number" || dryRun === true;
   const client = _client ?? createClient();
@@ -278,11 +290,6 @@ export async function runExperiment({
     runs,
   };
 
-  // Shut down the provider so that the experiments run
-  if (provider) {
-    await provider.shutdown?.();
-  }
-
   const { evaluationRuns } = await evaluateExperiment({
     experiment: ranExperiment,
     evaluators: evaluators ?? [],
@@ -290,8 +297,7 @@ export async function runExperiment({
     logger,
     concurrency,
     dryRun,
-    setGlobalTracerProvider,
-    useBatchSpanProcessor,
+    tracerProvider: provider,
   });
   ranExperiment.evaluationRuns = evaluationRuns;
 
@@ -348,6 +354,12 @@ function runTaskWithExamples({
   /** Number of repetitions per example */
   repetitions?: number;
 }): Promise<void> {
+  // Validate the input
+  assert(
+    isValidRepetitionParam(repetitions),
+    "repetitions must be an integer greater than 0"
+  );
+
   logger.info(`🔧 Running task "${task.name}" on dataset "${dataset.id}"`);
   const run = async ({
     example,
@@ -426,7 +438,7 @@ function runTaskWithExamples({
     .flatMap((example) =>
       Array.from({ length: repetitions }, (_, index) => ({
         example,
-        repetitionNumber: index,
+        repetitionNumber: index + 1, // Repetitions start at 1
       }))
     )
     .forEach((exampleWithRepetition) =>
@@ -455,6 +467,7 @@ export async function evaluateExperiment({
   dryRun = false,
   setGlobalTracerProvider = true,
   useBatchSpanProcessor = true,
+  tracerProvider: paramsTracerProvider,
 }: {
   /**
    * The experiment to evaluate
@@ -484,6 +497,11 @@ export async function evaluateExperiment({
    * @default true
    */
   useBatchSpanProcessor?: boolean;
+  /**
+   * The tracer provider to use. If set, the other parameters will be ignored and the passed tracer provider will get used
+   * Intended as a pass-through from runExperiment
+   */
+  tracerProvider?: NodeTracerProvider | null;
 }): Promise<RanExperiment> {
   const isDryRun = typeof dryRun === "number" || dryRun === true;
   const client = _client ?? createClient();
@@ -493,7 +511,11 @@ export async function evaluateExperiment({
     "Phoenix base URL not found. Please set PHOENIX_HOST or set baseUrl on the client."
   );
   let provider: NodeTracerProvider;
-  if (!isDryRun) {
+
+  // Always allow changing of tracer providers
+  if (paramsTracerProvider) {
+    provider = paramsTracerProvider;
+  } else if (!isDryRun) {
     provider = createProvider({
       projectName: "evaluators",
       baseUrl,
@@ -650,7 +672,11 @@ export async function evaluateExperiment({
   logger.info(`✅ Evaluation runs completed`);
 
   if (provider) {
-    await provider.shutdown?.();
+    await provider.shutdown();
+    // Make sure it's not set globally anymore
+    if (setGlobalTracerProvider) {
+      trace.disable();
+    }
   }
 
   return {

package/src/types/experiments.ts

@@ -51,7 +51,7 @@ export interface ExperimentRun extends Node {
   traceId: string | null;
 }
 
-export type EvaluatorParams = {
+export type EvaluatorParams<TaskOutputType = TaskOutput> = {
   /**
    * The input field of the Dataset Example
    */
@@ -59,7 +59,7 @@ export type EvaluatorParams = {
   /**
    * The output of the task
    */
-  output: TaskOutput;
+  output: TaskOutputType;
   /**
    * The expected or reference output of the Dataset Example
    */
@@ -79,10 +79,10 @@ export type Evaluator = {
 };
 
 export type EvaluationResult = {
-  score: number | null;
-  label: string | null;
-  metadata: Record<string, unknown>;
-  explanation: string | null;
+  score?: number | null;
+  label?: string | null;
+  metadata?: Record<string, unknown>;
+  explanation?: string | null;
 };
 
 export interface ExperimentEvaluationRun extends Node {

package/dist/esm/experiments/instrumention.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"instrumention.d.ts","sourceRoot":"","sources":["../../../src/experiments/instrumention.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,kBAAkB,EAEnB,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAM/C;;GAEG;AACH,wBAAgB,cAAc,CAAC,EAC7B,WAAW,EACX,OAAO,EACP,OAAO,EACP,qBAA4B,GAC7B,EAAE;IACD,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,cAAc,CAAC;IACxB;;;OAGG;IACH,qBAAqB,EAAE,OAAO,CAAC;IAC/B;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;CACjB,sBAuBA;AAED;;GAEG;AACH,wBAAgB,kBAAkB,uBAIjC"}

package/dist/esm/experiments/instrumention.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"instrumention.js","sourceRoot":"","sources":["../../../src/experiments/instrumention.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAC3E,OAAO,EAAE,iBAAiB,EAAE,MAAM,0CAA0C,CAAC;AAC7E,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EACL,kBAAkB,GAEnB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,wBAAwB,EAAE,MAAM,6CAA6C,CAAC;AAEvF,OAAO,EACL,+BAA+B,EAC/B,gCAAgC,GACjC,MAAM,+BAA+B,CAAC;AAEvC;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,EAC7B,WAAW,EACX,OAAO,EACP,OAAO,EACP,qBAAqB,GAAG,IAAI,GAa7B;IACC,IAAI,CAAC,SAAS,CAAC,IAAI,iBAAiB,EAAE,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC;IAE5D,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC;QACrC,GAAG,EAAE,GAAG,OAAO,YAAY;QAC3B,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO;KACxE,CAAC,CAAC;IAEH,IAAI,aAA4B,CAAC;IACjC,IAAI,qBAAqB,EAAE,CAAC;QAC1B,aAAa,GAAG,IAAI,+BAA+B,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IACpE,CAAC;SAAM,CAAC;QACN,aAAa,GAAG,IAAI,gCAAgC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,kBAAkB,CAAC;QACtC,QAAQ,EAAE,sBAAsB,CAAC;YAC/B,CAAC,wBAAwB,CAAC,EAAE,WAAW;SACxC,CAAC;QACF,cAAc,EAAE,CAAC,aAAa,CAAC;KAChC,CAAC,CAAC;IAEH,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB;IAChC,MAAM,QAAQ,GAAG,IAAI,kBAAkB,CAAC,EAAE,CAAC,CAAC;IAE5C,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/src/experiments/instrumention.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"instrumention.d.ts","sourceRoot":"","sources":["../../../src/experiments/instrumention.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,kBAAkB,EAEnB,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAM/C;;GAEG;AACH,wBAAgB,cAAc,CAAC,EAC7B,WAAW,EACX,OAAO,EACP,OAAO,EACP,qBAA4B,GAC7B,EAAE;IACD,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,cAAc,CAAC;IACxB;;;OAGG;IACH,qBAAqB,EAAE,OAAO,CAAC;IAC/B;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;CACjB,sBAuBA;AAED;;GAEG;AACH,wBAAgB,kBAAkB,uBAIjC"}

package/dist/src/experiments/instrumention.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"instrumention.js","sourceRoot":"","sources":["../../../src/experiments/instrumention.ts"],"names":[],"mappings":";;AAiBA,wCAwCC;AAKD,gDAIC;AAlED,4CAA2E;AAC3E,wFAA6E;AAC7E,wDAAkE;AAClE,kEAGuC;AACvC,oGAAuF;AAEvF,wEAGuC;AAEvC;;GAEG;AACH,SAAgB,cAAc,CAAC,EAC7B,WAAW,EACX,OAAO,EACP,OAAO,EACP,qBAAqB,GAAG,IAAI,GAa7B;IACC,UAAI,CAAC,SAAS,CAAC,IAAI,uBAAiB,EAAE,EAAE,kBAAY,CAAC,KAAK,CAAC,CAAC;IAE5D,MAAM,QAAQ,GAAG,IAAI,6CAAiB,CAAC;QACrC,GAAG,EAAE,GAAG,OAAO,YAAY;QAC3B,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO;KACxE,CAAC,CAAC;IAEH,IAAI,aAA4B,CAAC;IACjC,IAAI,qBAAqB,EAAE,CAAC;QAC1B,aAAa,GAAG,IAAI,sDAA+B,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IACpE,CAAC;SAAM,CAAC;QACN,aAAa,GAAG,IAAI,uDAAgC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,mCAAkB,CAAC;QACtC,QAAQ,EAAE,IAAA,kCAAsB,EAAC;YAC/B,CAAC,6DAAwB,CAAC,EAAE,WAAW;SACxC,CAAC;QACF,cAAc,EAAE,CAAC,aAAa,CAAC;KAChC,CAAC,CAAC;IAEH,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,SAAgB,kBAAkB;IAChC,MAAM,QAAQ,GAAG,IAAI,mCAAkB,CAAC,EAAE,CAAC,CAAC;IAE5C,OAAO,QAAQ,CAAC;AAClB,CAAC"}