@arizeai/phoenix-client 3.0.0 → 3.2.0

package/src/__generated__/api/v1.ts

@@ -290,7 +290,10 @@ export interface paths {
             path?: never;
             cookie?: never;
         };
-        /** List runs for an experiment */
+        /**
+         * List runs for an experiment
+         * @description Retrieve a paginated list of runs for an experiment
+         */
         get: operations["listExperimentRuns"];
         put?: never;
         /** Create run for an experiment */
@@ -633,6 +636,23 @@ export interface paths {
         patch?: never;
         trace?: never;
     };
+    "/v1/document_annotations": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        get?: never;
+        put?: never;
+        /** Annotate Span Documents */
+        post: operations["annotateSpanDocuments"];
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
     "/v1/users": {
         parameters: {
             query?: never;
@@ -681,6 +701,16 @@ export interface paths {
 export type webhooks = Record<string, never>;
 export interface components {
     schemas: {
+        /** AnnotateSpanDocumentsRequestBody */
+        AnnotateSpanDocumentsRequestBody: {
+            /** Data */
+            data: components["schemas"]["SpanDocumentAnnotationData"][];
+        };
+        /** AnnotateSpanDocumentsResponseBody */
+        AnnotateSpanDocumentsResponseBody: {
+            /** Data */
+            data: components["schemas"]["InsertedSpanDocumentAnnotation"][];
+        };
         /** AnnotateSpansRequestBody */
         AnnotateSpansRequestBody: {
             /** Data */
@@ -691,6 +721,24 @@ export interface components {
             /** Data */
             data: components["schemas"]["InsertedSpanAnnotation"][];
         };
+        /** AnnotationResult */
+        AnnotationResult: {
+            /**
+             * Label
+             * @description The label assigned by the annotation
+             */
+            label?: string | null;
+            /**
+             * Score
+             * @description The score assigned by the annotation
+             */
+            score?: number | null;
+            /**
+             * Explanation
+             * @description Explanation of the annotation result
+             */
+            explanation?: string | null;
+        };
         /** CategoricalAnnotationConfig */
         CategoricalAnnotationConfig: {
             /** Name */
@@ -1228,6 +1276,14 @@ export interface components {
              */
             id: string;
         };
+        /** InsertedSpanDocumentAnnotation */
+        InsertedSpanDocumentAnnotation: {
+            /**
+             * Id
+             * @description The ID of the inserted span document annotation
+             */
+            id: string;
+        };
         /** ListDatasetExamplesData */
         ListDatasetExamplesData: {
             /** Dataset Id */
@@ -1259,6 +1315,8 @@ export interface components {
         ListExperimentRunsResponseBody: {
             /** Data */
             data: components["schemas"]["ExperimentRunResponse"][];
+            /** Next Cursor */
+            next_cursor: string | null;
         };
         /** ListExperimentsResponseBody */
         ListExperimentsResponseBody: {
@@ -2097,11 +2155,25 @@ export interface components {
         };
         /** SpanAnnotation */
         SpanAnnotation: {
+            /** Id */
+            id: string;
             /**
-             * Span Id
-             * @description OpenTelemetry Span ID (hex format w/o 0x prefix)
+             * Created At
+             * Format: date-time
              */
-            span_id: string;
+            created_at: string;
+            /**
+             * Updated At
+             * Format: date-time
+             */
+            updated_at: string;
+            /**
+             * Source
+             * @enum {string}
+             */
+            source: "API" | "APP";
+            /** User Id */
+            user_id: string | null;
             /**
              * Name
              * @description The name of the annotation
@@ -2114,7 +2186,7 @@ export interface components {
              */
             annotator_kind: "LLM" | "CODE" | "HUMAN";
             /** @description The result of the annotation */
-            result?: components["schemas"]["SpanAnnotationResult"] | null;
+            result?: components["schemas"]["AnnotationResult"] | null;
             /**
              * Metadata
              * @description Metadata for the annotation
@@ -2128,33 +2200,14 @@ export interface components {
              * @default
              */
             identifier?: string;
-            /** Id */
-            id: string;
-            /**
-             * Created At
-             * Format: date-time
-             */
-            created_at: string;
-            /**
-             * Updated At
-             * Format: date-time
-             */
-            updated_at: string;
-            /**
-             * Source
-             * @enum {string}
-             */
-            source: "API" | "APP";
-            /** User Id */
-            user_id: string | null;
-        };
-        /** SpanAnnotationData */
-        SpanAnnotationData: {
             /**
              * Span Id
              * @description OpenTelemetry Span ID (hex format w/o 0x prefix)
              */
             span_id: string;
+        };
+        /** SpanAnnotationData */
+        SpanAnnotationData: {
             /**
              * Name
              * @description The name of the annotation
@@ -2167,7 +2220,7 @@ export interface components {
              */
             annotator_kind: "LLM" | "CODE" | "HUMAN";
             /** @description The result of the annotation */
-            result?: components["schemas"]["SpanAnnotationResult"] | null;
+            result?: components["schemas"]["AnnotationResult"] | null;
             /**
              * Metadata
              * @description Metadata for the annotation
@@ -2181,24 +2234,11 @@ export interface components {
              * @default
              */
             identifier?: string;
-        };
-        /** SpanAnnotationResult */
-        SpanAnnotationResult: {
-            /**
-             * Label
-             * @description The label assigned by the annotation
-             */
-            label?: string | null;
             /**
-             * Score
-             * @description The score assigned by the annotation
-             */
-            score?: number | null;
-            /**
-             * Explanation
-             * @description Explanation of the annotation result
+             * Span Id
+             * @description OpenTelemetry Span ID (hex format w/o 0x prefix)
              */
-            explanation?: string | null;
+            span_id: string;
         };
         /** SpanAnnotationsResponseBody */
         SpanAnnotationsResponseBody: {
@@ -2220,6 +2260,45 @@ export interface components {
              */
             span_id: string;
         };
+        /** SpanDocumentAnnotationData */
+        SpanDocumentAnnotationData: {
+            /**
+             * Name
+             * @description The name of the annotation
+             */
+            name: string;
+            /**
+             * Annotator Kind
+             * @description The kind of annotator used for the annotation
+             * @enum {string}
+             */
+            annotator_kind: "LLM" | "CODE" | "HUMAN";
+            /** @description The result of the annotation */
+            result?: components["schemas"]["AnnotationResult"] | null;
+            /**
+             * Metadata
+             * @description Metadata for the annotation
+             */
+            metadata?: {
+                [key: string]: unknown;
+            } | null;
+            /**
+             * Identifier
+             * @description The identifier of the annotation. If provided, the annotation will be updated if it already exists.
+             * @default
+             */
+            identifier?: string;
+            /**
+             * Span Id
+             * @description OpenTelemetry Span ID (hex format w/o 0x prefix)
+             */
+            span_id: string;
+            /**
+             * Document Position
+             * @description A 0 based index of the document. E.x. the first document during retrieval is 0
+             */
+            document_position: number;
+        };
         /** SpanEvent */
         SpanEvent: {
             /**
@@ -2351,11 +2430,11 @@ export interface components {
              * @description The end time of the evaluation in ISO format
              */
             end_time: string;
-            /** @description The result of the evaluation */
-            result: components["schemas"]["ExperimentEvaluationResult"];
+            /** @description The result of the evaluation. Either result or error must be provided. */
+            result?: components["schemas"]["ExperimentEvaluationResult"] | null;
             /**
              * Error
-             * @description Optional error message if the evaluation encountered an error
+             * @description Error message if the evaluation encountered an error. Either result or error must be provided.
              */
             error?: string | null;
             /**
@@ -3355,7 +3434,12 @@ export interface operations {
     };
     listExperimentRuns: {
         parameters: {
-            query?: never;
+            query?: {
+                /** @description Cursor for pagination (base64-encoded experiment run ID) */
+                cursor?: string | null;
+                /** @description The max number of experiment runs to return at a time. If not specified, returns all results. */
+                limit?: number | null;
+            };
             header?: never;
             path: {
                 experiment_id: string;
@@ -3391,13 +3475,13 @@ export interface operations {
                     "text/plain": string;
                 };
             };
-            /** @description Validation Error */
+            /** @description Invalid cursor format */
             422: {
                 headers: {
                     [name: string]: unknown;
                 };
                 content: {
-                    "application/json": components["schemas"]["HTTPValidationError"];
+                    "text/plain": string;
                 };
             };
         };
@@ -4587,6 +4671,60 @@ export interface operations {
             };
         };
     };
+    annotateSpanDocuments: {
+        parameters: {
+            query?: {
+                /** @description If set to true, the annotations are inserted synchronously. */
+                sync?: boolean;
+            };
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody: {
+            content: {
+                "application/json": components["schemas"]["AnnotateSpanDocumentsRequestBody"];
+            };
+        };
+        responses: {
+            /** @description Span document annotation inserted successfully */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["AnnotateSpanDocumentsResponseBody"];
+                };
+            };
+            /** @description Forbidden */
+            403: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "text/plain": string;
+                };
+            };
+            /** @description Span not found */
+            404: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "text/plain": string;
+                };
+            };
+            /** @description Invalid request - non-empty identifier not supported */
+            422: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "text/plain": string;
+                };
+            };
+        };
+    };
     getUsers: {
         parameters: {
             query?: {

package/src/experiments/instrumention.ts

@@ -1,10 +1,16 @@
 import { diag, DiagConsoleLogger, DiagLogLevel } from "@opentelemetry/api";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";
 import { resourceFromAttributes } from "@opentelemetry/resources";
-import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
+import {
+  NodeTracerProvider,
+  SpanProcessor,
+} from "@opentelemetry/sdk-trace-node";
 import { SEMRESATTRS_PROJECT_NAME } from "@arizeai/openinference-semantic-conventions";
 import { HeadersOptions } from "openapi-fetch";
-import { OpenInferenceSimpleSpanProcessor } from "@arizeai/openinference-vercel";
+import {
+  OpenInferenceBatchSpanProcessor,
+  OpenInferenceSimpleSpanProcessor,
+} from "@arizeai/openinference-vercel";
 
 /**
  * Creates a provider that exports traces to Phoenix.
@@ -13,9 +19,15 @@ export function createProvider({
   projectName,
   baseUrl,
   headers,
+  useBatchSpanProcessor = true,
 }: {
   projectName: string;
   headers: HeadersOptions;
+  /**
+   * Whether to use batching for the span processor.
+   * @default true
+   */
+  useBatchSpanProcessor: boolean;
   /**
    * The base URL of the Phoenix. Doesn't include the /v1/traces path.
    */
@@ -23,22 +35,23 @@ export function createProvider({
 }) {
   diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.ERROR);
 
+  const exporter = new OTLPTraceExporter({
+    url: `${baseUrl}/v1/traces`,
+    headers: Array.isArray(headers) ? Object.fromEntries(headers) : headers,
+  });
+
+  let spanProcessor: SpanProcessor;
+  if (useBatchSpanProcessor) {
+    spanProcessor = new OpenInferenceBatchSpanProcessor({ exporter });
+  } else {
+    spanProcessor = new OpenInferenceSimpleSpanProcessor({ exporter });
+  }
+
   const provider = new NodeTracerProvider({
     resource: resourceFromAttributes({
       [SEMRESATTRS_PROJECT_NAME]: projectName,
     }),
-    spanProcessors: [
-      // We opt to use the OpenInferenceSimpleSpanProcessor instead of the SimpleSpanProcessor
-      // Since so many AI applications use the AI SDK
-      new OpenInferenceSimpleSpanProcessor({
-        exporter: new OTLPTraceExporter({
-          url: `${baseUrl}/v1/traces`,
-          headers: Array.isArray(headers)
-            ? Object.fromEntries(headers)
-            : headers,
-        }),
-      }),
-    ],
+    spanProcessors: [spanProcessor],
   });
 
   return provider;

package/src/experiments/runExperiment.ts

@@ -94,6 +94,11 @@ export type RunExperimentParams = ClientFn & {
    * @default true
    */
   setGlobalTracerProvider?: boolean;
+  /**
+   * Whether to use batching for the span processor.
+   * @default true
+   */
+  useBatchSpanProcessor?: boolean;
 };
 
 /**
@@ -141,6 +146,7 @@ export async function runExperiment({
   concurrency = 5,
   dryRun = false,
   setGlobalTracerProvider = true,
+  useBatchSpanProcessor = true,
 }: RunExperimentParams): Promise<RanExperiment> {
   let provider: NodeTracerProvider | undefined;
   const isDryRun = typeof dryRun === "number" || dryRun === true;
@@ -201,6 +207,7 @@ export async function runExperiment({
       projectName,
       baseUrl,
       headers: client.config.headers ?? {},
+      useBatchSpanProcessor,
     });
     // Register the provider
     if (setGlobalTracerProvider) {
@@ -276,6 +283,7 @@ export async function runExperiment({
     concurrency,
     dryRun,
     setGlobalTracerProvider,
+    useBatchSpanProcessor,
   });
   ranExperiment.evaluationRuns = evaluationRuns;
 
@@ -421,6 +429,7 @@ export async function evaluateExperiment({
   concurrency = 5,
   dryRun = false,
   setGlobalTracerProvider = true,
+  useBatchSpanProcessor = true,
 }: {
   /**
    * The experiment to evaluate
@@ -445,6 +454,11 @@ export async function evaluateExperiment({
    * @default true
    */
   setGlobalTracerProvider?: boolean;
+  /**
+   * Whether to use batching for the span processor.
+   * @default true
+   */
+  useBatchSpanProcessor?: boolean;
 }): Promise<RanExperiment> {
   const isDryRun = typeof dryRun === "number" || dryRun === true;
   const client = _client ?? createClient();
@@ -459,6 +473,7 @@ export async function evaluateExperiment({
       projectName: "evaluators",
       baseUrl,
       headers: client.config.headers ?? {},
+      useBatchSpanProcessor,
     });
     if (setGlobalTracerProvider) {
       provider.register();

package/src/schemas/llm/converters.ts

@@ -38,7 +38,7 @@ export const safelyConvertMessageToProvider = <
     );
     // convert the OpenAI format to the target provider format
     return fromOpenAIMessage({ message: openAIMessage, targetProvider });
-  } catch (e) {
+  } catch {
     return null;
   }
 };
@@ -64,7 +64,7 @@ export const safelyConvertToolCallToProvider = <
       toolCall: openAIToolCall,
       targetProvider,
     });
-  } catch (e) {
+  } catch {
     return null;
   }
 };
@@ -90,7 +90,7 @@ export const safelyConvertToolDefinitionToProvider = <
       toolDefinition: openAIToolDefinition,
       targetProvider,
     });
-  } catch (e) {
+  } catch {
     return null;
   }
 };
@@ -116,7 +116,7 @@ export const safelyConvertToolChoiceToProvider = <
       toolChoice: openAIToolChoice,
       targetProvider,
     });
-  } catch (e) {
+  } catch {
     return null;
   }
 };

package/src/spans/addDocumentAnnotation.ts

@@ -0,0 +1,66 @@
+import { createClient } from "../client";
+import { ClientFn } from "../types/core";
+import { DocumentAnnotation, toDocumentAnnotationData } from "./types";
+
+/**
+ * Parameters to add a document annotation
+ */
+export interface AddDocumentAnnotationParams extends ClientFn {
+  documentAnnotation: DocumentAnnotation;
+  /**
+   * If true, the request will be fulfilled synchronously and return the annotation ID.
+   * If false, the request will be processed asynchronously and return null.
+   * @default false
+   */
+  sync?: boolean;
+}
+
+/**
+ * Add an annotation to a document within a span.
+ *
+ * The annotation can be of type "LLM", "CODE", or "HUMAN" and can include a label, score, explanation, and metadata.
+ * At least one of label, score, or explanation must be provided.
+ *
+ * @param params - The parameters to add a document annotation
+ * @returns The ID of the created annotation
+ *
+ * @example
+ * ```ts
+ * const result = await addDocumentAnnotation({
+ *   documentAnnotation: {
+ *     spanId: "123abc",
+ *     documentPosition: 0,
+ *     name: "relevance_score",
+ *     label: "relevant",
+ *     score: 0.95,
+ *     annotatorKind: "LLM",
+ *     explanation: "Document is highly relevant to the query",
+ *     metadata: {
+ *       model: "gpt-4"
+ *     }
+ *   }
+ * });
+ * ```
+ */
+export async function addDocumentAnnotation({
+  client: _client,
+  documentAnnotation,
+  sync = false,
+}: AddDocumentAnnotationParams): Promise<{ id: string } | null> {
+  const client = _client ?? createClient();
+
+  const { data, error } = await client.POST("/v1/document_annotations", {
+    params: {
+      query: { sync },
+    },
+    body: {
+      data: [toDocumentAnnotationData(documentAnnotation)],
+    },
+  });
+
+  if (error) {
+    throw new Error(`Failed to add document annotation: ${error}`);
+  }
+
+  return data?.data?.[0] || null;
+}

package/src/spans/addSpanAnnotation.ts

@@ -7,6 +7,12 @@ import { SpanAnnotation, toSpanAnnotationData } from "./types";
  */
 export interface AddSpanAnnotationParams extends ClientFn {
   spanAnnotation: SpanAnnotation;
+  /**
+   * If true, the request will be fulfilled synchronously and return the annotation ID.
+   * If false, the request will be processed asynchronously and return null.
+   * @default false
+   */
+  sync?: boolean;
 }
 
 /**
@@ -38,10 +44,14 @@ export interface AddSpanAnnotationParams extends ClientFn {
 export async function addSpanAnnotation({
   client: _client,
   spanAnnotation,
-}: AddSpanAnnotationParams): Promise<{ id: string }> {
+  sync = false,
+}: AddSpanAnnotationParams): Promise<{ id: string } | null> {
   const client = _client ?? createClient();
 
   const { data, error } = await client.POST("/v1/span_annotations", {
+    params: {
+      query: { sync },
+    },
     body: {
       data: [toSpanAnnotationData(spanAnnotation)],
     },
@@ -51,9 +61,5 @@ export async function addSpanAnnotation({
     throw new Error(`Failed to add span annotation: ${error}`);
   }
 
-  if (!data?.data?.[0]?.id) {
-    throw new Error("No annotation ID returned from server");
-  }
-
-  return data.data[0];
+  return data?.data?.[0] || null;
 }

package/src/spans/index.ts

@@ -1,5 +1,7 @@
 export * from "./addSpanAnnotation";
 export * from "./logSpanAnnotations";
+export * from "./addDocumentAnnotation";
+export * from "./logDocumentAnnotations";
 export * from "./getSpans";
 export * from "./getSpanAnnotations";
 export * from "./deleteSpan";