@arizeai/phoenix-client 3.1.0 → 4.0.0

package/src/__generated__/api/v1.ts

@@ -290,7 +290,10 @@ export interface paths {
             path?: never;
             cookie?: never;
         };
-        /** List runs for an experiment */
+        /**
+         * List runs for an experiment
+         * @description Retrieve a paginated list of runs for an experiment
+         */
         get: operations["listExperimentRuns"];
         put?: never;
         /** Create run for an experiment */
@@ -633,6 +636,23 @@ export interface paths {
         patch?: never;
         trace?: never;
     };
+    "/v1/document_annotations": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        get?: never;
+        put?: never;
+        /** Annotate Span Documents */
+        post: operations["annotateSpanDocuments"];
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
     "/v1/users": {
         parameters: {
             query?: never;
@@ -681,6 +701,16 @@ export interface paths {
 export type webhooks = Record<string, never>;
 export interface components {
     schemas: {
+        /** AnnotateSpanDocumentsRequestBody */
+        AnnotateSpanDocumentsRequestBody: {
+            /** Data */
+            data: components["schemas"]["SpanDocumentAnnotationData"][];
+        };
+        /** AnnotateSpanDocumentsResponseBody */
+        AnnotateSpanDocumentsResponseBody: {
+            /** Data */
+            data: components["schemas"]["InsertedSpanDocumentAnnotation"][];
+        };
         /** AnnotateSpansRequestBody */
         AnnotateSpansRequestBody: {
             /** Data */
@@ -691,6 +721,24 @@ export interface components {
             /** Data */
             data: components["schemas"]["InsertedSpanAnnotation"][];
         };
+        /** AnnotationResult */
+        AnnotationResult: {
+            /**
+             * Label
+             * @description The label assigned by the annotation
+             */
+            label?: string | null;
+            /**
+             * Score
+             * @description The score assigned by the annotation
+             */
+            score?: number | null;
+            /**
+             * Explanation
+             * @description Explanation of the annotation result
+             */
+            explanation?: string | null;
+        };
         /** CategoricalAnnotationConfig */
         CategoricalAnnotationConfig: {
             /** Name */
@@ -1228,6 +1276,14 @@ export interface components {
              */
             id: string;
         };
+        /** InsertedSpanDocumentAnnotation */
+        InsertedSpanDocumentAnnotation: {
+            /**
+             * Id
+             * @description The ID of the inserted span document annotation
+             */
+            id: string;
+        };
         /** ListDatasetExamplesData */
         ListDatasetExamplesData: {
             /** Dataset Id */
@@ -1259,6 +1315,8 @@ export interface components {
         ListExperimentRunsResponseBody: {
             /** Data */
             data: components["schemas"]["ExperimentRunResponse"][];
+            /** Next Cursor */
+            next_cursor: string | null;
         };
         /** ListExperimentsResponseBody */
         ListExperimentsResponseBody: {
@@ -2097,11 +2155,25 @@ export interface components {
         };
         /** SpanAnnotation */
         SpanAnnotation: {
+            /** Id */
+            id: string;
             /**
-             * Span Id
-             * @description OpenTelemetry Span ID (hex format w/o 0x prefix)
+             * Created At
+             * Format: date-time
              */
-            span_id: string;
+            created_at: string;
+            /**
+             * Updated At
+             * Format: date-time
+             */
+            updated_at: string;
+            /**
+             * Source
+             * @enum {string}
+             */
+            source: "API" | "APP";
+            /** User Id */
+            user_id: string | null;
             /**
              * Name
              * @description The name of the annotation
@@ -2114,7 +2186,7 @@ export interface components {
              */
             annotator_kind: "LLM" | "CODE" | "HUMAN";
             /** @description The result of the annotation */
-            result?: components["schemas"]["SpanAnnotationResult"] | null;
+            result?: components["schemas"]["AnnotationResult"] | null;
             /**
              * Metadata
              * @description Metadata for the annotation
@@ -2128,33 +2200,14 @@ export interface components {
              * @default
              */
             identifier?: string;
-            /** Id */
-            id: string;
-            /**
-             * Created At
-             * Format: date-time
-             */
-            created_at: string;
-            /**
-             * Updated At
-             * Format: date-time
-             */
-            updated_at: string;
-            /**
-             * Source
-             * @enum {string}
-             */
-            source: "API" | "APP";
-            /** User Id */
-            user_id: string | null;
-        };
-        /** SpanAnnotationData */
-        SpanAnnotationData: {
             /**
              * Span Id
              * @description OpenTelemetry Span ID (hex format w/o 0x prefix)
              */
             span_id: string;
+        };
+        /** SpanAnnotationData */
+        SpanAnnotationData: {
             /**
              * Name
              * @description The name of the annotation
@@ -2167,7 +2220,7 @@ export interface components {
              */
             annotator_kind: "LLM" | "CODE" | "HUMAN";
             /** @description The result of the annotation */
-            result?: components["schemas"]["SpanAnnotationResult"] | null;
+            result?: components["schemas"]["AnnotationResult"] | null;
             /**
              * Metadata
              * @description Metadata for the annotation
@@ -2181,24 +2234,11 @@ export interface components {
              * @default
              */
             identifier?: string;
-        };
-        /** SpanAnnotationResult */
-        SpanAnnotationResult: {
-            /**
-             * Label
-             * @description The label assigned by the annotation
-             */
-            label?: string | null;
             /**
-             * Score
-             * @description The score assigned by the annotation
-             */
-            score?: number | null;
-            /**
-             * Explanation
-             * @description Explanation of the annotation result
+             * Span Id
+             * @description OpenTelemetry Span ID (hex format w/o 0x prefix)
              */
-            explanation?: string | null;
+            span_id: string;
         };
         /** SpanAnnotationsResponseBody */
         SpanAnnotationsResponseBody: {
@@ -2220,6 +2260,45 @@ export interface components {
              */
             span_id: string;
         };
+        /** SpanDocumentAnnotationData */
+        SpanDocumentAnnotationData: {
+            /**
+             * Name
+             * @description The name of the annotation
+             */
+            name: string;
+            /**
+             * Annotator Kind
+             * @description The kind of annotator used for the annotation
+             * @enum {string}
+             */
+            annotator_kind: "LLM" | "CODE" | "HUMAN";
+            /** @description The result of the annotation */
+            result?: components["schemas"]["AnnotationResult"] | null;
+            /**
+             * Metadata
+             * @description Metadata for the annotation
+             */
+            metadata?: {
+                [key: string]: unknown;
+            } | null;
+            /**
+             * Identifier
+             * @description The identifier of the annotation. If provided, the annotation will be updated if it already exists.
+             * @default
+             */
+            identifier?: string;
+            /**
+             * Span Id
+             * @description OpenTelemetry Span ID (hex format w/o 0x prefix)
+             */
+            span_id: string;
+            /**
+             * Document Position
+             * @description A 0 based index of the document. E.x. the first document during retrieval is 0
+             */
+            document_position: number;
+        };
         /** SpanEvent */
         SpanEvent: {
             /**
@@ -3355,7 +3434,12 @@ export interface operations {
     };
     listExperimentRuns: {
         parameters: {
-            query?: never;
+            query?: {
+                /** @description Cursor for pagination (base64-encoded experiment run ID) */
+                cursor?: string | null;
+                /** @description The max number of experiment runs to return at a time. If not specified, returns all results. */
+                limit?: number | null;
+            };
             header?: never;
             path: {
                 experiment_id: string;
@@ -3391,13 +3475,13 @@ export interface operations {
                     "text/plain": string;
                 };
             };
-            /** @description Validation Error */
+            /** @description Invalid cursor format */
             422: {
                 headers: {
                     [name: string]: unknown;
                 };
                 content: {
-                    "application/json": components["schemas"]["HTTPValidationError"];
+                    "text/plain": string;
                 };
             };
         };
@@ -4587,6 +4671,60 @@ export interface operations {
             };
         };
     };
+    annotateSpanDocuments: {
+        parameters: {
+            query?: {
+                /** @description If set to true, the annotations are inserted synchronously. */
+                sync?: boolean;
+            };
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody: {
+            content: {
+                "application/json": components["schemas"]["AnnotateSpanDocumentsRequestBody"];
+            };
+        };
+        responses: {
+            /** @description Span document annotation inserted successfully */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["AnnotateSpanDocumentsResponseBody"];
+                };
+            };
+            /** @description Forbidden */
+            403: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "text/plain": string;
+                };
+            };
+            /** @description Span not found */
+            404: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "text/plain": string;
+                };
+            };
+            /** @description Invalid request - non-empty identifier not supported */
+            422: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "text/plain": string;
+                };
+            };
+        };
+    };
     getUsers: {
         parameters: {
             query?: {

package/src/prompts/sdks/toAI.ts

@@ -2,22 +2,23 @@ import invariant from "tiny-invariant";
 import {
   safelyConvertMessageToProvider,
   safelyConvertToolChoiceToProvider,
-  safelyConvertToolDefinitionToProvider,
 } from "../../schemas/llm/converters";
 import { formatPromptMessages } from "../../utils/formatPromptMessages";
 import { Variables, toSDKParamsBase } from "./types";
-import {
-  type streamText,
-  type generateText,
-  type ToolSet,
-  type Tool,
-} from "ai";
+import { type ToolSet, type ModelMessage, type ToolChoice } from "ai";
 import { VercelAIToolChoice } from "../../schemas/llm/vercel/toolChoiceSchemas";
 
-export type PartialStreamTextParams = Omit<
-  Parameters<typeof streamText>[0] | Parameters<typeof generateText>[0],
-  "model"
->;
+export type PartialAIParams = {
+  messages: ModelMessage[];
+  /**
+The tools that the model can call. The model needs to support calling tools.
+    */
+  tools?: ToolSet;
+  /**
+The tool choice strategy. Default: 'auto'.
+     */
+  toolChoice?: ToolChoice<ToolSet>;
+};
 
 export type ToAIParams<V extends Variables> = toSDKParamsBase<V>;
 
@@ -30,16 +31,16 @@ export type ToAIParams<V extends Variables> = toSDKParamsBase<V>;
 export const toAI = <V extends Variables>({
   prompt,
   variables,
-}: ToAIParams<V>): PartialStreamTextParams | null => {
+}: ToAIParams<V>): PartialAIParams | null => {
   // eslint-disable-next-line no-console
   console.warn(
     "Prompt invocation parameters not currently supported in AI SDK, falling back to default invocation parameters"
   );
   try {
     // parts of the prompt that can be directly converted to OpenAI params
-    const baseCompletionParams = {
+    const baseCompletionParams: Partial<PartialAIParams> = {
       // Invocation parameters are validated on the phoenix-side
-    } satisfies Partial<PartialStreamTextParams>;
+    };
 
     if (!("messages" in prompt.template)) {
       return null;
@@ -55,7 +56,7 @@ export const toAI = <V extends Variables>({
       );
     }
 
-    const messages = formattedMessages.map((message) => {
+    const messages: ModelMessage[] = formattedMessages.map((message) => {
       const vercelAIMessage = safelyConvertMessageToProvider({
         message,
         targetProvider: "VERCEL_AI",
@@ -65,21 +66,30 @@ export const toAI = <V extends Variables>({
     });
 
     // convert tools to Vercel AI tool set, which is a map of tool name to tool
-    let tools: ToolSet | undefined = prompt.tools?.tools.reduce((acc, tool) => {
-      if (!tool.function.parameters) {
-        return acc;
-      }
-      const vercelAIToolDefinition = safelyConvertToolDefinitionToProvider({
-        toolDefinition: tool,
-        targetProvider: "VERCEL_AI",
-      });
-      invariant(vercelAIToolDefinition, "Tool definition is not valid");
-      acc[tool.function.name] = vercelAIToolDefinition satisfies Tool;
-      return acc;
-    }, {} as ToolSet);
-    const hasTools = Object.keys(tools ?? {}).length > 0;
-    tools = hasTools ? tools : undefined;
-
+    // TODO: Vercel AI SDK 5 has complex tool schema
+    // let tools: ToolSet | undefined = prompt.tools?.tools.reduce((acc, tool) => {
+    //   if (!tool.function.parameters) {
+    //     return acc;
+    //   }
+    //   const vercelAIToolDefinition = safelyConvertToolDefinitionToProvider({
+    //     toolDefinition: tool,
+    //     targetProvider: "VERCEL_AI",
+    //   });
+    //   invariant(vercelAIToolDefinition, "Tool definition is not valid");
+    //   // TODO: get the symbol working here for validators
+    //   acc[tool.function.name] = vercelAIToolDefinition as unknown as Tool;
+    //   return acc;
+    // }, {} as ToolSet);
+    // const hasTools = Object.keys(tools ?? {}).length > 0;
+    // tools = hasTools ? tools : undefined;
+    const hasTools = false;
+    const tools = undefined;
+    if (prompt.tools?.tools && prompt.tools?.tools.length) {
+      // eslint-disable-next-line no-console
+      console.warn(
+        "Prompt tools not currently supported in the AI SDK, falling back to no tools"
+      );
+    }
     let toolChoice: VercelAIToolChoice | undefined =
       safelyConvertToolChoiceToProvider({
         toolChoice: prompt.tools?.tool_choice,
@@ -88,12 +98,12 @@ export const toAI = <V extends Variables>({
     toolChoice = hasTools ? toolChoice : undefined;
 
     // combine base and computed params
-    const completionParams = {
+    const completionParams: PartialAIParams = {
       ...baseCompletionParams,
       messages,
       tools,
       toolChoice,
-    } satisfies Partial<PartialStreamTextParams>;
+    };
 
     return completionParams;
   } catch (error) {

package/src/schemas/llm/openai/converters.ts

@@ -272,7 +272,7 @@ export const openAIMessageToVercelAI = openAIMessageSchema.transform(
               type: "tool-call",
               toolCallId: tc.id,
               toolName: tc.function.name,
-              args: tc.function.arguments,
+              input: tc.function.arguments,
             });
           });
         }
@@ -289,7 +289,7 @@ export const openAIMessageToVercelAI = openAIMessageSchema.transform(
             type: "tool-result",
             toolCallId: openai.tool_call_id,
             toolName: "", // We don't have this??
-            result: openai.content,
+            output: { type: "text", value: openai.content },
           });
         } else {
           openai.content.forEach((part) => {
@@ -298,7 +298,10 @@ export const openAIMessageToVercelAI = openAIMessageSchema.transform(
                 type: "tool-result",
                 toolCallId: openai.tool_call_id,
                 toolName: "", // We don't have this??
-                result: part.text,
+                output: {
+                  type: "text",
+                  value: part.text,
+                },
               });
               return;
             }
@@ -391,7 +394,7 @@ export const openAIToolDefinitionToVercelAI =
     (openai): VercelAIToolDefinition => ({
       type: "function",
       description: openai.function.description,
-      parameters: {
+      inputSchema: {
         _type: undefined,
         jsonSchema: openai.function.parameters,
         validate: undefined,

package/src/schemas/llm/vercel/messagePartSchemas.ts

@@ -26,7 +26,7 @@ export const vercelAIChatPartToolCallSchema = z.object({
   type: z.literal("tool-call"),
   toolCallId: z.string(),
   toolName: z.string(),
-  args: jsonLiteralSchema, // json serializable parameters
+  input: jsonLiteralSchema, // json serializable parameters
 });
 
 export type VercelAIChatPartToolCall = z.infer<
@@ -37,7 +37,10 @@ export const vercelAIChatPartToolResultSchema = z.object({
   type: z.literal("tool-result"),
   toolCallId: z.string(),
   toolName: z.string(),
-  result: jsonLiteralSchema, // json serializable result
+  output: z.object({
+    type: z.literal("text"), // TODO: extend to support other output types
+    value: z.string(),
+  }),
 });
 
 export type VercelAIChatPartToolResult = z.infer<

package/src/schemas/llm/vercel/toolSchemas.ts

@@ -9,7 +9,7 @@ import { z } from "zod";
 export const vercelAIToolDefinitionSchema = z.object({
   type: z.literal("function"),
   description: z.string().optional(),
-  parameters: z.object({
+  inputSchema: z.object({
     _type: z.unknown().optional().default(undefined),
     validate: z.unknown().optional().default(undefined),
     jsonSchema: z.record(z.string(), z.unknown()).optional(),

package/src/spans/addDocumentAnnotation.ts

@@ -0,0 +1,66 @@
+import { createClient } from "../client";
+import { ClientFn } from "../types/core";
+import { DocumentAnnotation, toDocumentAnnotationData } from "./types";
+
+/**
+ * Parameters to add a document annotation
+ */
+export interface AddDocumentAnnotationParams extends ClientFn {
+  documentAnnotation: DocumentAnnotation;
+  /**
+   * If true, the request will be fulfilled synchronously and return the annotation ID.
+   * If false, the request will be processed asynchronously and return null.
+   * @default false
+   */
+  sync?: boolean;
+}
+
+/**
+ * Add an annotation to a document within a span.
+ *
+ * The annotation can be of type "LLM", "CODE", or "HUMAN" and can include a label, score, explanation, and metadata.
+ * At least one of label, score, or explanation must be provided.
+ *
+ * @param params - The parameters to add a document annotation
+ * @returns The ID of the created annotation
+ *
+ * @example
+ * ```ts
+ * const result = await addDocumentAnnotation({
+ *   documentAnnotation: {
+ *     spanId: "123abc",
+ *     documentPosition: 0,
+ *     name: "relevance_score",
+ *     label: "relevant",
+ *     score: 0.95,
+ *     annotatorKind: "LLM",
+ *     explanation: "Document is highly relevant to the query",
+ *     metadata: {
+ *       model: "gpt-4"
+ *     }
+ *   }
+ * });
+ * ```
+ */
+export async function addDocumentAnnotation({
+  client: _client,
+  documentAnnotation,
+  sync = false,
+}: AddDocumentAnnotationParams): Promise<{ id: string } | null> {
+  const client = _client ?? createClient();
+
+  const { data, error } = await client.POST("/v1/document_annotations", {
+    params: {
+      query: { sync },
+    },
+    body: {
+      data: [toDocumentAnnotationData(documentAnnotation)],
+    },
+  });
+
+  if (error) {
+    throw new Error(`Failed to add document annotation: ${error}`);
+  }
+
+  return data?.data?.[0] || null;
+}

package/src/spans/addSpanAnnotation.ts

@@ -7,6 +7,12 @@ import { SpanAnnotation, toSpanAnnotationData } from "./types";
  */
 export interface AddSpanAnnotationParams extends ClientFn {
   spanAnnotation: SpanAnnotation;
+  /**
+   * If true, the request will be fulfilled synchronously and return the annotation ID.
+   * If false, the request will be processed asynchronously and return null.
+   * @default false
+   */
+  sync?: boolean;
 }
 
 /**
@@ -38,10 +44,14 @@ export interface AddSpanAnnotationParams extends ClientFn {
 export async function addSpanAnnotation({
   client: _client,
   spanAnnotation,
-}: AddSpanAnnotationParams): Promise<{ id: string }> {
+  sync = false,
+}: AddSpanAnnotationParams): Promise<{ id: string } | null> {
   const client = _client ?? createClient();
 
   const { data, error } = await client.POST("/v1/span_annotations", {
+    params: {
+      query: { sync },
+    },
     body: {
       data: [toSpanAnnotationData(spanAnnotation)],
     },
@@ -51,9 +61,5 @@ export async function addSpanAnnotation({
     throw new Error(`Failed to add span annotation: ${error}`);
   }
 
-  if (!data?.data?.[0]?.id) {
-    throw new Error("No annotation ID returned from server");
-  }
-
-  return data.data[0];
+  return data?.data?.[0] || null;
 }

package/src/spans/index.ts

@@ -1,5 +1,7 @@
 export * from "./addSpanAnnotation";
 export * from "./logSpanAnnotations";
+export * from "./addDocumentAnnotation";
+export * from "./logDocumentAnnotations";
 export * from "./getSpans";
 export * from "./getSpanAnnotations";
 export * from "./deleteSpan";