@arizeai/phoenix-client 3.2.0 → 4.0.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arizeai/phoenix-client",
-  "version": "3.2.0",
+  "version": "4.0.1",
   "description": "A client for the Phoenix API",
   "main": "dist/src/index.js",
   "module": "dist/esm/index.js",
@@ -53,11 +53,11 @@
   "author": "oss@arize.com",
   "license": "ELv2",
   "devDependencies": {
-    "@ai-sdk/openai": "^1.1.15",
+    "@ai-sdk/openai": "^2.0.27",
     "@anthropic-ai/sdk": "^0.35.0",
     "@types/async": "^3.2.24",
     "@types/node": "^20.17.22",
-    "ai": "^4.1.24",
+    "ai": "^5.0.38",
     "openai": "^4.77.0",
     "openapi-typescript": "^7.6.1",
     "tsx": "^4.19.3",
@@ -84,7 +84,7 @@
   },
   "optionalDependencies": {
     "@anthropic-ai/sdk": "^0.35.0",
-    "ai": "^4.1.47",
+    "ai": "^5.0.38",
     "openai": "^5.12.1"
   },
   "scripts": {

package/src/client.ts

@@ -1,4 +1,7 @@
-import createOpenApiClient, { type ClientOptions } from "openapi-fetch";
+import createOpenApiClient, {
+  type Middleware,
+  type ClientOptions,
+} from "openapi-fetch";
 import type {
   paths as oapiPathsV1,
   components as oapiComponentsV1,
@@ -49,6 +52,20 @@ export const getMergedOptions = ({
   };
 };
 
+/**
+ * Middleware to take non-successful API calls throw instead of being swallowed
+ */
+const middleware: Middleware = {
+  onResponse({ response }) {
+    if (!response.ok) {
+      // Will produce error messages like "https://example.org/api/v1/example: 404 Not Found".
+      throw new Error(
+        `${response.url}: ${response.status} ${response.statusText}`
+      );
+    }
+  },
+};
+
 /**
  * Create a Phoenix client.
  *
@@ -79,8 +96,10 @@ export const createClient = (
   } = {}
 ) => {
   const mergedOptions = getMergedOptions(config);
+  const openApiClient = createOpenApiClient<pathsV1>(mergedOptions);
+  openApiClient.use(middleware);
   return {
-    ...createOpenApiClient<pathsV1>(mergedOptions),
+    ...openApiClient,
     config: mergedOptions,
   };
 };

package/src/datasets/appendDatasetExamples.ts

@@ -33,7 +33,7 @@ export async function appendDatasetExamples({
   const client = _client || createClient();
   const inputs = examples.map((example) => example.input);
   const outputs = examples.map((example) => example.output ?? {}); // Treat null as an empty object
-  const metadata = examples.map((example) => example.metadata);
+  const metadata = examples.map((example) => example.metadata ?? {});
   let datasetName: string;
   if ("datasetName" in dataset) {
     datasetName = dataset.datasetName;

package/src/datasets/createDataset.ts

@@ -34,8 +34,8 @@ export async function createDataset({
 }: CreateDatasetParams): Promise<CreateDatasetResponse> {
   const client = _client || createClient();
   const inputs = examples.map((example) => example.input);
-  const outputs = examples.map((example) => example.output ?? {}); // Treat null as an empty object
-  const metadata = examples.map((example) => example.metadata);
+  const outputs = examples.map((example) => example?.output ?? {}); // Treat null as an empty object
+  const metadata = examples.map((example) => example?.metadata ?? {});
   const createDatasetResponse = await client.POST("/v1/datasets/upload", {
     params: {
       query: {

package/src/experiments/runExperiment.ts

@@ -95,6 +95,11 @@ export type RunExperimentParams = ClientFn & {
    */
   setGlobalTracerProvider?: boolean;
   /**
+   * Number of times to repeat each dataset example
+   * @default 1
+   */
+  repetitions?: number;
+  /*
    * Whether to use batching for the span processor.
    * @default true
    */
@@ -146,6 +151,7 @@ export async function runExperiment({
   concurrency = 5,
   dryRun = false,
   setGlobalTracerProvider = true,
+  repetitions = 1,
   useBatchSpanProcessor = true,
 }: RunExperimentParams): Promise<RanExperiment> {
   let provider: NodeTracerProvider | undefined;
@@ -185,6 +191,7 @@ export async function runExperiment({
           description: experimentDescription,
           metadata: experimentMetadata,
           project_name: projectName,
+          repetitions,
         },
       })
       .then((res) => res.data?.data);
@@ -262,6 +269,7 @@ export async function runExperiment({
     isDryRun,
     nExamples,
     tracer: taskTracer,
+    repetitions,
   });
   logger.info(`✅ Task runs completed`);
 
@@ -315,6 +323,7 @@ function runTaskWithExamples({
   isDryRun,
   nExamples,
   tracer,
+  repetitions = 1,
 }: {
   /** The client to use */
   client: PhoenixClient;
@@ -336,9 +345,17 @@ function runTaskWithExamples({
   nExamples: number;
   /** TraceProvider instance that will be used to create spans from task calls */
   tracer: Tracer;
+  /** Number of repetitions per example */
+  repetitions?: number;
 }): Promise<void> {
   logger.info(`🔧 Running task "${task.name}" on dataset "${dataset.id}"`);
-  const run = async (example: ExampleWithId) => {
+  const run = async ({
+    example,
+    repetitionNumber,
+  }: {
+    example: ExampleWithId;
+    repetitionNumber: number;
+  }) => {
     return tracer.startActiveSpan(`Task: ${task.name}`, async (span) => {
       logger.info(
         `🔧 Running task "${task.name}" on example "${example.id} of dataset "${dataset.id}"`
@@ -374,7 +391,7 @@ function runTaskWithExamples({
           body: {
             dataset_example_id: example.id,
             output: thisRun.output,
-            repetition_number: 0,
+            repetition_number: repetitionNumber,
             start_time: thisRun.startTime.toISOString(),
             end_time: thisRun.endTime.toISOString(),
             trace_id: thisRun.traceId,
@@ -404,15 +421,23 @@ function runTaskWithExamples({
   };
   const q = queue(run, concurrency);
   const examplesToUse = dataset.examples.slice(0, nExamples);
-  examplesToUse.forEach((example) =>
-    q.push(example, (err) => {
-      if (err) {
-        logger.error(
-          `Error running task "${task.name}" on example "${example.id}": ${err}`
-        );
-      }
-    })
-  );
+
+  examplesToUse
+    .flatMap((example) =>
+      Array.from({ length: repetitions }, (_, index) => ({
+        example,
+        repetitionNumber: index,
+      }))
+    )
+    .forEach((exampleWithRepetition) =>
+      q.push(exampleWithRepetition, (err) => {
+        if (err) {
+          logger.error(
+            `Error running task "${task.name}" on example "${exampleWithRepetition.example.id}" repetition ${exampleWithRepetition.repetitionNumber}: ${err}`
+          );
+        }
+      })
+    );
   return q.drain();
 }
 
@@ -674,7 +699,7 @@ async function runEvaluator({
         input: example.input,
         output: run.output ?? null,
         expected: example.output,
-        metadata: example.metadata,
+        metadata: example?.metadata,
       });
       thisEval.result = result;
       logger.info(

package/src/prompts/sdks/toAI.ts

@@ -2,22 +2,23 @@ import invariant from "tiny-invariant";
 import {
   safelyConvertMessageToProvider,
   safelyConvertToolChoiceToProvider,
-  safelyConvertToolDefinitionToProvider,
 } from "../../schemas/llm/converters";
 import { formatPromptMessages } from "../../utils/formatPromptMessages";
 import { Variables, toSDKParamsBase } from "./types";
-import {
-  type streamText,
-  type generateText,
-  type ToolSet,
-  type Tool,
-} from "ai";
+import { type ToolSet, type ModelMessage, type ToolChoice } from "ai";
 import { VercelAIToolChoice } from "../../schemas/llm/vercel/toolChoiceSchemas";
 
-export type PartialStreamTextParams = Omit<
-  Parameters<typeof streamText>[0] | Parameters<typeof generateText>[0],
-  "model"
->;
+export type PartialAIParams = {
+  messages: ModelMessage[];
+  /**
+The tools that the model can call. The model needs to support calling tools.
+    */
+  tools?: ToolSet;
+  /**
+The tool choice strategy. Default: 'auto'.
+     */
+  toolChoice?: ToolChoice<ToolSet>;
+};
 
 export type ToAIParams<V extends Variables> = toSDKParamsBase<V>;
 
@@ -30,16 +31,16 @@ export type ToAIParams<V extends Variables> = toSDKParamsBase<V>;
 export const toAI = <V extends Variables>({
   prompt,
   variables,
-}: ToAIParams<V>): PartialStreamTextParams | null => {
+}: ToAIParams<V>): PartialAIParams | null => {
   // eslint-disable-next-line no-console
   console.warn(
     "Prompt invocation parameters not currently supported in AI SDK, falling back to default invocation parameters"
   );
   try {
     // parts of the prompt that can be directly converted to OpenAI params
-    const baseCompletionParams = {
+    const baseCompletionParams: Partial<PartialAIParams> = {
       // Invocation parameters are validated on the phoenix-side
-    } satisfies Partial<PartialStreamTextParams>;
+    };
 
     if (!("messages" in prompt.template)) {
       return null;
@@ -55,7 +56,7 @@ export const toAI = <V extends Variables>({
       );
     }
 
-    const messages = formattedMessages.map((message) => {
+    const messages: ModelMessage[] = formattedMessages.map((message) => {
       const vercelAIMessage = safelyConvertMessageToProvider({
         message,
         targetProvider: "VERCEL_AI",
@@ -65,21 +66,30 @@ export const toAI = <V extends Variables>({
     });
 
     // convert tools to Vercel AI tool set, which is a map of tool name to tool
-    let tools: ToolSet | undefined = prompt.tools?.tools.reduce((acc, tool) => {
-      if (!tool.function.parameters) {
-        return acc;
-      }
-      const vercelAIToolDefinition = safelyConvertToolDefinitionToProvider({
-        toolDefinition: tool,
-        targetProvider: "VERCEL_AI",
-      });
-      invariant(vercelAIToolDefinition, "Tool definition is not valid");
-      acc[tool.function.name] = vercelAIToolDefinition satisfies Tool;
-      return acc;
-    }, {} as ToolSet);
-    const hasTools = Object.keys(tools ?? {}).length > 0;
-    tools = hasTools ? tools : undefined;
-
+    // TODO: Vercel AI SDK 5 has complex tool schema
+    // let tools: ToolSet | undefined = prompt.tools?.tools.reduce((acc, tool) => {
+    //   if (!tool.function.parameters) {
+    //     return acc;
+    //   }
+    //   const vercelAIToolDefinition = safelyConvertToolDefinitionToProvider({
+    //     toolDefinition: tool,
+    //     targetProvider: "VERCEL_AI",
+    //   });
+    //   invariant(vercelAIToolDefinition, "Tool definition is not valid");
+    //   // TODO: get the symbol working here for validators
+    //   acc[tool.function.name] = vercelAIToolDefinition as unknown as Tool;
+    //   return acc;
+    // }, {} as ToolSet);
+    // const hasTools = Object.keys(tools ?? {}).length > 0;
+    // tools = hasTools ? tools : undefined;
+    const hasTools = false;
+    const tools = undefined;
+    if (prompt.tools?.tools && prompt.tools?.tools.length) {
+      // eslint-disable-next-line no-console
+      console.warn(
+        "Prompt tools not currently supported in the AI SDK, falling back to no tools"
+      );
+    }
     let toolChoice: VercelAIToolChoice | undefined =
       safelyConvertToolChoiceToProvider({
         toolChoice: prompt.tools?.tool_choice,
@@ -88,12 +98,12 @@ export const toAI = <V extends Variables>({
     toolChoice = hasTools ? toolChoice : undefined;
 
     // combine base and computed params
-    const completionParams = {
+    const completionParams: PartialAIParams = {
       ...baseCompletionParams,
       messages,
       tools,
       toolChoice,
-    } satisfies Partial<PartialStreamTextParams>;
+    };
 
     return completionParams;
   } catch (error) {

package/src/schemas/llm/openai/converters.ts

@@ -272,7 +272,7 @@ export const openAIMessageToVercelAI = openAIMessageSchema.transform(
               type: "tool-call",
               toolCallId: tc.id,
               toolName: tc.function.name,
-              args: tc.function.arguments,
+              input: tc.function.arguments,
             });
           });
         }
@@ -289,7 +289,7 @@ export const openAIMessageToVercelAI = openAIMessageSchema.transform(
             type: "tool-result",
             toolCallId: openai.tool_call_id,
             toolName: "", // We don't have this??
-            result: openai.content,
+            output: { type: "text", value: openai.content },
           });
         } else {
           openai.content.forEach((part) => {
@@ -298,7 +298,10 @@ export const openAIMessageToVercelAI = openAIMessageSchema.transform(
                 type: "tool-result",
                 toolCallId: openai.tool_call_id,
                 toolName: "", // We don't have this??
-                result: part.text,
+                output: {
+                  type: "text",
+                  value: part.text,
+                },
               });
               return;
             }
@@ -391,7 +394,7 @@ export const openAIToolDefinitionToVercelAI =
     (openai): VercelAIToolDefinition => ({
       type: "function",
       description: openai.function.description,
-      parameters: {
+      inputSchema: {
         _type: undefined,
         jsonSchema: openai.function.parameters,
         validate: undefined,

package/src/schemas/llm/vercel/messagePartSchemas.ts

@@ -26,7 +26,7 @@ export const vercelAIChatPartToolCallSchema = z.object({
   type: z.literal("tool-call"),
   toolCallId: z.string(),
   toolName: z.string(),
-  args: jsonLiteralSchema, // json serializable parameters
+  input: jsonLiteralSchema, // json serializable parameters
 });
 
 export type VercelAIChatPartToolCall = z.infer<
@@ -37,7 +37,10 @@ export const vercelAIChatPartToolResultSchema = z.object({
   type: z.literal("tool-result"),
   toolCallId: z.string(),
   toolName: z.string(),
-  result: jsonLiteralSchema, // json serializable result
+  output: z.object({
+    type: z.literal("text"), // TODO: extend to support other output types
+    value: z.string(),
+  }),
 });
 
 export type VercelAIChatPartToolResult = z.infer<

package/src/schemas/llm/vercel/toolSchemas.ts

@@ -9,7 +9,7 @@ import { z } from "zod";
 export const vercelAIToolDefinitionSchema = z.object({
   type: z.literal("function"),
   description: z.string().optional(),
-  parameters: z.object({
+  inputSchema: z.object({
     _type: z.unknown().optional().default(undefined),
     validate: z.unknown().optional().default(undefined),
     jsonSchema: z.record(z.string(), z.unknown()).optional(),

package/src/types/datasets.ts

@@ -47,8 +47,8 @@ export interface DatasetExamples {
  */
 export interface Example {
   input: Record<string, unknown>;
-  output: Record<string, unknown> | null;
-  metadata: Record<string, unknown>;
+  output?: Record<string, unknown> | null;
+  metadata?: Record<string, unknown> | null;
 }
 
 /**

package/src/types/experiments.ts

@@ -67,7 +67,7 @@ export type EvaluatorParams = {
   /**
    * Metadata associated with the Dataset Example
    */
-  metadata?: Record<string, unknown>;
+  metadata?: Example["metadata"];
 };
 
 export type Evaluator = {