@arizeai/phoenix-client 0.0.1 → 1.0.0

package/src/schemas/llm/utils.ts

@@ -0,0 +1,259 @@
+import { anthropicMessagePartSchema } from "./anthropic/messagePartSchemas";
+import { anthropicMessageSchema } from "./anthropic/messageSchemas";
+import { openAIToolCallSchema } from "./openai/toolCallSchemas";
+import { openaiChatPartSchema } from "./openai/messagePartSchemas";
+import { openAIMessageSchema } from "./openai/messageSchemas";
+import { openAIToolChoiceSchema } from "./openai/toolChoiceSchemas";
+import { anthropicToolChoiceSchema } from "./anthropic/toolChoiceSchemas";
+import { openAIToolDefinitionSchema } from "./openai/toolSchemas";
+import { anthropicToolDefinitionSchema } from "./anthropic/toolSchemas";
+import { anthropicToolCallSchema } from "./anthropic/toolCallSchemas";
+import { llmProviderToolDefinitionSchema } from "./schemas";
+import type {
+  LLMMessagePart,
+  MessagePartWithProvider,
+  MessageWithProvider,
+  SDKConverters,
+  ToolCallWithProvider,
+  ToolChoiceWithProvider,
+  ToolDefinitionWithProvider,
+} from "./types";
+import { isObject } from "../../utils/isObject";
+import type { ZodTypeAny } from "zod";
+import { vercelAIMessageSchema } from "./vercel/messageSchemas";
+import { phoenixMessageSchema } from "./phoenixPrompt/messageSchemas";
+import { phoenixToolCallSchema } from "./phoenixPrompt/toolCallSchemas";
+import { phoenixToolChoiceSchema } from "./phoenixPrompt/toolChoiceSchemas";
+import { phoenixToolDefinitionSchema } from "./phoenixPrompt/toolSchemas";
+import { phoenixContentPartSchema } from "./phoenixPrompt/messagePartSchemas";
+
+export const makeSDKConverters = <
+  MessageSchema extends ZodTypeAny,
+  MessagePartSchema extends ZodTypeAny,
+  ToolChoiceSchema extends ZodTypeAny,
+  ToolCallSchema extends ZodTypeAny,
+  ToolDefinitionSchema extends ZodTypeAny,
+  ResponseFormatSchema extends ZodTypeAny,
+>({
+  messages,
+  messageParts,
+  toolChoices,
+  toolCalls,
+  toolDefinitions,
+  responseFormat,
+}: SDKConverters<
+  MessageSchema,
+  MessagePartSchema,
+  ToolChoiceSchema,
+  ToolCallSchema,
+  ToolDefinitionSchema,
+  ResponseFormatSchema
+>): SDKConverters<
+  MessageSchema,
+  MessagePartSchema,
+  ToolChoiceSchema,
+  ToolCallSchema,
+  ToolDefinitionSchema,
+  ResponseFormatSchema
+> => {
+  return {
+    messages,
+    messageParts,
+    toolChoices,
+    toolCalls,
+    toolDefinitions,
+    responseFormat,
+  };
+};
+
+/**
+ * Detect the provider of a message object
+ */
+export const detectMessageProvider = (
+  message: unknown
+): MessageWithProvider => {
+  const { success: openaiSuccess, data: openaiData } =
+    openAIMessageSchema.safeParse(message);
+  if (openaiSuccess) {
+    return {
+      // we cannot disambiguate between azure openai and openai here
+      provider: "OPENAI",
+      validatedMessage: openaiData,
+    };
+  }
+  const { success: anthropicSuccess, data: anthropicData } =
+    anthropicMessageSchema.safeParse(message);
+  if (anthropicSuccess) {
+    return {
+      provider: "ANTHROPIC",
+      validatedMessage: anthropicData,
+    };
+  }
+  const { success: vercelSuccess, data: vercelData } =
+    vercelAIMessageSchema.safeParse(message);
+  if (vercelSuccess) {
+    return { provider: "VERCEL_AI", validatedMessage: vercelData };
+  }
+  const { success: phoenixSuccess, data: phoenixData } =
+    phoenixMessageSchema.safeParse(message);
+  if (phoenixSuccess) {
+    return { provider: "PHOENIX", validatedMessage: phoenixData };
+  }
+  return { provider: null, validatedMessage: null };
+};
+
+export const detectMessagePartProvider = (
+  part: LLMMessagePart
+): MessagePartWithProvider => {
+  const { success: openaiSuccess, data: openaiData } =
+    openaiChatPartSchema.safeParse(part);
+  if (openaiSuccess) {
+    return {
+      provider: "OPENAI",
+      validatedMessage: openaiData,
+    };
+  }
+  const { success: anthropicSuccess, data: anthropicData } =
+    anthropicMessagePartSchema.safeParse(part);
+  if (anthropicSuccess) {
+    return {
+      provider: "ANTHROPIC",
+      validatedMessage: anthropicData,
+    };
+  }
+  const { success: phoenixSuccess, data: phoenixData } =
+    phoenixContentPartSchema.safeParse(part);
+  if (phoenixSuccess) {
+    return { provider: "PHOENIX", validatedMessage: phoenixData };
+  }
+  return { provider: null, validatedMessage: null };
+};
+
+/**
+ * Detect the provider of a tool call object
+ */
+export const detectToolCallProvider = (
+  toolCall: unknown
+): ToolCallWithProvider => {
+  const { success: openaiSuccess, data: openaiData } =
+    openAIToolCallSchema.safeParse(toolCall);
+  if (openaiSuccess) {
+    // we cannot disambiguate between azure openai and openai here
+    return { provider: "OPENAI", validatedToolCall: openaiData };
+  }
+  const { success: anthropicSuccess, data: anthropicData } =
+    anthropicToolCallSchema.safeParse(toolCall);
+  if (anthropicSuccess) {
+    return { provider: "ANTHROPIC", validatedToolCall: anthropicData };
+  }
+  const { success: phoenixSuccess, data: phoenixData } =
+    phoenixToolCallSchema.safeParse(toolCall);
+  if (phoenixSuccess) {
+    return { provider: "PHOENIX", validatedToolCall: phoenixData };
+  }
+  return { provider: null, validatedToolCall: null };
+};
+
+/**
+ * Detects the provider of a tool choice
+ * @param toolChoice the tool choice to detect the provider of
+ * @returns the provider of the tool choice
+ */
+export const detectToolChoiceProvider = (
+  toolChoice: unknown
+): ToolChoiceWithProvider => {
+  const { success: openAISuccess, data: openAIData } =
+    openAIToolChoiceSchema.safeParse(toolChoice);
+  if (openAISuccess) {
+    return { provider: "OPENAI", toolChoice: openAIData };
+  }
+  const { success: anthropicSuccess, data: anthropicData } =
+    anthropicToolChoiceSchema.safeParse(toolChoice);
+  if (anthropicSuccess) {
+    return { provider: "ANTHROPIC", toolChoice: anthropicData };
+  }
+  const { success: phoenixSuccess, data: phoenixData } =
+    phoenixToolChoiceSchema.safeParse(toolChoice);
+  if (phoenixSuccess) {
+    return { provider: "PHOENIX", toolChoice: phoenixData };
+  }
+  return { provider: null, toolChoice: null };
+};
+
+/**
+ * Detect the provider of a tool call object
+ */
+export const detectToolDefinitionProvider = (
+  toolDefinition: unknown
+): ToolDefinitionWithProvider => {
+  const { success: openaiSuccess, data: openaiData } =
+    openAIToolDefinitionSchema.safeParse(toolDefinition);
+  if (openaiSuccess) {
+    return {
+      // we cannot disambiguate between azure openai and openai here
+      provider: "OPENAI",
+      validatedToolDefinition: openaiData,
+    };
+  }
+  const { success: anthropicSuccess, data: anthropicData } =
+    anthropicToolDefinitionSchema.safeParse(toolDefinition);
+  if (anthropicSuccess) {
+    return {
+      provider: "ANTHROPIC",
+      validatedToolDefinition: anthropicData,
+    };
+  }
+  const { success: phoenixSuccess, data: phoenixData } =
+    phoenixToolDefinitionSchema.safeParse(toolDefinition);
+  if (phoenixSuccess) {
+    return { provider: "PHOENIX", validatedToolDefinition: phoenixData };
+  }
+  return { provider: null, validatedToolDefinition: null };
+};
+
+export const findToolDefinitionName = (toolDefinition: unknown) => {
+  const parsed = llmProviderToolDefinitionSchema.safeParse(toolDefinition);
+  if (!parsed.success || parsed.data === null || !isObject(parsed.data)) {
+    return null;
+  }
+
+  if (
+    "function" in parsed.data &&
+    isObject(parsed.data.function) &&
+    "name" in parsed.data.function &&
+    typeof parsed.data.function.name === "string"
+  ) {
+    return parsed.data.function.name;
+  }
+
+  if ("name" in parsed.data && typeof parsed.data.name === "string") {
+    return parsed.data.name;
+  }
+
+  return null;
+};
+
+export const findToolDefinitionDescription = (toolDefinition: unknown) => {
+  const parsed = llmProviderToolDefinitionSchema.safeParse(toolDefinition);
+  if (!parsed.success || parsed.data === null || !isObject(parsed.data)) {
+    return null;
+  }
+
+  if (
+    "function" in parsed.data &&
+    isObject(parsed.data.function) &&
+    "description" in parsed.data.function &&
+    typeof parsed.data.function.description === "string"
+  ) {
+    return parsed.data.function.description;
+  }
+
+  if (
+    "description" in parsed.data &&
+    typeof parsed.data.description === "string"
+  ) {
+    return parsed.data.description;
+  }
+
+  return null;
+};

package/src/schemas/llm/vercel/messagePartSchemas.ts

@@ -0,0 +1,54 @@
+import z from "zod";
+import { jsonLiteralSchema } from "../../jsonLiteralSchema";
+
+/*
+ *
+ * Vercel AI SDK Message Part Schemas
+ *
+ */
+
+export const vercelAIChatPartTextSchema = z.object({
+  type: z.literal("text"),
+  text: z.string(),
+});
+
+export type VercelAIChatPartText = z.infer<typeof vercelAIChatPartTextSchema>;
+
+export const vercelAIChatPartImageSchema = z.object({
+  type: z.literal("image"),
+  image: z.string(), // ai supports more, but we will just support base64 for now
+  mimeType: z.string().optional(),
+});
+
+export type VercelAIChatPartImage = z.infer<typeof vercelAIChatPartImageSchema>;
+
+export const vercelAIChatPartToolCallSchema = z.object({
+  type: z.literal("tool-call"),
+  toolCallId: z.string(),
+  toolName: z.string(),
+  args: jsonLiteralSchema, // json serializable parameters
+});
+
+export type VercelAIChatPartToolCall = z.infer<
+  typeof vercelAIChatPartToolCallSchema
+>;
+
+export const vercelAIChatPartToolResultSchema = z.object({
+  type: z.literal("tool-result"),
+  toolCallId: z.string(),
+  toolName: z.string(),
+  result: jsonLiteralSchema, // json serializable result
+});
+
+export type VercelAIChatPartToolResult = z.infer<
+  typeof vercelAIChatPartToolResultSchema
+>;
+
+export const vercelAIChatPartSchema = z.discriminatedUnion("type", [
+  vercelAIChatPartTextSchema,
+  vercelAIChatPartImageSchema,
+  vercelAIChatPartToolCallSchema,
+  vercelAIChatPartToolResultSchema,
+]);
+
+export type VercelAIChatPart = z.infer<typeof vercelAIChatPartSchema>;

package/src/schemas/llm/vercel/messageSchemas.ts

@@ -0,0 +1,54 @@
+import z from "zod";
+
+import {
+  vercelAIChatPartImageSchema,
+  vercelAIChatPartTextSchema,
+  vercelAIChatPartToolCallSchema,
+  vercelAIChatPartToolResultSchema,
+} from "./messagePartSchemas";
+
+/*
+ *
+ * Vercel AI SDK Message Schemas
+ *
+ */
+
+export const vercelAIMessageRoleSchema = z.enum([
+  "system",
+  "user",
+  "assistant",
+  "tool",
+]);
+
+export type VercelAIMessageRole = z.infer<typeof vercelAIMessageRoleSchema>;
+
+export const vercelAIMessageSchema = z.discriminatedUnion("role", [
+  z.object({
+    role: z.literal("system"),
+    content: z.string(),
+  }),
+  z.object({
+    role: z.literal("user"),
+    content: z.union([
+      z
+        .union([vercelAIChatPartTextSchema, vercelAIChatPartImageSchema])
+        .array(),
+      z.string(),
+    ]),
+  }),
+  z.object({
+    role: z.literal("assistant"),
+    content: z.union([
+      z
+        .union([vercelAIChatPartTextSchema, vercelAIChatPartToolCallSchema])
+        .array(),
+      z.string(),
+    ]),
+  }),
+  z.object({
+    role: z.literal("tool"),
+    content: vercelAIChatPartToolResultSchema.array(),
+  }),
+]);
+
+export type VercelAIMessage = z.infer<typeof vercelAIMessageSchema>;

package/src/schemas/llm/vercel/toolChoiceSchemas.ts

@@ -0,0 +1,16 @@
+import z from "zod";
+
+/**
+ * Vercel AI SDK Tool Choice Schema
+ */
+export const vercelAIToolChoiceSchema = z.union([
+  z.literal("auto"),
+  z.literal("none"),
+  z.literal("required"),
+  z.object({
+    type: z.literal("tool"),
+    toolName: z.string(),
+  }),
+]);
+
+export type VercelAIToolChoice = z.infer<typeof vercelAIToolChoiceSchema>;

package/src/types/core.ts

@@ -1,6 +1,15 @@
+import { PhoenixClient } from "../client";
 /**
  * A uniquely identifiable node in Phoenix
  */
 export interface Node {
   id: string;
 }
+
+export interface ClientFn {
+  /**
+   * An instance of the Phoenix client.
+   * If not provided, the client will be created using the default configuration.
+   */
+  client?: PhoenixClient;
+}

package/src/types/logger.ts

@@ -0,0 +1,5 @@
+export type Logger = {
+  info: (message: string) => void;
+  error: (message: string) => void;
+  log: (message: string) => void;
+};

package/src/types/prompts.ts

@@ -0,0 +1,151 @@
+import { components } from "../__generated__/api/v1";
+
+/**
+ * Supported prompt model providers
+ */
+export type PromptModelProvider =
+  | "OPENAI"
+  | "AZURE_OPENAI"
+  | "ANTHROPIC"
+  | "GOOGLE";
+
+/**
+ * Supported prompt provider SDKs
+ *
+ * These are possible sdk formats for prompt templates
+ */
+export type PromptProviderSDKs =
+  | "PHOENIX"
+  | "AZURE_OPENAI"
+  | "OPENAI"
+  | "ANTHROPIC"
+  | "VERCEL_AI";
+
+/**
+ * The role of a prompt chat message
+ */
+export type PromptChatMessageRole = "user" | "system" | "ai" | "tool";
+
+/**
+ * Selector for a prompt by id.
+ */
+export interface GetPromptByIdSelector {
+  promptId: string;
+}
+
+/**
+ * Selector for a prompt by name.
+ */
+export interface GetPromptByNameSelector {
+  name: string;
+}
+
+/**
+ * Selector for a prompt by version id.
+ */
+export interface GetPromptByVersionSelector {
+  versionId: string;
+}
+
+/**
+ * Selector for a named prompt by tag.
+ */
+export interface GetPromptByTagSelector {
+  tag: string;
+  name: string;
+}
+
+/**
+ * A prompt like object.
+ *
+ * Can be a prompt id, a prompt name, a prompt version id, or a prompt name + tag.
+ */
+export type PromptSelector =
+  | GetPromptByIdSelector
+  | GetPromptByNameSelector
+  | GetPromptByVersionSelector
+  | GetPromptByTagSelector;
+
+/**
+ * The prompt data needed to create a prompt.
+ */
+export type PromptData = components["schemas"]["PromptData"];
+
+/**
+ * The prompt version type from the API.
+ *
+ * aka the prompt at a specific point in time
+ */
+export type PromptVersion = components["schemas"]["PromptVersion"];
+
+/**
+ * The prompt version data needed to create a prompt version.
+ */
+export type PromptVersionData = components["schemas"]["PromptVersionData"];
+
+/**
+ * The invocation parameters for a prompt version for OpenAI.
+ */
+export type OpenAIInvocationParameters =
+  components["schemas"]["PromptOpenAIInvocationParametersContent"];
+
+/**
+ * The invocation parameters for a prompt version for Azure OpenAI.
+ */
+export type AzureOpenAIInvocationParameters =
+  components["schemas"]["PromptAzureOpenAIInvocationParametersContent"];
+
+/**
+ * The invocation parameters for a prompt version for Anthropic.
+ */
+export type AnthropicInvocationParameters =
+  components["schemas"]["PromptAnthropicInvocationParametersContent"];
+
+/**
+ * The invocation parameters for a prompt version for Google.
+ */
+export type GoogleInvocationParameters =
+  components["schemas"]["PromptGoogleInvocationParametersContent"];
+
+/**
+ * The format of the prompt template message(s).
+ */
+export type PromptTemplateFormat = PromptVersion["template_format"];
+
+/**
+ * Extracts the chat message type from the prompt template who may be a StringTemplate or ChatTemplate.
+ */
+export type PromptChatMessage = Extract<
+  PromptVersion["template"],
+  { messages: unknown[] }
+>["messages"][number];
+
+/**
+ * Extracts the chat message part type from the prompt chat message.
+ */
+export type PromptChatMessagePart =
+  | components["schemas"]["TextContentPart"]
+  | components["schemas"]["ToolCallContentPart"]
+  | components["schemas"]["ToolResultContentPart"];
+
+/**
+ * The Phoenix prompt tool type from the API.
+ */
+export type PromptTool = components["schemas"]["PromptTools"]["tools"][number];
+
+/**
+ * The Phoenix prompt tool choice type from the API.
+ */
+export type PromptToolChoice = NonNullable<
+  components["schemas"]["PromptTools"]["tool_choice"]
+>;
+/**
+ * The Phoenix prompt output schema type from the API.
+ */
+export type PromptResponseFormat =
+  components["schemas"]["PromptResponseFormatJSONSchema"];
+
+/**
+ * The prompt type from the API.
+ */
+export type Prompt = components["schemas"]["Prompt"];

package/src/utils/assertUnreachable.ts

@@ -0,0 +1,9 @@
+/**
+ * Utility function that uses the type system to check if a switch statement is exhaustive.
+ * If the switch statement is not exhaustive, there will be a type error caught in typescript
+ *
+ * See https://stackoverflow.com/questions/39419170/how-do-i-check-that-a-switch-block-is-exhaustive-in-typescript for more details.
+ */
+export function assertUnreachable(_: never): never {
+  throw new Error("Unreachable");
+}

package/src/utils/formatPromptMessages.ts

@@ -0,0 +1,79 @@
+import { Variables } from "../prompts/sdks/types";
+import { PromptChatMessage, PromptTemplateFormat } from "../types/prompts";
+import { assertUnreachable } from "./assertUnreachable";
+import {
+  TextPart,
+  asTextPart,
+} from "../schemas/llm/phoenixPrompt/messagePartSchemas";
+
+/**
+ * Format a list of prompt messages
+ *
+ * @param format - The format of the prompt message variables, e.g. MUSTACHE, F_STRING, NONE
+ * @param promptMessages - The prompt messages to format
+ * @param variables - The variables to use in the formatting
+ * @returns The formatted prompt messages
+ */
+export function formatPromptMessages(
+  format: PromptTemplateFormat,
+  promptMessages: PromptChatMessage[],
+  variables: Variables = {}
+) {
+  const replacements: [RegExp, string][] = [];
+  switch (format) {
+    case "MUSTACHE": {
+      const asMustache = Object.entries(variables).map(([key, value]) => [
+        new RegExp(`\\{\\{\\s*${key}\\s*\\}\\}(?!\\})`, "g"),
+        value.toString(),
+      ]) satisfies [RegExp, string][];
+      replacements.push(...asMustache);
+      break;
+    }
+    case "F_STRING": {
+      const asF_STRING = Object.entries(variables).map(([key, value]) => [
+        new RegExp(`(?<!\\{)\\{\\s*${key}\\s*\\}(?!\\})`, "g"),
+        value.toString(),
+      ]) satisfies [RegExp, string][];
+      replacements.push(...asF_STRING);
+      break;
+    }
+    case "NONE":
+      break;
+    default:
+      assertUnreachable(format);
+  }
+
+  return promptMessages.map((message) => ({
+    ...message,
+    content:
+      typeof message.content == "string"
+        ? applyReplacements(message.content, replacements)
+        : message.content.map((content) => {
+            const textPart = asTextPart(content);
+            if (textPart) {
+              return {
+                ...textPart,
+                text: applyReplacements(textPart.text, replacements),
+              } satisfies TextPart;
+            }
+            return content;
+          }),
+  }));
+}
+
+/**
+ * Apply a list of replacements to a string
+ * @param text - The text to apply the replacements to
+ * @param replacements - The replacements to apply
+ * @returns The text with the replacements applied
+ */
+function applyReplacements(
+  text: string,
+  replacements: [RegExp, string][]
+): string {
+  let newText = text;
+  for (const [key, value] of replacements) {
+    newText = newText.replaceAll(key, value);
+  }
+  return newText;
+}

package/src/utils/getDatasetLike.ts

@@ -0,0 +1,52 @@
+import invariant from "tiny-invariant";
+import { PhoenixClient } from "../client";
+import { Example } from "../types/datasets";
+
+import { Dataset } from "../types/datasets";
+
+/**
+ * Parameters for the getDatasetLike function
+ */
+export type GetDatasetLikeParams = {
+  dataset: Dataset | string | Example[];
+  client: PhoenixClient;
+};
+
+/**
+ * Return a dataset object from the input.
+ *
+ * If the input is a string, assume it is a dataset id and fetch the dataset from the client.
+ * If the input is an array of examples, create a new dataset from the examples then return it.
+ * If the input is a dataset, return it as is.
+ *
+ * @param dataset - The dataset to get.
+ * @returns The dataset.
+ */
+export async function getDatasetLike({
+  dataset,
+  client,
+}: GetDatasetLikeParams): Promise<Dataset | null> {
+  if (typeof dataset === "string") {
+    const datasetResponse = await client
+      .GET(`/v1/datasets/{id}`, { params: { path: { id: dataset } } })
+      .then((d) => d.data?.data);
+    invariant(datasetResponse, `Dataset ${dataset} not found`);
+    const examples = await client
+      .GET(`/v1/datasets/{id}/examples`, { params: { path: { id: dataset } } })
+      .then((e) => e.data?.data);
+    invariant(examples, `Examples for dataset ${dataset} not found`);
+    const datasetWithExamples: Dataset = {
+      ...datasetResponse,
+      examples: examples.examples.map((example) => ({
+        ...example,
+        updatedAt: new Date(example.updated_at),
+      })),
+      versionId: examples.version_id,
+    };
+    return datasetWithExamples;
+  }
+  if (Array.isArray(dataset)) {
+    throw new Error("TODO: implement dataset creation from examples");
+  }
+  return dataset;
+}