@arizeai/phoenix-client 0.0.1 → 1.0.0

package/src/client.ts

@@ -13,6 +13,9 @@ type pathsV1 = oapiPathsV1;
 type componentsV1 = oapiComponentsV1;
 type operationsV1 = oapiOperationsV1;
 
+/**
+ * Generated openapi types for the Phoenix client
+ */
 export type Types = {
   V1: {
     paths: pathsV1;
@@ -46,8 +49,9 @@ export const getMergedOptions = ({
 /**
  * Create a Phoenix client.
  *
- * @param configuration - The configuration to use for the client.
- * @param configuration.options - The options to use for the client's OpenAPI Fetch wrapper.
+ * @param config - The configuration to use for the client.
+ * @param config.options - The options to use for the client's OpenAPI Fetch wrapper.
+ * @param config.getEnvironmentOptions - The function to use to get the environment options.
  * @returns The Phoenix client.
  */
 export const createClient = (
@@ -60,4 +64,7 @@ export const createClient = (
   return createOpenApiClient<pathsV1>(mergedOptions);
 };
 
+/**
+ * The type of the Phoenix client
+ */
 export type PhoenixClient = ReturnType<typeof createClient>;

package/src/config.ts

@@ -2,6 +2,7 @@ import type { ClientOptions } from "openapi-fetch";
 import z from "zod";
 
 const phoenixEnvironmentSchema = z.object({
+  PHOENIX_API_KEY: z.string().optional(),
   PHOENIX_HOST: z.string().optional(),
   PHOENIX_CLIENT_HEADERS: z
     .string()
@@ -33,9 +34,19 @@ const phoenixEnvironmentToClientOptions = (
 ): Partial<ClientOptions> => {
   const options: Partial<ClientOptions> = {
     baseUrl: environment.PHOENIX_HOST,
-    headers: environment.PHOENIX_CLIENT_HEADERS,
+    headers: {
+      ...(environment.PHOENIX_CLIENT_HEADERS ?? {}),
+      ...(environment.PHOENIX_API_KEY
+        ? { Authorization: `Bearer ${environment.PHOENIX_API_KEY}` }
+        : {}),
+    },
   };
 
+  // if headers is an empty object, delete it
+  if (options.headers && Object.keys(options.headers).length === 0) {
+    delete options.headers;
+  }
+
   // filter out undefined values
   // this will prevent clobbering over default values when merging
   return Object.fromEntries(

package/src/experiments/runExperiment.ts

@@ -12,33 +12,51 @@ import type {
 import { promisifyResult } from "../utils/promisifyResult";
 import invariant from "tiny-invariant";
 import { pluralize } from "../utils/pluralize";
+import { ClientFn } from "../types/core";
+import { getDatasetLike } from "../utils/getDatasetLike";
+import { type Logger } from "../types/logger";
 
-export type Logger = {
-  info: (message: string) => void;
-  error: (message: string) => void;
-  log: (message: string) => void;
-};
-
-export type RunExperimentParams = {
+export type RunExperimentParams = ClientFn & {
   /**
    * An optional name for the experiment.
    * Defaults to the dataset name + a timestamp
    */
   experimentName?: string;
-  client?: PhoenixClient;
+  /**
+   * The dataset to run the experiment on
+   */
   dataset: Dataset | string | Example[];
+  /**
+   * The task to run
+   */
   task: ExperimentTask;
+  /**
+   * The evaluators to use
+   */
   evaluators?: Evaluator[];
+  /**
+   * The number of repetitions to run
+   */
   repetitions?: number;
   /**
    * The project under which the experiment task traces are recorded
    */
   projectName?: string;
+  /**
+   * The logger to use
+   */
   logger?: Logger;
+  /**
+   * Whether to record the experiment results
+   */
+  record?: boolean;
 };
 
 /**
  * Run an experiment.
+ *
+ * @experimental This feature is not complete, and will change in the future.
+ * @deprecated This function will be un-marked as deprecated once the experimental feature flag is removed.
  */
 export async function runExperiment({
   experimentName: _experimentName,
@@ -49,9 +67,10 @@ export async function runExperiment({
   repetitions = 1,
   projectName = "default",
   logger = console,
+  record = true,
 }: RunExperimentParams): Promise<RanExperiment> {
   const client = _client ?? createClient();
-  const dataset = await getDataset({ dataset: _dataset, client });
+  const dataset = await getDatasetLike({ dataset: _dataset, client });
   invariant(dataset, `Dataset not found`);
   invariant(dataset.examples.length > 0, `Dataset has no examples`);
   const experimentName =
@@ -69,6 +88,12 @@ export async function runExperiment({
     projectName,
   };
 
+  if (!record) {
+    logger.info(
+      `🔧 Running experiment in readonly mode. Results will not be recorded.`
+    );
+  }
+
   logger.info(
     `🧪 Starting experiment "${experimentName}" on dataset "${dataset.id}" with task "${task.name}" and ${evaluators?.length ?? 0} ${pluralize(
       "evaluator",
@@ -160,7 +185,10 @@ function runTask({
     try {
       const taskOutput = await promisifyResult(task(example));
       // TODO: why doesn't run output type match task output type?
-      thisRun.output = JSON.stringify(taskOutput);
+      thisRun.output =
+        typeof taskOutput === "string"
+          ? taskOutput
+          : JSON.stringify(taskOutput);
     } catch (error) {
       thisRun.error = error instanceof Error ? error.message : "Unknown error";
     }
@@ -170,6 +198,12 @@ function runTask({
   return Promise.all(dataset.examples.map(run));
 }
 
+/**
+ * Evaluate an experiment.
+ *
+ * @experimental This feature is not complete, and will change in the future.
+ * @deprecated This function will be un-marked as deprecated once the experimental feature flag is removed.
+ */
 export async function evaluateExperiment({
   experiment,
   evaluators,
@@ -189,7 +223,10 @@ export async function evaluateExperiment({
   logger: Logger;
 }): Promise<RanExperiment> {
   const client = _client ?? createClient();
-  const dataset = await getDataset({ dataset: experiment.datasetId, client });
+  const dataset = await getDatasetLike({
+    dataset: experiment.datasetId,
+    client,
+  });
   invariant(dataset, `Dataset "${experiment.datasetId}" not found`);
   invariant(
     dataset.examples.length > 0,
@@ -248,6 +285,9 @@ export async function evaluateExperiment({
 
 /**
  * Run an evaluator against a run.
+ *
+ * @experimental This feature is not complete, and will change in the future.
+ * @deprecated This function will be un-marked as deprecated once the experimental feature flag is removed.
  */
 async function runEvaluator({
   evaluator,
@@ -292,51 +332,12 @@ async function runEvaluator({
   return evaluate();
 }
 
-/**
- * Return a dataset object from the input.
- *
- * If the input is a string, assume it is a dataset id and fetch the dataset from the client.
- * If the input is an array of examples, create a new dataset from the examples then return it.
- * If the input is a dataset, return it as is.
- *
- * @param dataset - The dataset to get.
- * @returns The dataset.
- */
-async function getDataset({
-  dataset,
-  client,
-}: {
-  dataset: Dataset | string | Example[];
-  client: PhoenixClient;
-}): Promise<Dataset> {
-  if (typeof dataset === "string") {
-    const datasetResponse = await client
-      .GET(`/v1/datasets/{id}`, { params: { path: { id: dataset } } })
-      .then((d) => d.data?.data);
-    invariant(datasetResponse, `Dataset ${dataset} not found`);
-    const examples = await client
-      .GET(`/v1/datasets/{id}/examples`, { params: { path: { id: dataset } } })
-      .then((e) => e.data?.data);
-    invariant(examples, `Examples for dataset ${dataset} not found`);
-    const datasetWithExamples: Dataset = {
-      ...datasetResponse,
-      examples: examples.examples.map((example) => ({
-        ...example,
-        updatedAt: new Date(example.updated_at),
-      })),
-      versionId: examples.version_id,
-    };
-    return datasetWithExamples;
-  }
-  if (Array.isArray(dataset)) {
-    throw new Error("TODO: implement dataset creation from examples");
-  }
-  return dataset;
-}
-
 /**
  * Wrap an evaluator function in an object with a name property.
  *
+ * @experimental This feature is not complete, and will change in the future.
+ * @deprecated This function will be un-marked as deprecated once the experimental feature flag is removed.
+ *
  * @param name - The name of the evaluator.
  * @param evaluate - The evaluator function.
  * @returns The evaluator object.

package/src/index.ts

@@ -1,3 +1 @@
 export * from "./client";
-
-export * from "./experiments";

package/src/prompts/constants.ts

@@ -0,0 +1,11 @@
+import type { PromptModelProvider } from "../types/prompts";
+
+/**
+ * A mapping of ModelProvider to a human-readable string
+ */
+export const PromptModelProviders: Record<PromptModelProvider, string> = {
+  OPENAI: "OpenAI",
+  AZURE_OPENAI: "Azure OpenAI",
+  ANTHROPIC: "Anthropic",
+  GOOGLE: "Google",
+};

package/src/prompts/createPrompt.ts

@@ -0,0 +1,181 @@
+import { createClient } from "../client";
+import { ClientFn } from "../types/core";
+import {
+  PromptData,
+  PromptVersionData,
+  PromptVersion,
+  OpenAIInvocationParameters,
+  AzureOpenAIInvocationParameters,
+  AnthropicInvocationParameters,
+  GoogleInvocationParameters,
+  PromptChatMessage,
+} from "../types/prompts";
+import { assertUnreachable } from "../utils/assertUnreachable";
+
+/**
+ * Parameters to crate a prompt
+ */
+export interface CreatePromptParams extends ClientFn, PromptData {
+  /**
+   * The name of the promt
+   */
+  name: string;
+  /**
+   * The description of the prompt
+   */
+  description?: string;
+  /**
+   * The prompt version to push onto the history of the prompt
+   */
+  version: PromptVersionData;
+}
+
+/**
+ * Create a prompt and store it in Phoenix
+ * If a prompt with the same name exists, a new version of the prompt will be appended to the history
+ */
+export async function createPrompt({
+  client: _client,
+  version,
+  ...promptParams
+}: CreatePromptParams): Promise<PromptVersion> {
+  const client = _client ?? createClient();
+  const response = await client.POST("/v1/prompts", {
+    body: {
+      prompt: promptParams,
+      version: version,
+    },
+  });
+  const createdPromptVersion = response.data?.data;
+  if (!createdPromptVersion) {
+    throw new Error("Failed to create prompt");
+  }
+  return createdPromptVersion;
+}
+
+interface PromptVersionInputBase {
+  description?: string;
+  modelName: PromptVersionData["model_name"];
+  /**
+   * The template for the prompt version.
+   * Currently only chat is supported.
+   */
+  template: PromptChatMessage[];
+  /**
+   * The format of the template.
+   * @default "MUSTACHE"
+   */
+  templateFormat?: PromptVersionData["template_format"];
+}
+
+interface OpenAIPromptVersionInput extends PromptVersionInputBase {
+  modelProvider: "OPENAI";
+  invocationParameters?: OpenAIInvocationParameters;
+}
+
+interface AzureOpenAIPromptVersionInput extends PromptVersionInputBase {
+  modelProvider: "AZURE_OPENAI";
+  invocationParameters?: AzureOpenAIInvocationParameters;
+}
+
+interface AnthropicPromptVersionInput extends PromptVersionInputBase {
+  modelProvider: "ANTHROPIC";
+  /**
+   * The invocation parameters for the prompt version.
+   * For Anthropic, the invocation parameters are required since max_tokens is required.
+   */
+  invocationParameters: AnthropicInvocationParameters;
+}
+
+interface GooglePromptVersionInput extends PromptVersionInputBase {
+  modelProvider: "GOOGLE";
+  invocationParameters?: GoogleInvocationParameters;
+}
+
+type PromptVersionInput =
+  | OpenAIPromptVersionInput
+  | AzureOpenAIPromptVersionInput
+  | AnthropicPromptVersionInput
+  | GooglePromptVersionInput;
+
+/**
+ * A helper function to construct a prompt version declaratively
+ */
+export function promptVersion(params: PromptVersionInput): PromptVersionData {
+  const {
+    description = "",
+    modelProvider: model_provider,
+    modelName: model_name,
+    template: templateMessages,
+    templateFormat: template_format = "MUSTACHE",
+    invocationParameters: invocation_parameters,
+  } = params;
+  switch (model_provider) {
+    case "OPENAI":
+      return {
+        description,
+        model_provider,
+        model_name,
+        template_type: "CHAT",
+        template_format,
+        template: {
+          type: "chat",
+          messages: templateMessages,
+        },
+        invocation_parameters: {
+          type: "openai",
+          openai: invocation_parameters ?? {},
+        },
+      };
+    case "AZURE_OPENAI":
+      return {
+        description,
+        model_provider,
+        model_name,
+        template_type: "CHAT",
+        template_format,
+        template: {
+          type: "chat",
+          messages: templateMessages,
+        },
+        invocation_parameters: {
+          type: "azure_openai",
+          azure_openai: invocation_parameters ?? {},
+        },
+      };
+    case "ANTHROPIC":
+      return {
+        description,
+        model_provider,
+        model_name,
+        template_type: "CHAT",
+        template_format,
+        template: {
+          type: "chat",
+          messages: templateMessages,
+        },
+        invocation_parameters: {
+          type: "anthropic",
+          anthropic: invocation_parameters,
+        },
+      };
+    case "GOOGLE":
+      return {
+        description,
+        model_provider,
+        model_name,
+        template_type: "CHAT",
+        template_format,
+        template: {
+          type: "chat",
+          messages: templateMessages,
+        },
+        invocation_parameters: {
+          type: "google",
+          google: invocation_parameters ?? {},
+        },
+      };
+    default:
+      assertUnreachable(model_provider);
+  }
+}

package/src/prompts/getPrompt.ts

@@ -0,0 +1,26 @@
+import { createClient } from "../client";
+import { ClientFn } from "../types/core";
+import { PromptSelector, PromptVersion } from "../types/prompts";
+import { getPromptBySelector } from "../utils/getPromptVersionLike";
+
+/**
+ * Parameters for the getPrompt function
+ */
+export interface GetPromptParams extends ClientFn {
+  /**
+   * The prompt to get. Can be in the form of a prompt id, a prompt version id, a prompt name, or a prompt name + tag.
+   */
+  prompt: PromptSelector;
+}
+
+/**
+ * Get a prompt from the Phoenix API.
+ */
+export async function getPrompt({
+  client: _client,
+  prompt: _prompt,
+}: GetPromptParams): Promise<PromptVersion | null> {
+  const client = _client ?? createClient();
+  const promptVersion = await getPromptBySelector({ client, prompt: _prompt });
+  return promptVersion;
+}

package/src/prompts/index.ts

@@ -0,0 +1,3 @@
+export * from "./getPrompt";
+export * from "./createPrompt";
+export * from "./sdks";

package/src/prompts/sdks/constants.ts

@@ -0,0 +1,10 @@
+import { PromptModelProvider } from "../../types/prompts";
+
+export const SUPPORTED_SDKS = ["openai", "anthropic", "ai"] as const;
+
+export const SUPPORTED_SDK_TO_PROMPT_MODEL_PROVIDER = {
+  openai: "OPENAI",
+  anthropic: "ANTHROPIC",
+} satisfies Partial<
+  Record<(typeof SUPPORTED_SDKS)[number], PromptModelProvider>
+>;

package/src/prompts/sdks/index.ts

@@ -0,0 +1 @@
+export * from "./toSDK";

package/src/prompts/sdks/toAI.ts

@@ -0,0 +1,105 @@
+import invariant from "tiny-invariant";
+import {
+  safelyConvertMessageToProvider,
+  safelyConvertToolChoiceToProvider,
+} from "../../schemas/llm/converters";
+import { formatPromptMessages } from "../../utils/formatPromptMessages";
+import { Variables, toSDKParamsBase } from "./types";
+import {
+  type streamText,
+  type generateText,
+  type ToolSet,
+  type Tool,
+  jsonSchema,
+} from "ai";
+import { VercelAIToolChoice } from "../../schemas/llm/vercel/toolChoiceSchemas";
+
+export type PartialStreamTextParams = Omit<
+  Parameters<typeof streamText>[0] | Parameters<typeof generateText>[0],
+  "model"
+>;
+
+export type ToAIParams<V extends Variables> = toSDKParamsBase<V>;
+
+/**
+ * Converts a Phoenix prompt to Vercel AI sdk params.
+ *
+ * - note: To use response format, you must pass `prompt.response_format.json` to generateObject or streamObject
+ *   directly, through the `schema` argument.
+ */
+export const toAI = <V extends Variables>({
+  prompt,
+  variables,
+}: ToAIParams<V>): PartialStreamTextParams | null => {
+  // eslint-disable-next-line no-console
+  console.warn(
+    "Prompt invocation parameters not currently supported in AI SDK, falling back to default invocation parameters"
+  );
+  try {
+    // parts of the prompt that can be directly converted to OpenAI params
+    const baseCompletionParams = {
+      // Invocation parameters are validated on the phoenix-side
+    } satisfies Partial<PartialStreamTextParams>;
+
+    if (!("messages" in prompt.template)) {
+      return null;
+    }
+
+    let formattedMessages = prompt.template.messages;
+
+    if (variables) {
+      formattedMessages = formatPromptMessages(
+        prompt.template_format,
+        formattedMessages,
+        variables
+      );
+    }
+
+    const messages = formattedMessages.map((message) => {
+      const vercelAIMessage = safelyConvertMessageToProvider({
+        message,
+        targetProvider: "VERCEL_AI",
+      });
+      invariant(vercelAIMessage, "Message is not valid");
+      return vercelAIMessage;
+    });
+
+    // convert tools to Vercel AI tool set, which is a map of tool name to tool
+    let tools: ToolSet | undefined = prompt.tools?.tools.reduce((acc, tool) => {
+      if (!tool.function.parameters) {
+        return acc;
+      }
+      acc[tool.function.name] = {
+        type: "function",
+        parameters: jsonSchema(tool.function.parameters),
+        description: tool.function.description,
+      } satisfies Tool;
+      return acc;
+    }, {} as ToolSet);
+    const hasTools = Object.keys(tools ?? {}).length > 0;
+    tools = hasTools ? tools : undefined;
+
+    let toolChoice: VercelAIToolChoice | undefined =
+      safelyConvertToolChoiceToProvider({
+        toolChoice: prompt.tools?.tool_choice,
+        targetProvider: "VERCEL_AI",
+      }) || undefined;
+    toolChoice = hasTools ? toolChoice : undefined;
+
+    // combine base and computed params
+    const completionParams = {
+      ...baseCompletionParams,
+      messages,
+      tools,
+      toolChoice,
+    } satisfies Partial<PartialStreamTextParams>;
+
+    return completionParams;
+  } catch (error) {
+    // eslint-disable-next-line no-console
+    console.warn(`Failed to convert prompt to AI params`);
+    // eslint-disable-next-line no-console
+    console.error(error);
+    return null;
+  }
+};

package/src/prompts/sdks/toAnthropic.ts

@@ -0,0 +1,98 @@
+import type { MessageCreateParams } from "@anthropic-ai/sdk/resources/messages/messages";
+import type { Variables, toSDKParamsBase } from "./types";
+import { formatPromptMessages } from "../../utils/formatPromptMessages";
+
+import invariant from "tiny-invariant";
+import {
+  safelyConvertMessageToProvider,
+  safelyConvertToolChoiceToProvider,
+  safelyConvertToolDefinitionToProvider,
+} from "../../schemas/llm/converters";
+import { AnthropicToolChoice } from "../../schemas/llm/anthropic/toolChoiceSchemas";
+
+// We must re-export these types so that they are included in the phoenix-client distribution
+export type { MessageCreateParams };
+
+export type ToAnthropicParams<V extends Variables> = toSDKParamsBase<V>;
+
+/**
+ * Convert a Phoenix prompt to Anthropic client sdk's message create parameters
+ */
+export const toAnthropic = <V extends Variables = Variables>({
+  prompt,
+  variables,
+}: ToAnthropicParams<V>): MessageCreateParams | null => {
+  try {
+    let invocationParameters: { max_tokens: number } | undefined;
+    if (prompt.invocation_parameters.type === "anthropic") {
+      invocationParameters = prompt.invocation_parameters.anthropic;
+    } else {
+      // eslint-disable-next-line no-console
+      console.warn(
+        "Prompt is not an Anthropic prompt, falling back to default Anthropic invocation parameters"
+      );
+      invocationParameters = { max_tokens: 1024 };
+    }
+    // parts of the prompt that can be directly converted to Anthropic params
+    const baseCompletionParams = {
+      model: prompt.model_name,
+      ...invocationParameters,
+    } satisfies Partial<MessageCreateParams>;
+
+    if (!("messages" in prompt.template)) {
+      return null;
+    }
+
+    let formattedMessages = prompt.template.messages;
+
+    if (variables) {
+      formattedMessages = formatPromptMessages(
+        prompt.template_format,
+        formattedMessages,
+        variables
+      );
+    }
+
+    const messages = formattedMessages.map((message) => {
+      const anthropicMessage = safelyConvertMessageToProvider({
+        message,
+        targetProvider: "ANTHROPIC",
+      });
+      invariant(anthropicMessage, "Message is not valid");
+      return anthropicMessage;
+    });
+
+    let tools = prompt.tools?.tools.map((tool) => {
+      const anthropicToolDefinition = safelyConvertToolDefinitionToProvider({
+        toolDefinition: tool,
+        targetProvider: "ANTHROPIC",
+      });
+      invariant(anthropicToolDefinition, "Tool definition is not valid");
+      return anthropicToolDefinition;
+    });
+    tools = (tools?.length ?? 0) > 0 ? tools : undefined;
+
+    let tool_choice: AnthropicToolChoice | undefined =
+      safelyConvertToolChoiceToProvider({
+        toolChoice: prompt?.tools?.tool_choice,
+        targetProvider: "ANTHROPIC",
+      }) || undefined;
+    tool_choice = tools?.length ? tool_choice : undefined;
+
+    // combine base and computed params
+    const completionParams = {
+      ...baseCompletionParams,
+      messages,
+      tools,
+      tool_choice,
+    } satisfies Partial<MessageCreateParams>;
+
+    return completionParams;
+  } catch (e) {
+    // eslint-disable-next-line no-console
+    console.warn(`Failed to convert prompt to Anthropic params`);
+    // eslint-disable-next-line no-console
+    console.error(e);
+    return null;
+  }
+};