@arizeai/phoenix-client 5.8.2 → 6.0.0

package/src/experiments/runExperiment.ts

@@ -3,22 +3,24 @@ import {
   OpenInferenceSpanKind,
   SemanticConventions,
 } from "@arizeai/openinference-semantic-conventions";
+import type { NodeTracerProvider, Tracer } from "@arizeai/phoenix-otel";
 import {
   createNoOpProvider,
   type DiagLogLevel,
-  NodeTracerProvider,
   objectAsAttributes,
   register,
   SpanStatusCode,
   trace,
-  Tracer,
 } from "@arizeai/phoenix-otel";
+import { queue } from "async";
+import invariant from "tiny-invariant";
 
 import { createClient, type PhoenixClient } from "../client";
 import { getDataset } from "../datasets/getDataset";
-import { AnnotatorKind } from "../types/annotations";
-import { ClientFn } from "../types/core";
-import {
+import { createLogger, type Logger } from "../logger";
+import type { AnnotatorKind } from "../types/annotations";
+import type { ClientFn } from "../types/core";
+import type {
   Dataset,
   DatasetSelector,
   Example,
@@ -34,7 +36,6 @@ import type {
   ExperimentTask,
   RanExperiment,
 } from "../types/experiments";
-import { type Logger } from "../types/logger";
 import { ensureString } from "../utils/ensureString";
 import { pluralize } from "../utils/pluralize";
 import { promisifyResult } from "../utils/promisifyResult";
@@ -44,12 +45,14 @@ import {
   getDatasetUrl,
   getExperimentUrl,
 } from "../utils/urlUtils";
-
 import { getExperimentInfo } from "./getExperimentInfo";
 import { getExperimentEvaluators } from "./helpers";
-
-import { queue } from "async";
-import invariant from "tiny-invariant";
+import {
+  logEvalSummary,
+  logLinks,
+  logTaskSummary,
+  PROGRESS_PREFIX,
+} from "./logging";
 
 /**
  * Validate that a repetition is valid
@@ -170,7 +173,7 @@ export async function runExperiment({
   dataset: datasetSelector,
   task,
   evaluators,
-  logger = console,
+  logger = createLogger(),
   record = true,
   concurrency = 5,
   dryRun = false,
@@ -284,35 +287,39 @@ export async function runExperiment({
   }
   if (!record) {
     logger.info(
-      `🔧 Running experiment in readonly mode. Results will not be recorded.`
+      `Running experiment in readonly mode. Results will not be recorded.`
     );
   }
 
+  const links: Array<{ label: string; url: string }> = [];
   if (!isDryRun && client.config.baseUrl) {
-    const datasetUrl = getDatasetUrl({
-      baseUrl: client.config.baseUrl,
-      datasetId: dataset.id,
+    links.push({
+      label: "Dataset",
+      url: getDatasetUrl({
+        baseUrl: client.config.baseUrl,
+        datasetId: dataset.id,
+      }),
     });
-    const datasetExperimentsUrl = getDatasetExperimentsUrl({
-      baseUrl: client.config.baseUrl,
-      datasetId: dataset.id,
+    links.push({
+      label: "Experiments",
+      url: getDatasetExperimentsUrl({
+        baseUrl: client.config.baseUrl,
+        datasetId: dataset.id,
+      }),
     });
-    const experimentUrl = getExperimentUrl({
-      baseUrl: client.config.baseUrl,
-      datasetId: dataset.id,
-      experimentId: experiment.id,
+    links.push({
+      label: "Experiment",
+      url: getExperimentUrl({
+        baseUrl: client.config.baseUrl,
+        datasetId: dataset.id,
+        experimentId: experiment.id,
+      }),
     });
-
-    logger.info(`📊 View dataset: ${datasetUrl}`);
-    logger.info(`📺 View dataset experiments: ${datasetExperimentsUrl}`);
-    logger.info(`🔗 View this experiment: ${experimentUrl}`);
   }
 
+  const evCount = evaluators?.length ?? 0;
   logger.info(
-    `🧪 Starting experiment "${experimentName || `<unnamed>`}" on dataset "${dataset.id}" with task "${task.name}" and ${evaluators?.length ?? 0} ${pluralize(
-      "evaluator",
-      evaluators?.length ?? 0
-    )} and ${concurrency} concurrent runs`
+    `${PROGRESS_PREFIX.start}Experiment ${experimentName || "<unnamed>"} (dataset ${dataset.name}, ${nExamples} ${pluralize("example", nExamples)}, ${evCount} ${pluralize("evaluator", evCount)})`
   );
 
   const runs: Record<ExperimentRunID, ExperimentRun> = {};
@@ -331,13 +338,27 @@ export async function runExperiment({
     tracer: taskTracer,
     repetitions,
   });
-  logger.info(`✅ Task runs completed`);
+  const taskRuns = Object.values(runs);
+  const taskErrors = taskRuns.filter((run) => run.error != null).length;
+  const taskTotal = nExamples * repetitions;
+  const taskOkStr =
+    taskErrors > 0
+      ? `${taskRuns.length - taskErrors}/${taskTotal} ok  (${taskErrors} failed)`
+      : `${taskRuns.length}/${taskTotal} ok`;
+  logger.info(`${PROGRESS_PREFIX.completed}Tasks ${taskOkStr}`);
 
   const ranExperiment: RanExperiment = {
     ...experiment,
     runs,
   };
 
+  if (evaluators && evaluators.length > 0) {
+    const evNames = getExperimentEvaluators(evaluators)
+      .map((evaluator) => evaluator.name)
+      .join(", ");
+    logger.info(`${PROGRESS_PREFIX.start}Evaluations (${evNames})`);
+  }
+
   const { evaluationRuns } = await evaluateExperiment({
     experiment: ranExperiment,
     evaluators: evaluators ?? [],
@@ -351,8 +372,6 @@ export async function runExperiment({
   });
   ranExperiment.evaluationRuns = evaluationRuns;
 
-  logger.info(`✅ Experiment ${experiment.id} completed`);
-
   // Refresh experiment info from server to get updated counts (non-dry-run only)
   if (!isDryRun) {
     const updatedExperiment = await getExperimentInfo({
@@ -363,15 +382,19 @@ export async function runExperiment({
     Object.assign(ranExperiment, updatedExperiment);
   }
 
-  if (!isDryRun && client.config.baseUrl) {
-    const experimentUrl = getExperimentUrl({
-      baseUrl: client.config.baseUrl,
-      datasetId: dataset.id,
-      experimentId: experiment.id,
-    });
-    logger.info(`🔍 View results: ${experimentUrl}`);
+  logTaskSummary(logger, {
+    nExamples,
+    repetitions,
+    nRuns: taskRuns.length,
+    nErrors: taskErrors,
+  });
+
+  if (ranExperiment.evaluationRuns && ranExperiment.evaluationRuns.length > 0) {
+    logEvalSummary(logger, ranExperiment.evaluationRuns);
   }
 
+  logLinks(logger, links);
+
   return ranExperiment;
 }
 
@@ -420,7 +443,9 @@ function runTaskWithExamples({
     "repetitions must be an integer greater than 0"
   );
 
-  logger.info(`🔧 Running task "${task.name}" on dataset "${dataset.id}"`);
+  logger.debug(
+    `${PROGRESS_PREFIX.start}Tasks (${nExamples} ${pluralize("example", nExamples)} × ${repetitions} ${pluralize("repetition", repetitions)})`
+  );
   const run = async ({
     example,
     repetitionNumber,
@@ -429,9 +454,7 @@ function runTaskWithExamples({
     repetitionNumber: number;
   }) => {
     return tracer.startActiveSpan(`Task: ${task.name}`, async (span) => {
-      logger.info(
-        `🔧 Running task "${task.name}" on example "${example.id} of dataset "${dataset.id}"`
-      );
+      logger.debug(`${PROGRESS_PREFIX.progress}Task on example ${example.id}`);
       const traceId = span.spanContext().traceId;
       const thisRun: ExperimentRun = {
         id: localId(), // initialized with local id, will be replaced with server-assigned id when dry run is false
@@ -522,7 +545,7 @@ export async function evaluateExperiment({
   experiment,
   evaluators,
   client: _client,
-  logger = console,
+  logger = createLogger(),
   concurrency = 5,
   dryRun = false,
   setGlobalTracerProvider = true,
@@ -624,21 +647,6 @@ export async function evaluateExperiment({
       evaluationRuns: [],
     };
   }
-  logger.info(
-    `🧠 Evaluating experiment "${experiment.id}" with ${evaluators?.length ?? 0} ${pluralize(
-      "evaluator",
-      evaluators?.length ?? 0
-    )}`
-  );
-
-  if (!isDryRun && client.config.baseUrl) {
-    const experimentUrl = getExperimentUrl({
-      baseUrl: client.config.baseUrl,
-      datasetId: experiment.datasetId,
-      experimentId: experiment.id,
-    });
-    logger.info(`🔗 View experiment evaluation: ${experimentUrl}`);
-  }
   type EvaluationId = string;
   const evaluationRuns: Record<EvaluationId, ExperimentEvaluationRun> = {};
 
@@ -725,7 +733,7 @@ export async function evaluateExperiment({
     concurrency
   );
   if (!evaluatorsAndRuns.length) {
-    logger.info(`⛔ No evaluators to run`);
+    logger.warn(`No evaluators to run`);
     return {
       ...experiment,
       evaluationRuns: [],
@@ -735,13 +743,22 @@ export async function evaluateExperiment({
     evaluatorsQueue.push(evaluatorAndRun, (err) => {
       if (err) {
         logger.error(
-          `❌ Error running evaluator "${evaluatorAndRun.evaluator.name}" on run "${evaluatorAndRun.run.id}": ${err}`
+          `Error running evaluator "${evaluatorAndRun.evaluator.name}" on run "${evaluatorAndRun.run.id}": ${err}`
         );
       }
     })
   );
   await evaluatorsQueue.drain();
-  logger.info(`✅ Evaluation runs completed`);
+  const evalTotal = Object.values(evaluationRuns).length;
+  const evalErrors = Object.values(evaluationRuns).filter(
+    (ev) => ev.error != null
+  ).length;
+  const evalExpected = runsToEvaluate.length * normalizedEvaluators.length;
+  const evalOkStr =
+    evalErrors > 0
+      ? `${evalTotal - evalErrors}/${evalExpected} ok  (${evalErrors} failed)`
+      : `${evalTotal}/${evalExpected} ok`;
+  logger.info(`${PROGRESS_PREFIX.completed}Evaluations ${evalOkStr}`);
 
   if (provider) {
     await provider.shutdown();
@@ -778,8 +795,8 @@ async function runEvaluator({
   const example = exampleCache[run.datasetExampleId];
   invariant(example, `Example "${run.datasetExampleId}" not found`);
   const evaluate = async () => {
-    logger.info(
-      `🧠 Evaluating run "${run.id}" with evaluator "${evaluator.name}"`
+    logger.debug(
+      `${PROGRESS_PREFIX.progress}Eval ${evaluator.name} on run ${run.id}`
     );
     const thisEval: ExperimentEvaluationRun = {
       id: localId(),
@@ -800,13 +817,10 @@ async function runEvaluator({
         metadata: example?.metadata,
       });
       thisEval.result = result;
-      logger.info(
-        `✅ Evaluator "${evaluator.name}" on run "${run.id}" completed`
-      );
     } catch (error) {
       thisEval.error = error instanceof Error ? error.message : "Unknown error";
       logger.error(
-        `❌ Evaluator "${evaluator.name}" on run "${run.id}" failed: ${thisEval.error}`
+        `Evaluator "${evaluator.name}" on run "${run.id}" failed: ${thisEval.error}`
       );
     }
     thisEval.endTime = new Date();

package/src/logger.ts

@@ -0,0 +1,82 @@
+import {
+  ENV_PHOENIX_LOG_LEVEL,
+  getStrFromEnvironment,
+} from "@arizeai/phoenix-config";
+
+export type LogLevel = "debug" | "info" | "warn" | "error" | "silent";
+
+export type Logger = {
+  info: (message: string) => void;
+  warn: (message: string) => void;
+  error: (message: string) => void;
+  log: (message: string) => void;
+  debug: (message: string) => void;
+  table: (data: unknown) => void;
+};
+
+/**
+ * A logger that does nothing
+ */
+export const noopLogger = {
+  info: () => undefined,
+  warn: () => undefined,
+  error: () => undefined,
+  log: () => undefined,
+  debug: () => undefined,
+  table: () => undefined,
+} satisfies Logger;
+
+const VALID_LOG_LEVELS: readonly LogLevel[] = [
+  "debug",
+  "info",
+  "warn",
+  "error",
+  "silent",
+];
+
+function isLogLevel(value: string): value is LogLevel {
+  return VALID_LOG_LEVELS.includes(value as LogLevel);
+}
+
+function getLogLevelFromEnvironment(): LogLevel {
+  const value = getStrFromEnvironment(ENV_PHOENIX_LOG_LEVEL);
+  if (value && isLogLevel(value)) {
+    return value;
+  }
+  return "info";
+}
+
+/**
+ * Creates a logger with the given log level, optionally wrapping a custom logger.
+ * If no level is provided, reads PHOENIX_LOG_LEVEL from the environment, defaulting to "info".
+ * If a custom logger is provided, it is used as the underlying output target instead of console.
+ *
+ * Log level hierarchy (lowest to highest): debug < info < warn < error < silent
+ * Messages below the configured level are suppressed.
+ *
+ * @example
+ * const logger = createLogger(); // uses PHOENIX_LOG_LEVEL env var, defaults to "info"
+ * const logger = createLogger({ level: "debug" }); // show all messages including progress
+ * const logger = createLogger({ level: "silent" }); // suppress all output
+ * const logger = createLogger({ logger: myWinstonLogger }); // wrap a custom logger
+ * const logger = createLogger({ level: "warn", logger: myWinstonLogger }); // custom + level filter
+ */
+export function createLogger({
+  level,
+  logger,
+}: { level?: LogLevel; logger?: Logger } = {}): Logger {
+  const resolvedLevel = level ?? getLogLevelFromEnvironment();
+  const levelIndex = VALID_LOG_LEVELS.indexOf(resolvedLevel);
+  const shouldLog = (l: LogLevel) => VALID_LOG_LEVELS.indexOf(l) >= levelIndex;
+
+  const base = logger ?? console;
+
+  return {
+    debug: shouldLog("debug") ? (msg) => base.debug(msg) : () => undefined,
+    info: shouldLog("info") ? (msg) => base.info(msg) : () => undefined,
+    warn: shouldLog("warn") ? (msg) => base.warn(msg) : () => undefined,
+    error: shouldLog("error") ? (msg) => base.error(msg) : () => undefined,
+    log: shouldLog("info") ? (msg) => base.log(msg) : () => undefined,
+    table: shouldLog("info") ? (data) => base.table(data) : () => undefined,
+  };
+}

package/src/prompts/createPrompt.ts

@@ -1,6 +1,6 @@
 import { createClient } from "../client";
-import { ClientFn } from "../types/core";
-import {
+import type { ClientFn } from "../types/core";
+import type {
   AnthropicInvocationParameters,
   AwsInvocationParameters,
   AzureOpenAIInvocationParameters,

package/src/prompts/getPrompt.ts

@@ -1,6 +1,6 @@
 import { createClient } from "../client";
-import { ClientFn } from "../types/core";
-import { PromptSelector, PromptVersion } from "../types/prompts";
+import type { ClientFn } from "../types/core";
+import type { PromptSelector, PromptVersion } from "../types/prompts";
 import { getPromptBySelector } from "../utils/getPromptBySelector";
 
 /**

package/src/prompts/listPrompts.ts

@@ -1,9 +1,9 @@
-import { createClient } from "../client";
-import { ClientFn } from "../types/core";
-import { Prompt } from "../types/prompts";
-
 import invariant from "tiny-invariant";
 
+import { createClient } from "../client";
+import type { ClientFn } from "../types/core";
+import type { Prompt } from "../types/prompts";
+
 export type ListPromptsParams = ClientFn;
 
 /**

package/src/prompts/sdks/constants.ts

@@ -1,4 +1,4 @@
-import { PromptModelProvider } from "../../types/prompts";
+import type { PromptModelProvider } from "../../types/prompts";
 
 export const SUPPORTED_SDKS = ["openai", "anthropic", "ai"] as const;
 

package/src/prompts/sdks/toAI.ts

@@ -1,15 +1,10 @@
-import {
-  safelyConvertMessageToProvider,
-  safelyConvertToolChoiceToProvider,
-} from "../../schemas/llm/converters";
-import { VercelAIToolChoice } from "../../schemas/llm/vercel/toolChoiceSchemas";
-import { formatPromptMessages } from "../../utils/formatPromptMessages";
-
-import { toSDKParamsBase, Variables } from "./types";
-
-import { type ModelMessage, type ToolChoice, type ToolSet } from "ai";
+import type { ModelMessage, ToolChoice, ToolSet } from "ai";
 import invariant from "tiny-invariant";
 
+import { safelyConvertMessageToProvider } from "../../schemas/llm/converters";
+import { formatPromptMessages } from "../../utils/formatPromptMessages";
+import type { Variables, toSDKParamsBase } from "./types";
+
 export type PartialAIParams = {
   messages: ModelMessage[];
   /**
@@ -27,8 +22,8 @@ export type ToAIParams<V extends Variables> = toSDKParamsBase<V>;
 /**
  * Converts a Phoenix prompt to Vercel AI sdk params.
  *
- * - note: To use response format, you must pass `prompt.response_format.json` to generateObject or streamObject
- *   directly, through the `schema` argument.
+ * - note: To use response format, you must pass `prompt.response_format.json_schema.schema` to generateObject or streamObject
+ *   via `jsonSchema()`, through the `schema` argument.
  */
 export const toAI = <V extends Variables>({
   prompt,
@@ -67,44 +62,19 @@ export const toAI = <V extends Variables>({
       return vercelAIMessage;
     });
 
-    // convert tools to Vercel AI tool set, which is a map of tool name to tool
-    // TODO: Vercel AI SDK 5 has complex tool schema
-    // let tools: ToolSet | undefined = prompt.tools?.tools.reduce((acc, tool) => {
-    //   if (!tool.function.parameters) {
-    //     return acc;
-    //   }
-    //   const vercelAIToolDefinition = safelyConvertToolDefinitionToProvider({
-    //     toolDefinition: tool,
-    //     targetProvider: "VERCEL_AI",
-    //   });
-    //   invariant(vercelAIToolDefinition, "Tool definition is not valid");
-    //   // TODO: get the symbol working here for validators
-    //   acc[tool.function.name] = vercelAIToolDefinition as unknown as Tool;
-    //   return acc;
-    // }, {} as ToolSet);
-    // const hasTools = Object.keys(tools ?? {}).length > 0;
-    // tools = hasTools ? tools : undefined;
-    const hasTools = false;
     const tools = undefined;
     if (prompt.tools?.tools && prompt.tools?.tools.length) {
       // eslint-disable-next-line no-console
       console.warn(
-        "Prompt tools not currently supported in the AI SDK, falling back to no tools"
+        "Phoenix client does not automatically convert tools to AI SDK tools, please manually convert them."
       );
     }
-    let toolChoice: VercelAIToolChoice | undefined =
-      safelyConvertToolChoiceToProvider({
-        toolChoice: prompt.tools?.tool_choice,
-        targetProvider: "VERCEL_AI",
-      }) || undefined;
-    toolChoice = hasTools ? toolChoice : undefined;
 
     // combine base and computed params
     const completionParams: PartialAIParams = {
       ...baseCompletionParams,
       messages,
       tools,
-      toolChoice,
     };
 
     return completionParams;

package/src/prompts/sdks/toAnthropic.ts

@@ -1,16 +1,15 @@
-import { AnthropicToolChoice } from "../../schemas/llm/anthropic/toolChoiceSchemas";
+import type { MessageCreateParams } from "@anthropic-ai/sdk/resources/messages/messages";
+import invariant from "tiny-invariant";
+
+import type { AnthropicToolChoice } from "../../schemas/llm/anthropic/toolChoiceSchemas";
 import {
   safelyConvertMessageToProvider,
   safelyConvertToolChoiceToProvider,
   safelyConvertToolDefinitionToProvider,
 } from "../../schemas/llm/converters";
 import { formatPromptMessages } from "../../utils/formatPromptMessages";
-
 import type { toSDKParamsBase, Variables } from "./types";
 
-import type { MessageCreateParams } from "@anthropic-ai/sdk/resources/messages/messages";
-import invariant from "tiny-invariant";
-
 // We must re-export these types so that they are included in the phoenix-client distribution
 export type { MessageCreateParams };
 

package/src/prompts/sdks/toOpenAI.ts

@@ -1,21 +1,20 @@
+import type {
+  ChatCompletionCreateParams,
+  ChatCompletionMessageParam,
+  ResponseFormatJSONSchema,
+} from "openai/resources";
+import invariant from "tiny-invariant";
+
 import {
   safelyConvertMessageToProvider,
   safelyConvertToolChoiceToProvider,
   safelyConvertToolDefinitionToProvider,
 } from "../../schemas/llm/converters";
-import { OpenaiToolChoice } from "../../schemas/llm/openai/toolChoiceSchemas";
+import type { OpenaiToolChoice } from "../../schemas/llm/openai/toolChoiceSchemas";
 import { phoenixResponseFormatToOpenAI } from "../../schemas/llm/phoenixPrompt/converters";
 import { formatPromptMessages } from "../../utils/formatPromptMessages";
-
 import type { toSDKParamsBase, Variables } from "./types";
 
-import type {
-  ChatCompletionCreateParams,
-  ChatCompletionMessageParam,
-  ResponseFormatJSONSchema,
-} from "openai/resources";
-import invariant from "tiny-invariant";
-
 // We must re-export these types so that they are included in the phoenix-client distribution
 export type {
   ChatCompletionCreateParams,

package/src/prompts/sdks/toSDK.ts

@@ -1,11 +1,10 @@
-import { assertUnreachable } from "../../utils/assertUnreachable";
+import invariant from "tiny-invariant";
 
+import { assertUnreachable } from "../../utils/assertUnreachable";
 import { toAI } from "./toAI";
 import { toAnthropic } from "./toAnthropic";
 import { toOpenAI } from "./toOpenAI";
-import { SupportedSDK, toSDKParamsBase, Variables } from "./types";
-
-import invariant from "tiny-invariant";
+import type { SupportedSDK, toSDKParamsBase, Variables } from "./types";
 
 /**
  * Parameters for an SDK conversion function

package/src/prompts/sdks/types.ts

@@ -1,5 +1,4 @@
 import type { PromptVersion } from "../../types/prompts";
-
 import type { SUPPORTED_SDKS } from "./constants";
 
 /**

package/src/schemas/jsonLiteralSchema.ts

@@ -24,6 +24,6 @@ export const jsonLiteralSchema: z.ZodType<JSONLiteral> = z.lazy(() =>
   z.union([
     literalSchema,
     z.array(jsonLiteralSchema),
-    z.record(jsonLiteralSchema),
+    z.record(z.string(), jsonLiteralSchema),
   ])
 );

package/src/schemas/jsonSchema.ts

@@ -1,7 +1,7 @@
 import z from "zod";
 
 const jsonSchemaPropertiesSchema = z
-  .object({
+  .looseObject({
     type: z
       .enum([
         "string",
@@ -19,29 +19,22 @@ const jsonSchemaPropertiesSchema = z
       .describe("A description of the parameter"),
     enum: z.array(z.string()).optional().describe("The allowed values"),
   })
-  .passthrough()
   .describe("A map of parameter names to their definitions");
 
-export const jsonSchemaZodSchema = z
-  .object({
-    type: z.literal("object"),
-    properties: z.record(
-      z.union([
-        jsonSchemaPropertiesSchema,
-        z
-          .object({ anyOf: z.array(jsonSchemaPropertiesSchema) })
-          .describe("A list of possible parameter names to their definitions"),
-      ])
-    ),
-    required: z
-      .array(z.string())
-      .optional()
-      .describe("The required parameters"),
-    additionalProperties: z
-      .boolean()
-      .optional()
-      .describe(
-        "Whether or not additional properties are allowed in the schema"
-      ),
-  })
-  .passthrough();
+export const jsonSchemaZodSchema = z.looseObject({
+  type: z.literal("object"),
+  properties: z.record(
+    z.string(),
+    z.union([
+      jsonSchemaPropertiesSchema,
+      z
+        .object({ anyOf: z.array(jsonSchemaPropertiesSchema) })
+        .describe("A list of possible parameter names to their definitions"),
+    ])
+  ),
+  required: z.array(z.string()).optional().describe("The required parameters"),
+  additionalProperties: z
+    .boolean()
+    .optional()
+    .describe("Whether or not additional properties are allowed in the schema"),
+});