@arizeai/phoenix-client 0.0.1

package/src/client.ts

@@ -0,0 +1,63 @@
+import createOpenApiClient, { ClientOptions } from "openapi-fetch";
+import type {
+  paths as oapiPathsV1,
+  components as oapiComponentsV1,
+  operations as oapiOperationsV1,
+} from "./__generated__/api/v1.d.ts";
+import {
+  defaultGetEnvironmentOptions,
+  makeDefaultClientOptions,
+} from "./config";
+
+type pathsV1 = oapiPathsV1;
+type componentsV1 = oapiComponentsV1;
+type operationsV1 = oapiOperationsV1;
+
+export type Types = {
+  V1: {
+    paths: pathsV1;
+    components: componentsV1;
+    operations: operationsV1;
+  };
+};
+
+/**
+ * Merge all configuration options according to priority:
+ * defaults < environment < explicit options
+ *
+ * Headers are simply replaced, not merged.
+ */
+export const getMergedOptions = ({
+  options = {},
+  getEnvironmentOptions = defaultGetEnvironmentOptions,
+}: {
+  options?: Partial<ClientOptions>;
+  getEnvironmentOptions?: () => Partial<ClientOptions>;
+} = {}): ClientOptions => {
+  const defaultOptions = makeDefaultClientOptions();
+  const environmentOptions = getEnvironmentOptions();
+  return {
+    ...defaultOptions,
+    ...environmentOptions,
+    ...options,
+  };
+};
+
+/**
+ * Create a Phoenix client.
+ *
+ * @param configuration - The configuration to use for the client.
+ * @param configuration.options - The options to use for the client's OpenAPI Fetch wrapper.
+ * @returns The Phoenix client.
+ */
+export const createClient = (
+  config: {
+    options?: Partial<ClientOptions>;
+    getEnvironmentOptions?: () => Partial<ClientOptions>;
+  } = {}
+) => {
+  const mergedOptions = getMergedOptions(config);
+  return createOpenApiClient<pathsV1>(mergedOptions);
+};
+
+export type PhoenixClient = ReturnType<typeof createClient>;

package/src/config.ts

@@ -0,0 +1,68 @@
+import type { ClientOptions } from "openapi-fetch";
+import z from "zod";
+
+const phoenixEnvironmentSchema = z.object({
+  PHOENIX_HOST: z.string().optional(),
+  PHOENIX_CLIENT_HEADERS: z
+    .string()
+    .transform((s) => JSON.parse(s))
+    .transform((o) => z.record(z.string()).parse(o))
+    .optional(),
+});
+
+type PhoenixEnvironment = z.infer<typeof phoenixEnvironmentSchema>;
+
+/**
+ * Parse environment variables from an opaque object into a PhoenixEnvironment object.
+ *
+ * @param environment - The environment variables object-like structure to parse.
+ * @returns The parsed PhoenixEnvironment object.
+ */
+const fromEnvironment = (environment: unknown) => {
+  return phoenixEnvironmentSchema.safeParse(environment)?.data ?? {};
+};
+
+/**
+ * Convert a PhoenixEnvironment object into a ClientOptions object.
+ *
+ * @param environment - The PhoenixEnvironment object to convert.
+ * @returns The converted ClientOptions object.
+ */
+const phoenixEnvironmentToClientOptions = (
+  environment: PhoenixEnvironment
+): Partial<ClientOptions> => {
+  const options: Partial<ClientOptions> = {
+    baseUrl: environment.PHOENIX_HOST,
+    headers: environment.PHOENIX_CLIENT_HEADERS,
+  };
+
+  // filter out undefined values
+  // this will prevent clobbering over default values when merging
+  return Object.fromEntries(
+    Object.entries(options).filter(([_, v]) => v !== undefined)
+  );
+};
+
+/**
+ * Get the environment options from the environment.
+ *
+ * @returns The environment options as a Partial<ClientOptions> object.
+ */
+export const defaultGetEnvironmentOptions = (): Partial<ClientOptions> => {
+  // feature detect process and process.env
+  if (typeof process !== "object" || typeof process.env !== "object") {
+    return {};
+  }
+  return phoenixEnvironmentToClientOptions(fromEnvironment(process.env));
+};
+
+/**
+ * Make the default client options.
+ *
+ * @returns The default client options as a Partial<ClientOptions> object.
+ */
+export const makeDefaultClientOptions = (): Partial<ClientOptions> => {
+  return {
+    baseUrl: "http://localhost:6006",
+  };
+};

package/src/experiments/index.ts

@@ -0,0 +1 @@
+export * from "./runExperiment";

package/src/experiments/runExperiment.ts

@@ -0,0 +1,367 @@
+import { Dataset, Example } from "../types/datasets";
+import { createClient, type PhoenixClient } from "../client";
+import type {
+  Evaluator,
+  Experiment,
+  ExperimentEvaluationRun,
+  ExperimentParameters,
+  ExperimentRun,
+  ExperimentTask,
+  RanExperiment,
+} from "../types/experiments";
+import { promisifyResult } from "../utils/promisifyResult";
+import invariant from "tiny-invariant";
+import { pluralize } from "../utils/pluralize";
+
+export type Logger = {
+  info: (message: string) => void;
+  error: (message: string) => void;
+  log: (message: string) => void;
+};
+
+export type RunExperimentParams = {
+  /**
+   * An optional name for the experiment.
+   * Defaults to the dataset name + a timestamp
+   */
+  experimentName?: string;
+  client?: PhoenixClient;
+  dataset: Dataset | string | Example[];
+  task: ExperimentTask;
+  evaluators?: Evaluator[];
+  repetitions?: number;
+  /**
+   * The project under which the experiment task traces are recorded
+   */
+  projectName?: string;
+  logger?: Logger;
+};
+
+/**
+ * Run an experiment.
+ */
+export async function runExperiment({
+  experimentName: _experimentName,
+  client: _client,
+  dataset: _dataset,
+  task,
+  evaluators,
+  repetitions = 1,
+  projectName = "default",
+  logger = console,
+}: RunExperimentParams): Promise<RanExperiment> {
+  const client = _client ?? createClient();
+  const dataset = await getDataset({ dataset: _dataset, client });
+  invariant(dataset, `Dataset not found`);
+  invariant(dataset.examples.length > 0, `Dataset has no examples`);
+  const experimentName =
+    _experimentName ?? `${dataset.name}-${new Date().toISOString()}`;
+  const experimentParams: ExperimentParameters = {
+    nRepetitions: repetitions,
+    // TODO: Make configurable?
+    nExamples: dataset.examples.length,
+  };
+  const experiment: Experiment = {
+    id: id(),
+    datasetId: dataset.id,
+    datasetVersionId: dataset.versionId,
+    repetitions,
+    projectName,
+  };
+
+  logger.info(
+    `🧪 Starting experiment "${experimentName}" on dataset "${dataset.id}" with task "${task.name}" and ${evaluators?.length ?? 0} ${pluralize(
+      "evaluator",
+      evaluators?.length ?? 0
+    )}`
+  );
+
+  logger.info(
+    `🔁 Running ${repetitions} ${pluralize("repetition", repetitions)} of task "${task.name}"`
+  );
+
+  // Run task against all examples, for each repetition
+  type ExperimentRunId = string;
+  const runs: Record<ExperimentRunId, ExperimentRun> = {};
+  await Promise.all(
+    Array.from({ length: repetitions }, (_, i) =>
+      runTask({
+        repetition: i + 1,
+        experimentId: experiment.id,
+        task,
+        dataset,
+        logger,
+        onComplete: (run) => {
+          runs[run.id] = run;
+        },
+      })
+    )
+  );
+  logger.info(`✅ Task runs completed`);
+
+  const ranExperiment: RanExperiment = {
+    ...experiment,
+    params: experimentParams,
+    runs,
+  };
+
+  const { evaluationRuns } = await evaluateExperiment({
+    experiment: ranExperiment,
+    evaluators: evaluators ?? [],
+    client,
+    logger,
+  });
+  ranExperiment.evaluationRuns = evaluationRuns;
+
+  logger.info(`✅ Experiment ${experiment.id} completed`);
+
+  return ranExperiment;
+}
+
+/**
+ * Run a task against all examples in a dataset.
+ */
+function runTask({
+  experimentId,
+  task,
+  dataset,
+  repetition,
+  onComplete,
+  logger,
+}: {
+  /** The id of the experiment */
+  experimentId: string;
+  /** The task to run */
+  task: ExperimentTask;
+  /** The dataset to run the task on */
+  dataset: Dataset;
+  /** The repetition number */
+  repetition: number;
+  /** A callback to call when the task is complete */
+  onComplete: (run: ExperimentRun) => void;
+  /** The logger to use */
+  logger: Logger;
+}) {
+  logger.info(
+    `🔧 (${repetition}) Running task "${task.name}" on dataset "${dataset.id}"`
+  );
+  const run = async (example: Example) => {
+    const thisRun: ExperimentRun = {
+      id: id(),
+      traceId: id(),
+      experimentId,
+      datasetExampleId: example.id,
+      startTime: new Date(),
+      repetitionNumber: repetition,
+      endTime: new Date(), // will get replaced with actual end time
+      output: null,
+      error: null,
+    };
+    try {
+      const taskOutput = await promisifyResult(task(example));
+      // TODO: why doesn't run output type match task output type?
+      thisRun.output = JSON.stringify(taskOutput);
+    } catch (error) {
+      thisRun.error = error instanceof Error ? error.message : "Unknown error";
+    }
+    thisRun.endTime = new Date();
+    onComplete(thisRun);
+  };
+  return Promise.all(dataset.examples.map(run));
+}
+
+export async function evaluateExperiment({
+  experiment,
+  evaluators,
+  client: _client,
+  logger,
+}: {
+  /**
+   * The experiment to evaluate
+   * @todo also accept Experiment, and attempt to fetch the runs from the server
+   **/
+  experiment: RanExperiment;
+  /** The evaluators to use */
+  evaluators: Evaluator[];
+  /** The client to use */
+  client?: PhoenixClient;
+  /** The logger to use */
+  logger: Logger;
+}): Promise<RanExperiment> {
+  const client = _client ?? createClient();
+  const dataset = await getDataset({ dataset: experiment.datasetId, client });
+  invariant(dataset, `Dataset "${experiment.datasetId}" not found`);
+  invariant(
+    dataset.examples.length > 0,
+    `Dataset "${experiment.datasetId}" has no examples`
+  );
+  invariant(experiment.runs, `Experiment "${experiment.id}" has no runs`);
+
+  if (evaluators?.length === 0) {
+    return {
+      ...experiment,
+      evaluationRuns: [],
+    };
+  }
+
+  logger.info(
+    `🧠 Evaluating experiment "${experiment.id}" with ${evaluators?.length ?? 0} ${pluralize(
+      "evaluator",
+      evaluators?.length ?? 0
+    )}`
+  );
+  type EvaluationId = string;
+  const evaluationRuns: Record<EvaluationId, ExperimentEvaluationRun> = {};
+
+  const examplesById: Record<string, Example> = {};
+  for (const example of dataset.examples) {
+    examplesById[example.id] = example;
+  }
+
+  const onEvaluationComplete = (run: ExperimentEvaluationRun) => {
+    evaluationRuns[run.id] = run;
+  };
+
+  // Run evaluators against all runs
+  await Promise.all(
+    evaluators.map((evaluator) =>
+      Promise.all(
+        Object.values(experiment.runs).map((run) =>
+          runEvaluator({
+            evaluator,
+            run,
+            exampleCache: examplesById,
+            onComplete: onEvaluationComplete,
+          })
+        )
+      )
+    )
+  );
+
+  logger.info(`✅ Evaluation runs completed`);
+
+  return {
+    ...experiment,
+    evaluationRuns: Object.values(evaluationRuns),
+  };
+}
+
+/**
+ * Run an evaluator against a run.
+ */
+async function runEvaluator({
+  evaluator,
+  run,
+  exampleCache,
+  onComplete,
+}: {
+  evaluator: Evaluator;
+  run: ExperimentRun;
+  exampleCache: Record<string, Example>;
+  onComplete: (run: ExperimentEvaluationRun) => void;
+}) {
+  const example = exampleCache[run.datasetExampleId];
+  invariant(example, `Example "${run.datasetExampleId}" not found`);
+  const evaluate = async () => {
+    const thisEval: ExperimentEvaluationRun = {
+      id: id(),
+      traceId: id(),
+      experimentRunId: run.id,
+      startTime: new Date(),
+      endTime: new Date(), // will get replaced with actual end time
+      name: evaluator.name,
+      result: null,
+      error: null,
+      annotatorKind: "LLM", // TODO: make configurable via evaluator def
+    };
+    try {
+      const result = await evaluator.evaluate({
+        input: example.input,
+        output: run.output ?? null,
+        expected: example.output,
+        metadata: example.metadata,
+      });
+      thisEval.result = result;
+    } catch (error) {
+      thisEval.error = error instanceof Error ? error.message : "Unknown error";
+    }
+    thisEval.endTime = new Date();
+    onComplete(thisEval);
+  };
+
+  return evaluate();
+}
+
+/**
+ * Return a dataset object from the input.
+ *
+ * If the input is a string, assume it is a dataset id and fetch the dataset from the client.
+ * If the input is an array of examples, create a new dataset from the examples then return it.
+ * If the input is a dataset, return it as is.
+ *
+ * @param dataset - The dataset to get.
+ * @returns The dataset.
+ */
+async function getDataset({
+  dataset,
+  client,
+}: {
+  dataset: Dataset | string | Example[];
+  client: PhoenixClient;
+}): Promise<Dataset> {
+  if (typeof dataset === "string") {
+    const datasetResponse = await client
+      .GET(`/v1/datasets/{id}`, { params: { path: { id: dataset } } })
+      .then((d) => d.data?.data);
+    invariant(datasetResponse, `Dataset ${dataset} not found`);
+    const examples = await client
+      .GET(`/v1/datasets/{id}/examples`, { params: { path: { id: dataset } } })
+      .then((e) => e.data?.data);
+    invariant(examples, `Examples for dataset ${dataset} not found`);
+    const datasetWithExamples: Dataset = {
+      ...datasetResponse,
+      examples: examples.examples.map((example) => ({
+        ...example,
+        updatedAt: new Date(example.updated_at),
+      })),
+      versionId: examples.version_id,
+    };
+    return datasetWithExamples;
+  }
+  if (Array.isArray(dataset)) {
+    throw new Error("TODO: implement dataset creation from examples");
+  }
+  return dataset;
+}
+
+/**
+ * Wrap an evaluator function in an object with a name property.
+ *
+ * @param name - The name of the evaluator.
+ * @param evaluate - The evaluator function.
+ * @returns The evaluator object.
+ */
+export function asEvaluator(
+  name: string,
+  evaluate: Evaluator["evaluate"]
+): Evaluator {
+  return {
+    name,
+    evaluate,
+  };
+}
+
+let _id = 1000;
+
+/**
+ * Generate a unique id.
+ *
+ * @deprecated Use id generated by phoenix instead.
+ * @returns A unique id.
+ */
+export function id(): string {
+  return (() => {
+    _id++;
+    return _id.toString();
+  })();
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+export * from "./client";
+
+export * from "./experiments";

package/src/types/annotations.ts

@@ -0,0 +1 @@
+export type AnnotatorKind = "HUMAN" | "LLM";

package/src/types/core.ts

@@ -0,0 +1,6 @@
+/**
+ * A uniquely identifiable node in Phoenix
+ */
+export interface Node {
+  id: string;
+}

package/src/types/datasets.ts

@@ -0,0 +1,22 @@
+import { Node } from "./core";
+
+/**
+ * An example is a record to feed into an AI task
+ */
+export interface Example extends Node {
+  id: string;
+  updatedAt: Date;
+  input: Record<string, unknown>;
+  output: Record<string, unknown> | null;
+  metadata: Record<string, unknown>;
+}
+
+/**
+ * A dataset is a collection of examples for an AI task
+ */
+export interface Dataset extends Node {
+  id: string;
+  name: string;
+  versionId: string;
+  examples: Example[];
+}

package/src/types/experiments.ts

@@ -0,0 +1,108 @@
+import { AnnotatorKind } from "./annotations";
+import { Node } from "./core";
+import { Example } from "./datasets";
+
+/**
+ * An experiment is a set of task runs on a dataset version
+ */
+export interface Experiment extends Node {
+  datasetId: string;
+  datasetVersionId: string;
+  repetitions: number;
+  /**
+   * The project under which the experiment task traces are recorded
+   */
+  projectName: string;
+}
+
+export interface RanExperiment extends Experiment {
+  params: ExperimentParameters;
+  runs: Record<string, ExperimentRun>;
+  evaluationRuns?: ExperimentEvaluationRun[];
+}
+
+/**
+ * The result of running an experiment on a single example
+ */
+export interface ExperimentRun extends Node {
+  startTime: Date;
+  endTime: Date;
+  /**
+   * What experiment the run belongs to
+   */
+  experimentId: string;
+  datasetExampleId: string;
+  repetitionNumber: number;
+  output?: string | Record<string, unknown> | null;
+  error: string | null;
+  traceId: string | null;
+}
+
+export type EvaluatorParams = {
+  /**
+   * The input field of the Dataset Example
+   */
+  input: Example["input"];
+  /**
+   * The output of the task
+   */
+  output: TaskOutput;
+  /**
+   * The expected or reference output of the Dataset Example
+   */
+  expected?: Example["output"];
+  /**
+   * Metadata associated with the Dataset Example
+   */
+  metadata?: Record<string, unknown>;
+};
+
+export type Evaluator = {
+  name: string;
+  evaluate: (
+    args: EvaluatorParams
+  ) => Promise<EvaluationResult> | EvaluationResult;
+};
+
+export type EvaluationResult = {
+  score: number | null;
+  label: string | null;
+  metadata: Record<string, unknown>;
+  explanation: string | null;
+};
+
+export interface ExperimentEvaluationRun extends Node {
+  experimentRunId: string;
+  startTime: Date;
+  endTime: Date;
+  /**
+   * THe name of the evaluation
+   */
+  name: string;
+  annotatorKind: AnnotatorKind;
+  error: string | null;
+  result: EvaluationResult | null;
+  /**
+   * The trace id of the evaluation
+   * This is null if the trace is deleted or never recorded
+   */
+  traceId: string | null;
+}
+
+export type TaskOutput = string | boolean | number | object | null;
+
+export type ExperimentTask = (
+  example: Example
+) => Promise<TaskOutput> | TaskOutput;
+
+export interface ExperimentParameters {
+  /**
+   * The number of examples to run the experiment on
+   */
+  nExamples: number;
+  /**
+   * The number of repetitions to run the experiment
+   * e.g. the number of times to run the task
+   */
+  nRepetitions: number;
+}

package/src/utils/pluralize.ts

@@ -0,0 +1,10 @@
+/**
+ * Pluralize a word based on the count.
+ *
+ * @param word - The word to pluralize.
+ * @param count - The count of the word.
+ * @returns The pluralized word.
+ */
+export function pluralize(word: string, count: number) {
+  return count === 1 ? word : `${word}s`;
+}

package/src/utils/promisifyResult.ts

@@ -0,0 +1,13 @@
+/**
+ * If the incoming function returns a promise, return the promise.
+ * Otherwise, return a promise that resolves to the incoming function's return value.
+ */
+export function promisifyResult<T>(result: T) {
+  if (result instanceof Promise) {
+    return result as T extends Promise<infer U> ? Promise<U> : never;
+  }
+
+  return Promise.resolve(result) as T extends Promise<unknown>
+    ? never
+    : Promise<T>;
+}