@arizeai/phoenix-client 6.5.4 → 6.6.0

package/docs/experiments.mdx

@@ -0,0 +1,376 @@
+---
+title: "Experiments"
+description: "Run experiments with @arizeai/phoenix-client"
+---
+
+The experiments module runs tasks over dataset examples, records experiment runs in Phoenix, and can evaluate each run with either plain experiment evaluators or `@arizeai/phoenix-evals` evaluators.
+
+<section className="hidden" data-agent-context="relevant-source-files" aria-label="Relevant source files">
+  <h2>Relevant Source Files</h2>
+  <ul>
+    <li><code>src/experiments/runExperiment.ts</code> for the task execution flow and return shape</li>
+    <li><code>src/experiments/helpers/getExperimentEvaluators.ts</code> for evaluator normalization</li>
+    <li><code>src/experiments/helpers/fromPhoenixLLMEvaluator.ts</code> for the phoenix-evals bridge</li>
+    <li><code>src/experiments/getExperimentRuns.ts</code> for reading runs back after execution</li>
+    <li><code>src/types/experiments.ts</code> for <code>EvaluatorParams</code> including <code>traceId</code></li>
+    <li><code>src/spans/getSpans.ts</code> for fetching spans by trace ID and span kind</li>
+  </ul>
+</section>
+
+## Two Common Patterns
+
+Use `asExperimentEvaluator()` when your evaluation logic is plain TypeScript.
+
+Use `@arizeai/phoenix-evals` evaluators directly when you want model-backed judging.
+
+## Code-Based Example
+
+If you just want to compare task output against a reference answer or apply deterministic checks, use `asExperimentEvaluator()`:
+
+```ts
+/* eslint-disable no-console */
+import { createDataset } from "@arizeai/phoenix-client/datasets";
+import {
+  asExperimentEvaluator,
+  runExperiment,
+} from "@arizeai/phoenix-client/experiments";
+
+async function main() {
+  const { datasetId } = await createDataset({
+    name: `simple-dataset-${Date.now()}`,
+    description: "a simple dataset",
+    examples: [
+      {
+        input: { name: "John" },
+        output: { text: "Hello, John!" },
+        metadata: {},
+      },
+      {
+        input: { name: "Jane" },
+        output: { text: "Hello, Jane!" },
+        metadata: {},
+      },
+      {
+        input: { name: "Bill" },
+        output: { text: "Hello, Bill!" },
+        metadata: {},
+      },
+    ],
+  });
+
+  const experiment = await runExperiment({
+    dataset: { datasetId },
+    task: async (example) => `hello ${example.input.name}`,
+    evaluators: [
+      asExperimentEvaluator({
+        name: "matches",
+        kind: "CODE",
+        evaluate: async ({ output, expected }) => {
+          const matches = output === expected?.text;
+          return {
+            label: matches ? "matches" : "does not match",
+            score: matches ? 1 : 0,
+            explanation: matches
+              ? "output matches expected"
+              : "output does not match expected",
+            metadata: {},
+          };
+        },
+      }),
+      asExperimentEvaluator({
+        name: "contains-hello",
+        kind: "CODE",
+        evaluate: async ({ output }) => {
+          const matches =
+            typeof output === "string" && output.includes("hello");
+          return {
+            label: matches ? "contains hello" : "does not contain hello",
+            score: matches ? 1 : 0,
+            explanation: matches
+              ? "output contains hello"
+              : "output does not contain hello",
+            metadata: {},
+          };
+        },
+      }),
+    ],
+  });
+
+  console.table(experiment.runs);
+  console.table(experiment.evaluationRuns);
+}
+
+main().catch(console.error);
+```
+
+This pattern is useful when:
+
+- you already know the exact correctness rule
+- you want fast, deterministic evaluation
+- you do not want to call another model during evaluation
+
+## Model-Backed Example
+
+If you want a model-backed experiment with automatic tracing and an LLM-as-a-judge evaluator, this is the core pattern:
+
+```ts
+import { openai } from "@ai-sdk/openai";
+import { createOrGetDataset } from "@arizeai/phoenix-client/datasets";
+import { runExperiment } from "@arizeai/phoenix-client/experiments";
+import type { ExperimentTask } from "@arizeai/phoenix-client/types/experiments";
+import { createClassificationEvaluator } from "@arizeai/phoenix-evals";
+import { generateText } from "ai";
+
+const model = openai("gpt-4o-mini");
+
+const main = async () => {
+  const answersQuestion = createClassificationEvaluator({
+    name: "answersQuestion",
+    model,
+    promptTemplate:
+      "Does the following answer the user's question: <question>{{input.question}}</question><answer>{{output}}</answer>",
+    choices: {
+      correct: 1,
+      incorrect: 0,
+    },
+  });
+
+  const dataset = await createOrGetDataset({
+    name: "correctness-eval",
+    description: "Evaluate the correctness of the model",
+    examples: [
+      {
+        input: {
+          question: "Is ArizeAI Phoenix Open-Source?",
+          context: "ArizeAI Phoenix is Open-Source.",
+        },
+      },
+      // ... more examples
+    ],
+  });
+
+  const task: ExperimentTask = async (example) => {
+    if (typeof example.input.question !== "string") {
+      throw new Error("Invalid input: question must be a string");
+    }
+    if (typeof example.input.context !== "string") {
+      throw new Error("Invalid input: context must be a string");
+    }
+
+    return generateText({
+      model,
+      experimental_telemetry: {
+        isEnabled: true,
+      },
+      prompt: [
+        {
+          role: "system",
+          content: `You answer questions based on this context: ${example.input.context}`,
+        },
+        {
+          role: "user",
+          content: example.input.question,
+        },
+      ],
+    }).then((response) => {
+      if (response.text) {
+        return response.text;
+      }
+      throw new Error("Invalid response: text is required");
+    });
+  };
+
+  const experiment = await runExperiment({
+    experimentName: "answers-question-eval",
+    experimentDescription:
+      "Evaluate the ability of the model to answer questions based on the context",
+    dataset,
+    task,
+    evaluators: [answersQuestion],
+    repetitions: 3,
+  });
+
+  console.log(experiment.id);
+  console.log(Object.values(experiment.runs).length);
+  console.log(experiment.evaluationRuns?.length ?? 0);
+};
+
+main().catch(console.error);
+```
+
+## What This Example Shows
+
+- `createOrGetDataset()` creates or reuses the dataset the experiment will run against
+- `task` receives the full dataset example object
+- `generateText()` emits traces that Phoenix can attach to the experiment when telemetry is enabled
+- `createClassificationEvaluator()` from `@arizeai/phoenix-evals` can be passed directly to `runExperiment()`
+- `runExperiment()` records both task runs and evaluation runs in Phoenix
+
+## Task Inputs
+
+`runExperiment()` calls your task with the full dataset example, not just `example.input`.
+
+That means your task should usually read:
+
+- `example.input` for the task inputs
+- `example.output` for any reference answer
+- `example.metadata` for additional context
+
+In the example above, the task validates `example.input.question` and `example.input.context` before generating a response.
+
+## Evaluator Inputs
+
+When an evaluator runs, it receives a normalized object with these fields:
+
+| Field | Description |
+|--------|-------------|
+| `input` | The dataset example's `input` object |
+| `output` | The task output for that run |
+| `expected` | The dataset example's `output` object |
+| `metadata` | The dataset example's `metadata` object |
+| `traceId` | The OpenTelemetry trace ID of the task run (optional, `string \| null`) |
+
+This is why the `createClassificationEvaluator()` prompt can reference `{{input.question}}` and `{{output}}`.
+
+For code-based evaluators created with `asExperimentEvaluator()`, those same fields are available inside `evaluate({ input, output, expected, metadata, traceId })`.
+
+## Trace-Based Evaluation
+
+Each task run captures an OpenTelemetry trace ID. Evaluators can use `traceId` to fetch the task's spans from Phoenix and evaluate the execution trajectory — for example, verifying that specific tool calls were made or inspecting intermediate steps.
+
+This pattern works best with `evaluateExperiment()` as a separate step after `runExperiment()`, so that all task spans are ingested into Phoenix before the evaluator queries them.
+
+```ts
+import { traceTool } from "@arizeai/openinference-core";
+import { createClient } from "@arizeai/phoenix-client";
+import { createDataset } from "@arizeai/phoenix-client/datasets";
+import {
+  asExperimentEvaluator,
+  evaluateExperiment,
+  runExperiment,
+} from "@arizeai/phoenix-client/experiments";
+import { getSpans } from "@arizeai/phoenix-client/spans";
+
+const client = createClient();
+
+const { datasetId } = await createDataset({
+  client,
+  name: "tool-call-dataset",
+  description: "Questions that require tool use",
+  examples: [
+    {
+      input: { question: "What is the weather in San Francisco?" },
+      output: { expectedTool: "getWeather" },
+      metadata: {},
+    },
+  ],
+});
+
+// Step 1: Run the experiment with traced tool calls
+const experiment = await runExperiment({
+  client,
+  dataset: { datasetId },
+  setGlobalTracerProvider: true,
+  task: async (example) => {
+    // traceTool wraps a function with a TOOL span
+    const getWeather = traceTool(
+      ({ location }: { location: string }) => ({
+        location,
+        temperature: 72,
+        condition: "sunny",
+      }),
+      { name: "getWeather" }
+    );
+
+    const city = (example.input.question as string).match(/in (.+)\?/)?.[1];
+    const result = getWeather({ location: city ?? "Unknown" });
+    return `The weather in ${result.location} is ${result.temperature}F.`;
+  },
+});
+
+const projectName = experiment.projectName!;
+
+// Step 2: Evaluate using traceId to inspect the task's spans
+const evaluated = await evaluateExperiment({
+  client,
+  experiment,
+  evaluators: [
+    asExperimentEvaluator({
+      name: "has-expected-tool-call",
+      kind: "CODE",
+      evaluate: async ({ traceId, expected }) => {
+        if (!traceId) {
+          return { label: "no trace", score: 0 };
+        }
+
+        // Fetch TOOL spans from this task's trace
+        const { spans: toolSpans } = await getSpans({
+          client,
+          project: { projectName },
+          traceIds: [traceId],
+          spanKind: "TOOL",
+        });
+
+        const expectedTool = (expected as { expectedTool?: string })
+          ?.expectedTool;
+        const toolNames = toolSpans.map((s) => s.name);
+        const found = toolNames.some((name) => name.includes(expectedTool!));
+
+        return {
+          label: found ? "tool called" : "no tool call",
+          score: found ? 1 : 0,
+          explanation: found
+            ? `Found: ${toolNames.join(", ")}`
+            : `Expected "${expectedTool}" but found none`,
+        };
+      },
+    }),
+  ],
+});
+```
+
+Key points:
+
+- Use `setGlobalTracerProvider: true` on `runExperiment()` so that child spans from `traceTool` or other OTel instrumentation land in the same trace as the task
+- Use `evaluateExperiment()` as a separate step so spans are ingested before querying
+- Use `getSpans()` with `traceIds` and `spanKind` filters to fetch specific spans from the task trace
+- `traceId` is `null` in dry-run mode since no real traces are recorded
+
+## What `runExperiment()` Returns
+
+The returned object includes the experiment metadata plus the task and evaluation results from the run.
+
+- `experiment.id` is the experiment ID in Phoenix
+- `experiment.projectName` is the Phoenix project that received the task traces
+- `experiment.runs` is a map of run IDs to task run objects
+- `experiment.evaluationRuns` contains evaluator results when evaluators are provided
+
+## Follow-Up Helpers
+
+Use these exports for follow-up workflows:
+
+- `createExperiment`
+- `getExperiment`
+- `getExperimentInfo`
+- `getExperimentRuns`
+- `listExperiments`
+- `resumeExperiment`
+- `resumeEvaluation`
+- `deleteExperiment`
+
+## Tracing Behavior
+
+`runExperiment()` can register a tracer provider for the task run so that task spans and evaluator spans show up in Phoenix during the experiment. This is why tasks that call the AI SDK can still emit traces to Phoenix when global tracing is enabled.
+
+<section className="hidden" data-agent-context="source-map" aria-label="Source map">
+  <h2>Source Map</h2>
+  <ul>
+    <li><code>src/experiments/runExperiment.ts</code></li>
+    <li><code>src/experiments/createExperiment.ts</code></li>
+    <li><code>src/experiments/getExperiment.ts</code></li>
+    <li><code>src/experiments/getExperimentRuns.ts</code></li>
+    <li><code>src/experiments/helpers/getExperimentEvaluators.ts</code></li>
+    <li><code>src/experiments/helpers/fromPhoenixLLMEvaluator.ts</code></li>
+    <li><code>src/experiments/helpers/asExperimentEvaluator.ts</code></li>
+  </ul>
+</section>

package/docs/overview.mdx

@@ -0,0 +1,176 @@
+---
+title: "Overview"
+description: "Typed TypeScript client for Phoenix platform APIs"
+---
+
+`@arizeai/phoenix-client` is the typed TypeScript client for Phoenix platform APIs. It ships a small root REST client plus focused module entrypoints for prompts, datasets, experiments, spans, sessions, and traces.
+
+## Install
+
+```bash
+npm install @arizeai/phoenix-client
+```
+
+## Minimal Example
+
+```ts
+import { createClient } from "@arizeai/phoenix-client";
+import { listDatasets } from "@arizeai/phoenix-client/datasets";
+
+const client = createClient();
+const datasets = await listDatasets({ client });
+```
+
+## Docs And Source In `node_modules`
+
+After install, a coding agent can inspect the installed package directly:
+
+```text
+node_modules/@arizeai/phoenix-client/docs/
+node_modules/@arizeai/phoenix-client/src/
+```
+
+That gives the agent version-matched docs plus the exact implementation and generated API types that shipped with your project.
+
+## Module Map
+
+| Import | Purpose |
+|--------|---------|
+| `@arizeai/phoenix-client` | `createClient`, generated OpenAPI types, config helpers |
+| `@arizeai/phoenix-client/prompts` | Prompt CRUD plus `toSDK` conversion |
+| `@arizeai/phoenix-client/datasets` | Dataset creation and retrieval |
+| `@arizeai/phoenix-client/experiments` | Experiment execution and lifecycle |
+| `@arizeai/phoenix-client/spans` | Span search, notes, and span/document annotations |
+| `@arizeai/phoenix-client/sessions` | Session listing, retrieval, and session annotations |
+| `@arizeai/phoenix-client/traces` | Project trace retrieval |
+
+## Configuration
+
+`createClient()` resolves Phoenix client options in this order: library defaults, environment variables, then explicit options. In most applications, the normal setup is to set `PHOENIX_HOST` and `PHOENIX_API_KEY` in the environment and call `createClient()` with no overrides.
+
+### Recommended Setup
+
+Use the environment-driven path unless you have a specific reason to override client options in code.
+
+```bash
+export PHOENIX_HOST=http://localhost:6006
+export PHOENIX_API_KEY=<your-api-key>
+```
+
+If you're using Phoenix Cloud, `PHOENIX_HOST` may look like `https://app.phoenix.arize.com/s/my-space`.
+
+```ts
+import { createClient } from "@arizeai/phoenix-client";
+
+const client = createClient();
+
+const datasets = await client.GET("/v1/datasets");
+```
+
+`PHOENIX_API_KEY` is converted into `Authorization: Bearer <key>` automatically. You do not need to build that header yourself unless you are explicitly overriding `headers`.
+
+### Explicit Overrides
+
+```ts
+import { createClient } from "@arizeai/phoenix-client";
+
+const client = createClient({
+  options: {
+    baseUrl: "https://phoenix.example.com",
+    headers: {
+      Authorization: `Bearer ${process.env.PHOENIX_API_KEY}`,
+    },
+  },
+});
+```
+
+Use explicit options when you want configuration to live in code or when you need to override the environment for a specific client instance.
+
+### createClient Parameters
+
+| Field | Type | Description |
+|--------|------|-------------|
+| `options` | `Partial<ClientOptions>` | Explicit options passed to the underlying `openapi-fetch` client. |
+| `getEnvironmentOptions` | `() => Partial<ClientOptions>` | Optional resolver for environment-derived options. The default implementation reads `process.env` when available. |
+
+### Resolved Phoenix Options
+
+These are the Phoenix-specific options this package resolves before creating the underlying OpenAPI client:
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `baseUrl` | `string` | Base Phoenix URL. Defaults to `http://localhost:6006`, or `PHOENIX_HOST` when that environment variable is set. |
+| `headers` | `ClientOptions["headers"]` | Headers sent on every request. `PHOENIX_API_KEY` populates `Authorization` automatically. Explicit `headers` replace environment-derived headers. |
+
+### Header Override Rule
+
+If you pass `options.headers`, they replace the environment-derived header object rather than deep-merging with it. That means if you override `headers` and still want API key authentication, include `Authorization` yourself:
+
+```ts
+const client = createClient({
+  options: {
+    headers: {
+      Authorization: `Bearer ${process.env.PHOENIX_API_KEY}`,
+    },
+  },
+});
+```
+
+### Environment Variables
+
+| Variable | Maps to | Description |
+|--------|---------|-------------|
+| `PHOENIX_HOST` | `options.baseUrl` | Base Phoenix URL, for example `http://localhost:6006`. |
+| `PHOENIX_API_KEY` | `options.headers.Authorization` | Bearer token for authenticated environments. |
+| `PHOENIX_CLIENT_HEADERS` | `options.headers` | Optional JSON-encoded object of additional headers to send on every request. Most setups do not need this. |
+
+## API Client
+
+`createClient()` returns an `openapi-fetch` client that is typed against Phoenix's generated OpenAPI schema. Use this layer when you need an endpoint that does not yet have a high-level helper.
+
+```ts
+import { createClient } from "@arizeai/phoenix-client";
+
+const client = createClient();
+
+const datasets = await client.GET("/v1/datasets");
+
+const prompt = await client.GET("/v1/prompts/{prompt_identifier}/latest", {
+  params: {
+    path: {
+      prompt_identifier: "support-response",
+    },
+  },
+});
+```
+
+The root export exposes generated API types: `pathsV1`, `componentsV1`, `operationsV1`, `Types`, and `PhoenixClient`.
+
+Prefer this layer when:
+
+- you need a newly added endpoint before a helper exists
+- you want direct control over route, body, and query params
+- you are building thin wrappers around Phoenix routes in your own codebase
+
+## Where To Start
+
+- [Prompts](./prompts), [Datasets](./datasets), [Experiments](./experiments) — higher-level workflows
+- [Annotations](./annotations) — annotation concepts, then [Span](./span-annotations), [Document](./document-annotations), and [Session](./session-annotations) annotations for detailed usage
+- [Spans](./spans), [Sessions](./sessions), [Traces](./traces) — retrieval and maintenance
+
+<section className="hidden" data-agent-context="source-map" aria-label="Source map">
+  <h2>Source Map</h2>
+  <ul>
+    <li><code>src/client.ts</code></li>
+    <li><code>src/config.ts</code></li>
+    <li><code>src/__generated__/api/v1.ts</code></li>
+    <li><code>src/types/core.ts</code></li>
+    <li><code>src/prompts/</code></li>
+    <li><code>src/datasets/</code></li>
+    <li><code>src/experiments/</code></li>
+    <li><code>src/spans/</code></li>
+    <li><code>src/sessions/</code></li>
+    <li><code>src/traces/</code></li>
+    <li><code>src/types/</code></li>
+  </ul>
+</section>

package/docs/prompts.mdx

@@ -0,0 +1,73 @@
+---
+title: "Prompts"
+description: "Manage prompts with @arizeai/phoenix-client"
+---
+
+The prompts module lets you create prompt versions in Phoenix, fetch them back by selector, list prompts, and adapt prompt versions to supported provider SDKs.
+
+<section className="hidden" data-agent-context="relevant-source-files" aria-label="Relevant source files">
+  <h2>Relevant Source Files</h2>
+  <ul>
+    <li><code>src/prompts/getPrompt.ts</code> for the exact selector shape</li>
+  </ul>
+</section>
+
+## Create A Prompt
+
+```ts
+import {
+  createPrompt,
+  promptVersion,
+} from "@arizeai/phoenix-client/prompts";
+
+await createPrompt({
+  name: "support-response",
+  description: "Customer support reply prompt",
+  version: promptVersion({
+    modelProvider: "OPENAI",
+    modelName: "gpt-4o-mini",
+    template: [{ role: "user", content: "Reply to {{question}}" }],
+  }),
+});
+```
+
+## Fetch By Selector
+
+```ts
+import { getPrompt } from "@arizeai/phoenix-client/prompts";
+
+const prompt = await getPrompt({
+  prompt: { name: "support-response", tag: "production" },
+});
+```
+
+`prompt` can be selected by `{ name }`, `{ name, tag }`, or `{ versionId }`.
+
+## Convert To Another SDK
+
+```ts
+import { toSDK } from "@arizeai/phoenix-client/prompts";
+
+const promptAsAI = toSDK({
+  sdk: "ai",
+  prompt,
+  variables: { question: "Where is my order?" },
+});
+```
+
+Supported `sdk` targets:
+
+- `ai`
+- `openai`
+- `anthropic`
+
+<section className="hidden" data-agent-context="source-map" aria-label="Source map">
+  <h2>Source Map</h2>
+  <ul>
+    <li><code>src/prompts/createPrompt.ts</code></li>
+    <li><code>src/prompts/getPrompt.ts</code></li>
+    <li><code>src/prompts/listPrompts.ts</code></li>
+    <li><code>src/prompts/sdks/toSDK.ts</code></li>
+    <li><code>src/types/prompts.ts</code></li>
+  </ul>
+</section>