@arizeai/phoenix-client 6.5.5 → 6.6.1

package/docs/experiments.mdx

@@ -12,6 +12,8 @@ The experiments module runs tasks over dataset examples, records experiment runs
     <li><code>src/experiments/helpers/getExperimentEvaluators.ts</code> for evaluator normalization</li>
     <li><code>src/experiments/helpers/fromPhoenixLLMEvaluator.ts</code> for the phoenix-evals bridge</li>
     <li><code>src/experiments/getExperimentRuns.ts</code> for reading runs back after execution</li>
+    <li><code>src/types/experiments.ts</code> for <code>EvaluatorParams</code> including <code>traceId</code></li>
+    <li><code>src/spans/getSpans.ts</code> for fetching spans by trace ID and span kind</li>
   </ul>
 </section>
 
@@ -226,10 +228,119 @@ When an evaluator runs, it receives a normalized object with these fields:
 | `output` | The task output for that run |
 | `expected` | The dataset example's `output` object |
 | `metadata` | The dataset example's `metadata` object |
+| `traceId` | The OpenTelemetry trace ID of the task run (optional, `string \| null`) |
 
 This is why the `createClassificationEvaluator()` prompt can reference `{{input.question}}` and `{{output}}`.
 
-For code-based evaluators created with `asExperimentEvaluator()`, those same fields are available inside `evaluate({ input, output, expected, metadata })`.
+For code-based evaluators created with `asExperimentEvaluator()`, those same fields are available inside `evaluate({ input, output, expected, metadata, traceId })`.
+
+## Trace-Based Evaluation
+
+Each task run captures an OpenTelemetry trace ID. Evaluators can use `traceId` to fetch the task's spans from Phoenix and evaluate the execution trajectory — for example, verifying that specific tool calls were made or inspecting intermediate steps.
+
+This pattern works best with `evaluateExperiment()` as a separate step after `runExperiment()`, so that all task spans are ingested into Phoenix before the evaluator queries them.
+
+If you want to trace task code with helpers like `traceTool`, install `@arizeai/phoenix-otel` alongside `@arizeai/phoenix-client`:
+
+```bash
+npm install @arizeai/phoenix-client @arizeai/phoenix-otel
+```
+
+```ts
+import { createClient } from "@arizeai/phoenix-client";
+import { createDataset } from "@arizeai/phoenix-client/datasets";
+import {
+  asExperimentEvaluator,
+  evaluateExperiment,
+  runExperiment,
+} from "@arizeai/phoenix-client/experiments";
+import { getSpans } from "@arizeai/phoenix-client/spans";
+import { traceTool } from "@arizeai/phoenix-otel";
+
+const client = createClient();
+
+const { datasetId } = await createDataset({
+  client,
+  name: "tool-call-dataset",
+  description: "Questions that require tool use",
+  examples: [
+    {
+      input: { question: "What is the weather in San Francisco?" },
+      output: { expectedTool: "getWeather" },
+      metadata: {},
+    },
+  ],
+});
+
+// Step 1: Run the experiment with traced tool calls
+const experiment = await runExperiment({
+  client,
+  dataset: { datasetId },
+  setGlobalTracerProvider: true,
+  task: async (example) => {
+    // traceTool wraps a function with a TOOL span
+    const getWeather = traceTool(
+      ({ location }: { location: string }) => ({
+        location,
+        temperature: 72,
+        condition: "sunny",
+      }),
+      { name: "getWeather" }
+    );
+
+    const city = (example.input.question as string).match(/in (.+)\?/)?.[1];
+    const result = getWeather({ location: city ?? "Unknown" });
+    return `The weather in ${result.location} is ${result.temperature}F.`;
+  },
+});
+
+const projectName = experiment.projectName!;
+
+// Step 2: Evaluate using traceId to inspect the task's spans
+const evaluated = await evaluateExperiment({
+  client,
+  experiment,
+  evaluators: [
+    asExperimentEvaluator({
+      name: "has-expected-tool-call",
+      kind: "CODE",
+      evaluate: async ({ traceId, expected }) => {
+        if (!traceId) {
+          return { label: "no trace", score: 0 };
+        }
+
+        // Fetch TOOL spans from this task's trace
+        const { spans: toolSpans } = await getSpans({
+          client,
+          project: { projectName },
+          traceIds: [traceId],
+          spanKind: "TOOL",
+        });
+
+        const expectedTool = (expected as { expectedTool?: string })
+          ?.expectedTool;
+        const toolNames = toolSpans.map((s) => s.name);
+        const found = toolNames.some((name) => name.includes(expectedTool!));
+
+        return {
+          label: found ? "tool called" : "no tool call",
+          score: found ? 1 : 0,
+          explanation: found
+            ? `Found: ${toolNames.join(", ")}`
+            : `Expected "${expectedTool}" but found none`,
+        };
+      },
+    }),
+  ],
+});
+```
+
+Key points:
+
+- Use `setGlobalTracerProvider: true` on `runExperiment()` so that child spans from `traceTool` or other OTel instrumentation land in the same trace as the task
+- Use `evaluateExperiment()` as a separate step so spans are ingested before querying
+- Use `getSpans()` with `traceIds` and `spanKind` filters to fetch specific spans from the task trace
+- `traceId` is `null` in dry-run mode since no real traces are recorded
 
 ## What `runExperiment()` Returns
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arizeai/phoenix-client",
-  "version": "6.5.5",
+  "version": "6.6.1",
   "description": "A client for the Phoenix API",
   "keywords": [
     "arize",
@@ -73,14 +73,14 @@
     }
   },
   "dependencies": {
-    "@arizeai/openinference-semantic-conventions": "^1.1.0",
+    "@arizeai/openinference-semantic-conventions": "^2.1.7",
     "@arizeai/openinference-vercel": "^2.7.0",
     "async": "^3.2.6",
     "openapi-fetch": "^0.12.5",
     "tiny-invariant": "^1.3.3",
     "zod": "^4.0.14",
-    "@arizeai/phoenix-otel": "0.4.3",
-    "@arizeai/phoenix-config": "0.1.3"
+    "@arizeai/phoenix-config": "0.1.3",
+    "@arizeai/phoenix-otel": "1.0.0"
   },
   "devDependencies": {
     "@ai-sdk/openai": "^3.0.29",

package/src/__generated__/api/v1.ts

@@ -595,24 +595,6 @@ export interface paths {
         patch?: never;
         trace?: never;
     };
-    "/v1/evaluations": {
-        parameters: {
-            query?: never;
-            header?: never;
-            path?: never;
-            cookie?: never;
-        };
-        /** Get span, trace, or document evaluations from a project */
-        get: operations["getEvaluations"];
-        put?: never;
-        /** Add span, trace, or document evaluations */
-        post: operations["addEvaluations"];
-        delete?: never;
-        options?: never;
-        head?: never;
-        patch?: never;
-        trace?: never;
-    };
     "/v1/prompts": {
         parameters: {
             query?: never;
@@ -2954,7 +2936,27 @@ export interface components {
              */
             end_time: string;
         };
-        /** Span */
+        /**
+         * Span
+         * @example {
+         *       "attributes": {
+         *         "llm.model_name": "gpt-4",
+         *         "llm.token_count.completion": 50,
+         *         "llm.token_count.prompt": 100
+         *       },
+         *       "context": {
+         *         "span_id": "1a2b3c4d5e6f7a8b",
+         *         "trace_id": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4"
+         *       },
+         *       "end_time": "2024-01-01T12:00:01Z",
+         *       "events": [],
+         *       "name": "llm_call",
+         *       "span_kind": "LLM",
+         *       "start_time": "2024-01-01T12:00:00Z",
+         *       "status_code": "OK",
+         *       "status_message": ""
+         *     }
+         */
         Span: {
             /**
              * Id
@@ -3109,7 +3111,13 @@ export interface components {
             /** Next Cursor */
             next_cursor: string | null;
         };
-        /** SpanContext */
+        /**
+         * SpanContext
+         * @example {
+         *       "span_id": "1a2b3c4d5e6f7a8b",
+         *       "trace_id": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4"
+         *     }
+         */
         SpanContext: {
             /**
              * Trace Id
@@ -3161,7 +3169,16 @@ export interface components {
              */
             document_position: number;
         };
-        /** SpanEvent */
+        /**
+         * SpanEvent
+         * @example {
+         *       "attributes": {
+         *         "exception.message": "Connection refused"
+         *       },
+         *       "name": "exception",
+         *       "timestamp": "2024-01-01T12:00:00Z"
+         *     }
+         */
         SpanEvent: {
             /**
              * Name
@@ -5465,109 +5482,6 @@ export interface operations {
             };
         };
     };
-    getEvaluations: {
-        parameters: {
-            query?: {
-                /** @description The name of the project to get evaluations from (if omitted, evaluations will be drawn from the `default` project) */
-                project_name?: string | null;
-            };
-            header?: never;
-            path?: never;
-            cookie?: never;
-        };
-        requestBody?: never;
-        responses: {
-            /** @description Successful Response */
-            200: {
-                headers: {
-                    [name: string]: unknown;
-                };
-                content: {
-                    "application/json": unknown;
-                };
-            };
-            /** @description Forbidden */
-            403: {
-                headers: {
-                    [name: string]: unknown;
-                };
-                content: {
-                    "text/plain": string;
-                };
-            };
-            /** @description Not Found */
-            404: {
-                headers: {
-                    [name: string]: unknown;
-                };
-                content: {
-                    "text/plain": string;
-                };
-            };
-            /** @description Validation Error */
-            422: {
-                headers: {
-                    [name: string]: unknown;
-                };
-                content: {
-                    "application/json": components["schemas"]["HTTPValidationError"];
-                };
-            };
-        };
-    };
-    addEvaluations: {
-        parameters: {
-            query?: never;
-            header?: {
-                "content-type"?: string | null;
-                "content-encoding"?: string | null;
-            };
-            path?: never;
-            cookie?: never;
-        };
-        requestBody: {
-            content: {
-                "application/x-protobuf": string;
-                "application/x-pandas-arrow": string;
-            };
-        };
-        responses: {
-            /** @description Successful Response */
-            204: {
-                headers: {
-                    [name: string]: unknown;
-                };
-                content?: never;
-            };
-            /** @description Forbidden */
-            403: {
-                headers: {
-                    [name: string]: unknown;
-                };
-                content: {
-                    "text/plain": string;
-                };
-            };
-            /** @description Unsupported content type, only gzipped protobuf and pandas-arrow are supported */
-            415: {
-                headers: {
-                    [name: string]: unknown;
-                };
-                content: {
-                    "text/plain": string;
-                };
-            };
-            /** @description Unprocessable Entity */
-            422: {
-                headers: {
-                    [name: string]: unknown;
-                };
-                content: {
-                    "text/plain": string;
-                };
-            };
-        };
-    };
     getPrompts: {
         parameters: {
             query?: {

package/src/experiments/resumeEvaluation.ts

@@ -1,18 +1,14 @@
-import {
-  MimeType,
-  OpenInferenceSpanKind,
-  SemanticConventions,
-} from "@arizeai/openinference-semantic-conventions";
-import type {
-  GlobalTracerProviderRegistration,
-  NodeTracerProvider,
-  Tracer,
-} from "@arizeai/phoenix-otel";
 import {
   attachGlobalTracerProvider,
   type DiagLogLevel,
+  type GlobalTracerProviderRegistration,
+  MimeType,
+  type NodeTracerProvider,
   objectAsAttributes,
+  OpenInferenceSpanKind,
   register,
+  SemanticConventions,
+  type Tracer,
   SpanStatusCode,
 } from "@arizeai/phoenix-otel";
 import invariant from "tiny-invariant";
@@ -692,6 +688,7 @@ async function runSingleEvaluation({
           output: taskOutput,
           expected: expectedOutput,
           metadata: datasetExample.metadata,
+          traceId: experimentRun.traceId,
         })
       );
       results = Array.isArray(result) ? result : [result];
@@ -746,6 +743,7 @@ async function runSingleEvaluation({
             output: taskOutput,
             expected: expectedOutput,
             metadata: datasetExample.metadata,
+            traceId: experimentRun.traceId,
           })
         );
 

package/src/experiments/resumeExperiment.ts

@@ -1,18 +1,14 @@
-import {
-  MimeType,
-  OpenInferenceSpanKind,
-  SemanticConventions,
-} from "@arizeai/openinference-semantic-conventions";
-import type {
-  GlobalTracerProviderRegistration,
-  NodeTracerProvider,
-  Tracer,
-} from "@arizeai/phoenix-otel";
 import {
   attachGlobalTracerProvider,
   type DiagLogLevel,
+  type GlobalTracerProviderRegistration,
+  MimeType,
+  type NodeTracerProvider,
   objectAsAttributes,
+  OpenInferenceSpanKind,
   register,
+  SemanticConventions,
+  type Tracer,
   SpanStatusCode,
 } from "@arizeai/phoenix-otel";
 import invariant from "tiny-invariant";

package/src/experiments/runExperiment.ts

@@ -1,18 +1,14 @@
-import {
-  MimeType,
-  OpenInferenceSpanKind,
-  SemanticConventions,
-} from "@arizeai/openinference-semantic-conventions";
-import type {
-  GlobalTracerProviderRegistration,
-  NodeTracerProvider,
-  Tracer,
-} from "@arizeai/phoenix-otel";
 import {
   attachGlobalTracerProvider,
   createNoOpProvider,
   type DiagLogLevel,
+  type GlobalTracerProviderRegistration,
+  MimeType,
+  type NodeTracerProvider,
   objectAsAttributes,
+  OpenInferenceSpanKind,
+  SemanticConventions,
+  type Tracer,
   register,
   SpanStatusCode,
 } from "@arizeai/phoenix-otel";
@@ -853,6 +849,7 @@ async function runEvaluator({
         output: run.output ?? null,
         expected: example.output,
         metadata: example?.metadata,
+        traceId: run.traceId,
       });
       thisEval.result = result;
     } catch (error) {

package/src/types/experiments.ts

@@ -131,6 +131,12 @@ export type EvaluatorParams<TaskOutputType = TaskOutput> = {
    * Metadata associated with the Dataset Example
    */
   metadata?: Example["metadata"];
+  /**
+   * The trace ID of the task run, if available.
+   * Can be used to fetch and analyze the task's trace
+   * (e.g., for trajectory evaluation or action verification).
+   */
+  traceId?: string | null;
 };
 
 export type Evaluator = {

package/src/types/spans.ts

@@ -1,4 +1,4 @@
-import type { OpenInferenceSpanKind } from "@arizeai/openinference-semantic-conventions";
+import type { OpenInferenceSpanKind } from "@arizeai/phoenix-otel";
 
 /**
  * Status codes for spans.