@arizeai/phoenix-client 6.14.2 → 7.0.1

package/docs/datasets.mdx

@@ -3,14 +3,14 @@ title: "Datasets"
 description: "Create and inspect datasets with @arizeai/phoenix-client"
 ---
 
-Datasets are the foundation for experiment runs. The dataset helpers cover creation, idempotent creation, record inspection, and example appends.
+Datasets are the foundation for experiment runs. The dataset helpers cover creation (which upserts by name), record inspection, and example appends.
 
 <section className="hidden" data-agent-context="relevant-source-files" aria-label="Relevant source files">
   <h2>Relevant Source Files</h2>
   <ul>
     <li>
-      <code>src/datasets/createOrGetDataset.ts</code> for the exact return
-      shape of the idempotent helper
+      <code>src/datasets/createDataset.ts</code> for the exact return shape and
+      upsert-by-name behavior
     </li>
   </ul>
 </section>
@@ -33,18 +33,25 @@ const { datasetId } = await createDataset({
 });
 ```
 
-## Reuse Or Append
+## Upsert Or Append
+
+`createDataset()` upserts by name: re-running it with the same name updates the existing dataset to match the examples you pass, and an unchanged upload is a no-op. To keep existing examples and add more, use `appendDatasetExamples()` instead of re-running `createDataset()` with the extra examples.
 
 ```ts
 import {
   appendDatasetExamples,
-  createOrGetDataset,
+  createDataset,
 } from "@arizeai/phoenix-client/datasets";
 
-const dataset = await createOrGetDataset({
+const dataset = await createDataset({
   name: "support-eval",
   description: "Support questions with expected answers",
-  examples: [],
+  examples: [
+    {
+      input: { question: "Where is my order?" },
+      output: { answer: "Use the tracking page in your account." },
+    },
+  ],
 });
 
 await appendDatasetExamples({
@@ -58,7 +65,7 @@ await appendDatasetExamples({
 });
 ```
 
-`createOrGetDataset()` returns `{ datasetId }`, so you can pass that object directly as the dataset selector for append or experiment calls.
+`createDataset()` returns `{ datasetId }`, so you can pass that object directly as the dataset selector for append or experiment calls.
 
 ## Read Back Dataset State
 
@@ -68,7 +75,6 @@ Use `getDataset`, `getDatasetExamples`, and `getDatasetInfo` to inspect datasets
   <h2>Source Map</h2>
   <ul>
     <li><code>src/datasets/createDataset.ts</code></li>
-    <li><code>src/datasets/createOrGetDataset.ts</code></li>
     <li><code>src/datasets/appendDatasetExamples.ts</code></li>
     <li><code>src/datasets/getDataset.ts</code></li>
     <li><code>src/datasets/getDatasetExamples.ts</code></li>

package/docs/experiments.mdx

@@ -111,11 +111,18 @@ This pattern is useful when:
 
 ## Model-Backed Example
 
-If you want a model-backed experiment with automatic tracing and an LLM-as-a-judge evaluator, this is the core pattern:
+If you want a model-backed experiment with automatic tracing and an LLM-as-a-judge evaluator, this is the core pattern.
+
+Install the AI SDK packages alongside the Phoenix packages — `@ai-sdk/otel` provides the telemetry integration that sends AI SDK traces to Phoenix:
+
+```bash
+npm install @arizeai/phoenix-client @arizeai/phoenix-evals ai @ai-sdk/openai @ai-sdk/otel
+```
 
 ```ts
 import { openai } from "@ai-sdk/openai";
-import { createOrGetDataset } from "@arizeai/phoenix-client/datasets";
+import { OpenTelemetry } from "@ai-sdk/otel";
+import { createDataset } from "@arizeai/phoenix-client/datasets";
 import { runExperiment } from "@arizeai/phoenix-client/experiments";
 import type { ExperimentTask } from "@arizeai/phoenix-client/types/experiments";
 import { createClassificationEvaluator } from "@arizeai/phoenix-evals";
@@ -135,7 +142,7 @@ const main = async () => {
     },
   });
 
-  const dataset = await createOrGetDataset({
+  const dataset = await createDataset({
     name: "correctness-eval",
     description: "Evaluate the correctness of the model",
     examples: [
@@ -157,23 +164,15 @@ const main = async () => {
       throw new Error("Invalid input: context must be a string");
     }
 
+    // The per-call `@ai-sdk/otel` integration traces this call through the
+    // experiment's tracer provider that runExperiment mounts while tasks run.
     return generateText({
       model,
-      experimental_telemetry: {
-        isEnabled: true,
-      },
-      prompt: [
-        {
-          role: "system",
-          content: `You answer questions based on this context: ${example.input.context}`,
-        },
-        {
-          role: "user",
-          content: example.input.question,
-        },
-      ],
+      instructions: `You answer questions based on this context: ${example.input.context}`,
+      prompt: example.input.question,
+      telemetry: { integrations: [new OpenTelemetry()] },
     }).then((response) => {
-      if (response.text) {
+      if (typeof response.text === "string") {
         return response.text;
       }
       throw new Error("Invalid response: text is required");
@@ -200,9 +199,9 @@ main().catch(console.error);
 
 ## What This Example Shows
 
-- `createOrGetDataset()` creates or reuses the dataset the experiment will run against
+- `createDataset()` creates or reuses the dataset the experiment will run against (it upserts by name)
 - `task` receives the full dataset example object
-- `generateText()` emits traces that Phoenix can attach to the experiment when telemetry is enabled
+- `generateText()` emits traces that Phoenix attaches to the experiment because the call passes the `@ai-sdk/otel` integration via `telemetry.integrations` — without it, no AI SDK spans reach Phoenix
 - `createClassificationEvaluator()` from `@arizeai/phoenix-evals` can be passed directly to `runExperiment()`
 - `runExperiment()` records both task runs and evaluation runs in Phoenix
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arizeai/phoenix-client",
-  "version": "6.14.2",
+  "version": "7.0.1",
   "description": "A client for the Phoenix API",
   "keywords": [
     "arize",
@@ -96,34 +96,34 @@
   },
   "dependencies": {
     "@arizeai/openinference-semantic-conventions": "^2.5.0",
-    "@arizeai/openinference-vercel": "^2.8.1",
     "async": "^3.2.6",
     "openapi-fetch": "^0.17.0",
     "tiny-invariant": "^1.3.3",
     "zod": "^4.4.3",
-    "@arizeai/phoenix-otel": "2.1.0",
-    "@arizeai/phoenix-config": "0.4.0"
+    "@arizeai/phoenix-config": "0.4.0",
+    "@arizeai/phoenix-otel": "2.1.0"
   },
   "devDependencies": {
-    "@ai-sdk/openai": "^3.0.86",
+    "@ai-sdk/openai": "^4.0.0",
+    "@ai-sdk/otel": "^1.0.0",
     "@anthropic-ai/sdk": "^0.111.0",
     "@opentelemetry/api": "^1.9.1",
     "@opentelemetry/sdk-trace-node": "^2.9.0",
     "@types/async": "^3.2.25",
     "@types/node": "^26.1.1",
-    "ai": "^6.0.230",
+    "ai": "^7.0.0",
     "dotenv": "^17.4.2",
     "jest": "^30.4.2",
     "openai": "^6.48.0",
     "openapi-typescript": "^7.13.0",
     "tsx": "^4.23.1",
     "vitest": "^4.1.10",
-    "@arizeai/phoenix-testing": "0.0.0",
-    "@arizeai/phoenix-evals": "1.2.0"
+    "@arizeai/phoenix-evals": "2.1.0",
+    "@arizeai/phoenix-testing": "0.0.0"
   },
   "peerDependencies": {
     "@anthropic-ai/sdk": "^0.35.0",
-    "ai": "^6.0.90",
+    "ai": "^7.0.0",
     "jest": ">=27",
     "openai": "^6.10.0",
     "vitest": ">=1"
@@ -156,6 +156,6 @@
     "prebuild": "pnpm run clean && pnpm run generate",
     "test": "vitest run",
     "test:watch": "vitest watch",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc --noEmit && tsc -p examples/tsconfig.examples.json"
   }
 }

package/src/__generated__/api/v1.ts

@@ -2417,6 +2417,14 @@ export interface components {
              * Format: date-time
              */
             updated_at: string;
+            source?: components["schemas"]["DatasetExampleSource"] | null;
+        };
+        /** DatasetExampleSource */
+        DatasetExampleSource: {
+            /** Span Id */
+            span_id: string;
+            /** Span Node Id */
+            span_node_id: string;
         };
         /** DatasetLabel */
         DatasetLabel: {

package/src/experiments/resumeEvaluation.ts

@@ -522,7 +522,7 @@ export async function resumeEvaluation({
     }
 
     // Start concurrent execution
-    // Wrap in try-finally to ensure channel is always closed, even if Promise.all throws
+    // Wrap in try-finally to ensure channel is always closed, even if a task throws
     let executionError: Error | null = null;
     try {
       const producerTask = fetchIncompleteEvaluations();
@@ -530,31 +530,61 @@ export async function resumeEvaluation({
         processEvaluationsFromChannel()
       );
 
-      // Wait for producer and all workers to finish
-      await Promise.all([producerTask, ...workerTasks]);
-    } catch (error) {
-      // Classify and handle errors based on their nature
-      const err = error instanceof Error ? error : new Error(String(error));
-
-      // Always surface producer/infrastructure errors
-      if (error instanceof EvaluationFetchError) {
-        // Producer failed - this is ALWAYS critical regardless of stopOnFirstError
-        logger.error(`Critical: Failed to fetch evaluations from server`);
-        executionError = err;
-      } else if (error instanceof ChannelError && signal.aborted) {
-        // Channel closed due to intentional abort - wrap in semantic error
-        executionError = new EvaluationAbortedError(
-          "Evaluation stopped due to error in concurrent evaluator",
-          err
+      // Wait for the producer AND every worker to settle before continuing.
+      // Using allSettled (rather than Promise.all) is important: on the first
+      // worker error, Promise.all rejects immediately while the remaining
+      // workers keep running detached, logging and hitting the API after this
+      // function has already returned/thrown. Draining all tasks guarantees no
+      // background work outlives the call (and avoids teardown races in tests
+      // where late console output is flushed after the test completes).
+      const settled = await Promise.allSettled([producerTask, ...workerTasks]);
+      const rejections = settled
+        .filter(
+          (result): result is PromiseRejectedResult =>
+            result.status === "rejected"
+        )
+        .map((result) => result.reason);
+
+      if (rejections.length > 0) {
+        // Classify and handle errors based on their nature. When multiple tasks
+        // reject, prefer the most meaningful error over incidental fallout
+        // (e.g. a ChannelError raised in a blocked worker when the channel
+        // closes on abort).
+        const fetchError = rejections.find(
+          (reason) => reason instanceof EvaluationFetchError
         );
-      } else if (stopOnFirstError) {
-        // Worker error in stopOnFirstError mode - already logged by worker
-        executionError = err;
-      } else {
-        // Unexpected error (not from worker, not from producer fetch)
-        // This could be a bug in our code or infrastructure failure
-        logger.error(`Unexpected error during evaluation: ${err.message}`);
-        executionError = err;
+        const workerError = rejections.find(
+          (reason) =>
+            reason instanceof Error &&
+            !(reason instanceof EvaluationFetchError) &&
+            !(reason instanceof ChannelError)
+        );
+        const channelError = rejections.find(
+          (reason) => reason instanceof ChannelError
+        );
+
+        if (fetchError) {
+          // Producer failed - this is ALWAYS critical regardless of stopOnFirstError
+          logger.error(`Critical: Failed to fetch evaluations from server`);
+          executionError = fetchError;
+        } else if (workerError) {
+          // Worker error in stopOnFirstError mode - already logged by worker
+          executionError = workerError;
+        } else if (channelError && signal.aborted) {
+          // Channel closed due to intentional abort - wrap in semantic error
+          executionError = new EvaluationAbortedError(
+            "Evaluation stopped due to error in concurrent evaluator",
+            channelError
+          );
+        } else {
+          // Unexpected error (not from worker, not from producer fetch)
+          // This could be a bug in our code or infrastructure failure
+          const reason = rejections[0];
+          const err =
+            reason instanceof Error ? reason : new Error(String(reason));
+          logger.error(`Unexpected error during evaluation: ${err.message}`);
+          executionError = err;
+        }
       }
     } finally {
       // Ensure channel is closed even if there are unexpected errors

package/src/experiments/resumeExperiment.ts

@@ -489,7 +489,7 @@ export async function resumeExperiment({
     }
 
     // Start concurrent execution
-    // Wrap in try-finally to ensure channel is always closed, even if Promise.all throws
+    // Wrap in try-finally to ensure channel is always closed, even if a task throws
     let executionError: Error | null = null;
     try {
       const producerTask = fetchIncompleteRuns();
@@ -497,31 +497,63 @@ export async function resumeExperiment({
         processTasksFromChannel()
       );
 
-      // Wait for producer and all workers to finish
-      await Promise.all([producerTask, ...workerTasks]);
-    } catch (error) {
-      // Classify and handle errors based on their nature
-      const err = error instanceof Error ? error : new Error(String(error));
-
-      // Always surface producer/infrastructure errors
-      if (error instanceof TaskFetchError) {
-        // Producer failed - this is ALWAYS critical regardless of stopOnFirstError
-        logger.error(`Critical: Failed to fetch incomplete runs from server`);
-        executionError = err;
-      } else if (error instanceof ChannelError && signal.aborted) {
-        // Channel closed due to intentional abort - wrap in semantic error
-        executionError = new TaskAbortedError(
-          "Task execution stopped due to error in concurrent worker",
-          err
+      // Wait for the producer AND every worker to settle before continuing.
+      // Using allSettled (rather than Promise.all) is important: on the first
+      // worker error, Promise.all rejects immediately while the remaining
+      // workers keep running detached, logging and hitting the API after this
+      // function has already returned/thrown. Draining all tasks guarantees no
+      // background work outlives the call (and avoids teardown races in tests
+      // where late console output is flushed after the test completes).
+      const settled = await Promise.allSettled([producerTask, ...workerTasks]);
+      const rejections = settled
+        .filter(
+          (result): result is PromiseRejectedResult =>
+            result.status === "rejected"
+        )
+        .map((result) => result.reason);
+
+      if (rejections.length > 0) {
+        // Classify and handle errors based on their nature. When multiple tasks
+        // reject, prefer the most meaningful error over incidental fallout
+        // (e.g. a ChannelError raised in a blocked worker when the channel
+        // closes on abort).
+        const fetchError = rejections.find(
+          (reason) => reason instanceof TaskFetchError
         );
-      } else if (stopOnFirstError) {
-        // Worker error in stopOnFirstError mode - already logged by worker
-        executionError = err;
-      } else {
-        // Unexpected error (not from worker, not from producer fetch)
-        // This could be a bug in our code or infrastructure failure
-        logger.error(`Unexpected error during task execution: ${err.message}`);
-        executionError = err;
+        const taskError = rejections.find(
+          (reason) =>
+            reason instanceof Error &&
+            !(reason instanceof TaskFetchError) &&
+            !(reason instanceof ChannelError)
+        );
+        const channelError = rejections.find(
+          (reason) => reason instanceof ChannelError
+        );
+
+        if (fetchError) {
+          // Producer failed - this is ALWAYS critical regardless of stopOnFirstError
+          logger.error(`Critical: Failed to fetch incomplete runs from server`);
+          executionError = fetchError;
+        } else if (taskError) {
+          // Worker error in stopOnFirstError mode - already logged by worker
+          executionError = taskError;
+        } else if (channelError && signal.aborted) {
+          // Channel closed due to intentional abort - wrap in semantic error
+          executionError = new TaskAbortedError(
+            "Task execution stopped due to error in concurrent worker",
+            channelError
+          );
+        } else {
+          // Unexpected error (not from worker, not from producer fetch)
+          // This could be a bug in our code or infrastructure failure
+          const reason = rejections[0];
+          const err =
+            reason instanceof Error ? reason : new Error(String(reason));
+          logger.error(
+            `Unexpected error during task execution: ${err.message}`
+          );
+          executionError = err;
+        }
       }
     } finally {
       // Ensure channel is closed even if there are unexpected errors

package/src/prompts/sdks/toAI.ts

@@ -1,4 +1,6 @@
-import type { ModelMessage, ToolChoice, ToolSet } from "ai";
+import type { ModelMessage, ToolChoice, ToolSet } from "ai" with {
+  "resolution-mode": "import",
+};
 import invariant from "tiny-invariant";
 
 import {