@arizeai/phoenix-client 6.10.1 → 6.11.0

package/README.md

@@ -298,6 +298,68 @@ const experiment = await runExperiment({
 
 > **Hint:** Tasks and evaluators are instrumented using [OpenTelemetry](https://opentelemetry.io/). You can view detailed traces of experiment runs and evaluations directly in the Phoenix UI for debugging and performance analysis.
 
+## Eval Tests
+
+The package also exposes Vitest and Jest submodules for writing
+dataset-backed evaluations as normal test suites.
+
+```bash
+npm install -D @arizeai/phoenix-client vitest dotenv
+```
+
+```ts
+import * as px from "@arizeai/phoenix-client/vitest";
+import { expect } from "vitest";
+
+px.describe(
+  "text-to-sql",
+  () => {
+    px.test(
+      "select all",
+      {
+        input: { userQuery: "Get all users" },
+        expected: { sql: "SELECT * FROM users;" },
+      },
+      async ({ input, expected }) => {
+        const sql = await generateSql(input.userQuery);
+        px.logOutput({ sql });
+        expect(sql).toEqual(expected?.sql);
+      }
+    );
+  },
+  {
+    acceptanceCriteria: [
+      // every test must pass (100% of the auto-logged `pass` boolean is true)
+      {
+        annotationName: "pass",
+        metric: "passRate",
+        passFn: (a) => a.score === true,
+        minPassRate: 1,
+      },
+    ],
+  }
+);
+```
+
+The reference output can be supplied under any one of three interchangeable
+keys — `expected`, `reference`, or `output` — so you can match whichever name
+your evaluators expect. They all resolve to the same slot: the dataset
+example's `output`, exposed to evaluators and the test body as `expected`.
+Supplying more than one at a time is a type error.
+
+Use `@arizeai/phoenix-client/vitest/reporter` or
+`@arizeai/phoenix-client/jest/reporter` to print Phoenix dataset and
+experiment links, annotation aggregates, and acceptance criteria at the end
+of a run. Acceptance criteria gate the suite after it finishes — `average`
+checks an aggregate bar (so CI can allow a mean of 80% while still running
+every case), and `passRate` requires a minimum fraction of runs to satisfy a
+per-run `passFn` predicate.
+
+See the [`docs/`](./docs) folder — `ci-evals.mdx`, `ci-evals-vitest.mdx`,
+`ci-evals-jest.mdx`, and `ci-evals-annotations.mdx` — for setup, the full
+`describe` / `test` / `test.each` API, acceptance criteria, repetitions,
+dry-run mode, and annotation details.
+
 ## Traces
 
 The `@arizeai/phoenix-client` package provides a `traces` export for retrieving trace data from Phoenix projects.

package/dist/esm/__generated__/api/v1.d.ts

@@ -387,7 +387,11 @@ export interface paths {
         delete: operations["deleteExperiment"];
         options?: never;
         head?: never;
-        patch?: never;
+        /**
+         * Update an experiment by ID
+         * @description Partially update an experiment's name, description, and/or metadata. Only the fields included in the request body are changed; omitted fields are left as-is. Patching an ephemeral experiment refreshes its last-update timestamp, which extends the window before it is swept away.
+         */
+        patch: operations["updateExperiment"];
         trace?: never;
     };
     "/v1/experiments/{experiment_id}/incomplete-runs": {
@@ -1609,7 +1613,7 @@ export interface components {
          * ChatContext
          * @description Discriminated union of every UI-state context the agent understands.
          */
-        ChatContext: components["schemas"]["AppContext"] | components["schemas"]["ProjectContext"] | components["schemas"]["TraceContext"] | components["schemas"]["AgentSpanContext"] | components["schemas"]["PlaygroundContext"] | components["schemas"]["CodeEvaluatorContext"] | components["schemas"]["LlmEvaluatorContext"] | components["schemas"]["DatasetContext"] | components["schemas"]["GraphQLContext"] | components["schemas"]["WebAccessContext"];
+        ChatContext: components["schemas"]["AppContext"] | components["schemas"]["ProjectContext"] | components["schemas"]["TraceContext"] | components["schemas"]["SessionContext"] | components["schemas"]["PromptContext"] | components["schemas"]["PromptVersionContext"] | components["schemas"]["AgentSpanContext"] | components["schemas"]["PlaygroundContext"] | components["schemas"]["CodeEvaluatorContext"] | components["schemas"]["LlmEvaluatorContext"] | components["schemas"]["DatasetContext"] | components["schemas"]["GraphQLContext"] | components["schemas"]["WebAccessContext"] | components["schemas"]["SubagentsContext"];
         /**
          * ChatRegenerateMessage
          * @description Regenerate message extended with Phoenix-specific fields.
@@ -1636,6 +1640,12 @@ export interface components {
              * @default false
              */
             exportRemoteTraces?: boolean;
+            /**
+             * Attachuserid
+             * @description When true and the request is authenticated as a PhoenixUser, attaches the user's email as the OpenInference ``user.id`` span attribute on all traced work for this request.
+             * @default false
+             */
+            attachUserId?: boolean;
             /** Contexts */
             contexts?: components["schemas"]["ChatContext"][];
             /**
@@ -1683,6 +1693,12 @@ export interface components {
              * @default false
              */
             exportRemoteTraces?: boolean;
+            /**
+             * Attachuserid
+             * @description When true and the request is authenticated as a PhoenixUser, attaches the user's email as the OpenInference ``user.id`` span attribute on all traced work for this request.
+             * @default false
+             */
+            attachUserId?: boolean;
             /** Contexts */
             contexts?: components["schemas"]["ChatContext"][];
             /**
@@ -2354,6 +2370,16 @@ export interface components {
              * @description The ID of the dataset version associated with the experiment
              */
             dataset_version_id: string;
+            /**
+             * Name
+             * @description The name of the experiment
+             */
+            name: string;
+            /**
+             * Description
+             * @description The description of the experiment
+             */
+            description: string | null;
             /**
              * Repetitions
              * @description Number of times the experiment is repeated
@@ -3174,6 +3200,7 @@ export interface components {
              * @default 1
              */
             repetitions?: number;
+            nextExperimentScaffold?: components["schemas"]["PlaygroundExperimentScaffoldContext"] | null;
             /** Instances */
             instances?: components["schemas"]["PlaygroundInstanceContext"][];
             /** Evaluators */
@@ -3218,6 +3245,37 @@ export interface components {
             /** Isapplied */
             isApplied: boolean;
         };
+        /**
+         * PlaygroundExperimentScaffoldContext
+         * @description Experiment name/description/metadata the user has staged for the playground's
+         *     *next* dataset-backed run, before that run has started.
+         *
+         *     The playground UI lets the user pre-set how the next recorded run's experiment
+         *     will be named, described, and tagged (via the ``set_playground_experiment_recording``
+         *     tool or the recording form). That staged state is surfaced here so the agent can
+         *     see what is already set and avoid re-staging it.
+         *
+         *     Field semantics:
+         *     - ``name`` / ``description``: the staged values, surfaced to the model verbatim,
+         *       or ``None`` when the user has not staged them.
+         *     - ``has_metadata``: a presence flag, not the value. Only *whether* metadata has
+         *       been staged is model-relevant (so the agent knows not to re-attach it); the
+         *       metadata object itself is deliberately kept out of the prompt.
+         *
+         *     A field left unstaged (``None`` / ``False``) falls back to the server default when
+         *     the run starts. The scaffold is consumed once that next run begins.
+         */
+        PlaygroundExperimentScaffoldContext: {
+            /** Name */
+            name?: string | null;
+            /** Description */
+            description?: string | null;
+            /**
+             * Hasmetadata
+             * @default false
+             */
+            hasMetadata?: boolean;
+        };
         /**
          * PlaygroundInstanceContext
          * @description One mounted playground instance and its current model selection.
@@ -3459,6 +3517,19 @@ export interface components {
             /** Messages */
             messages: components["schemas"]["PromptMessage"][];
         };
+        /**
+         * PromptContext
+         * @description Prompt the user is currently viewing.
+         */
+        PromptContext: {
+            /**
+             * @description discriminator enum property added by openapi-typescript
+             * @enum {string}
+             */
+            type: "prompt";
+            /** Promptnodeid */
+            promptNodeId: string;
+        };
         /** PromptData */
         PromptData: {
             name: components["schemas"]["Identifier"];
@@ -3959,6 +4030,21 @@ export interface components {
             /** Id */
             id: string;
         };
+        /**
+         * PromptVersionContext
+         * @description Prompt version the user is currently viewing.
+         */
+        PromptVersionContext: {
+            /**
+             * @description discriminator enum property added by openapi-typescript
+             * @enum {string}
+             */
+            type: "prompt_version";
+            /** Promptnodeid */
+            promptNodeId: string;
+            /** Promptversionnodeid */
+            promptVersionNodeId: string;
+        };
         /** PromptVersionData */
         PromptVersionData: {
             /** Description */
@@ -4160,6 +4246,21 @@ export interface components {
             /** Next Cursor */
             next_cursor: string | null;
         };
+        /**
+         * SessionContext
+         * @description Session the user is currently viewing.
+         */
+        SessionContext: {
+            /**
+             * @description discriminator enum property added by openapi-typescript
+             * @enum {string}
+             */
+            type: "session";
+            /** Projectnodeid */
+            projectNodeId: string;
+            /** Sessionnodeid */
+            sessionNodeId: string;
+        };
         /** SessionData */
         SessionData: {
             /** Id */
@@ -4586,6 +4687,19 @@ export interface components {
              */
             type?: "step-start";
         };
+        /**
+         * SubagentsContext
+         * @description User's per-turn request to expose the subagent-spawning tool.
+         */
+        SubagentsContext: {
+            /**
+             * @description discriminator enum property added by openapi-typescript
+             * @enum {string}
+             */
+            type: "subagents";
+            /** Enabled */
+            enabled: boolean;
+        };
         /** TextContentPart */
         TextContentPart: {
             /**
@@ -5105,6 +5219,33 @@ export interface components {
             /** Data */
             data: components["schemas"]["CategoricalAnnotationConfig"] | components["schemas"]["ContinuousAnnotationConfig"] | components["schemas"]["FreeformAnnotationConfig"];
         };
+        /**
+         * UpdateExperimentRequestBody
+         * @description Fields to update on an experiment. Omit a field to leave it unchanged.
+         */
+        UpdateExperimentRequestBody: {
+            /**
+             * Name
+             * @description New name for the experiment (null is rejected; name is required)
+             */
+            name?: string | null;
+            /**
+             * Description
+             * @description New description for the experiment (null clears the description)
+             */
+            description?: string | null;
+            /**
+             * Metadata
+             * @description New metadata object for the experiment (replaces the existing metadata as a whole; null is rejected)
+             */
+            metadata?: {
+                [key: string]: unknown;
+            } | null;
+        };
+        /** UpdateExperimentResponseBody */
+        UpdateExperimentResponseBody: {
+            data: components["schemas"]["Experiment"];
+        };
         /** UpdateProjectRequestBody */
         UpdateProjectRequestBody: {
             /** Description */
@@ -5255,6 +5396,12 @@ export interface components {
              * @default false
              */
             exportRemoteTraces?: boolean;
+            /**
+             * Attachuserid
+             * @description When true and the request is authenticated as a PhoenixUser, attaches the user's email as the OpenInference ``user.id`` span attribute on all traced work for this request.
+             * @default false
+             */
+            attachUserId?: boolean;
             /** Messages */
             messages: components["schemas"]["UIMessage"][];
             /** Model */
@@ -6577,6 +6724,59 @@ export interface operations {
             };
         };
     };
+    updateExperiment: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path: {
+                experiment_id: string;
+            };
+            cookie?: never;
+        };
+        requestBody: {
+            content: {
+                "application/json": components["schemas"]["UpdateExperimentRequestBody"];
+            };
+        };
+        responses: {
+            /** @description Experiment updated successfully */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["UpdateExperimentResponseBody"];
+                };
+            };
+            /** @description Forbidden */
+            403: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "text/plain": string;
+                };
+            };
+            /** @description Experiment not found */
+            404: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "text/plain": string;
+                };
+            };
+            /** @description Invalid experiment ID or request body */
+            422: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "text/plain": string;
+                };
+            };
+        };
+    };
     getIncompleteExperimentRuns: {
         parameters: {
             query?: {