@arizeai/phoenix-client 6.10.0 → 6.11.0

package/src/testing/types.ts

@@ -0,0 +1,366 @@
+import type { PhoenixClient } from "../index";
+import type { AnnotatorKind } from "../types/annotations";
+import type {
+  EvaluatorParams,
+  EvaluationResult as ExperimentEvaluationResult,
+} from "../types/experiments";
+
+/**
+ * Phoenix annotator kind, re-exported from the shared client types so the
+ * testing module and the rest of the client agree on a single definition.
+ */
+export type { AnnotatorKind };
+
+/** A JSON-serializable map. */
+export type KVMap = Record<string, unknown>;
+
+/**
+ * Domain language
+ * ---------------
+ * The unit these tests evaluate over is an **Example**: a single AI example —
+ * an `input`, its `expected` output, and optional `metadata` / `splits` — over
+ * which the task under test is run and then scored. This is the same notion as
+ * the dataset `Example` (`../types/datasets`): each test case _is_ one example.
+ * When tracked, a case is recorded to Phoenix as a dataset example and
+ * evaluated as one experiment run.
+ *
+ * These field names form the shared vocabulary across this module:
+ * - `input`    — the example's input, passed to the task under evaluation.
+ * - `expected` — the example's expected (reference / ground-truth) output.
+ * - `metadata` — extra fields carried on the example.
+ * - `splits`   — slice labels for the example.
+ * - `id`       — stable example id, used to upsert the example across runs.
+ */
+
+/**
+ * The expected output of an `Example`, accepted under any one of three
+ * interchangeable keys. All three normalize to the same slot: when recorded to
+ * Phoenix the value becomes the dataset example's `output`, and it is exposed
+ * to evaluators as `expected` on `EvaluatorParams`. At most one key may be set.
+ *
+ * - `expected` — the canonical name (the ground-truth / reference output).
+ * - `reference` — alias preferred by frameworks that name the slot "reference".
+ * - `output` — alias for callers who think in terms of the example's `output`.
+ *
+ * Modeled as a union so supplying more than one key at a time is a type error.
+ */
+export type ReferenceOutput<Expected extends KVMap = KVMap> =
+  | { expected?: Expected; reference?: never; output?: never }
+  | { reference?: Expected; expected?: never; output?: never }
+  | { output?: Expected; expected?: never; reference?: never };
+
+/**
+ * The `Example` fields that define a single test case, excluding its
+ * expected output (which is supplied separately via {@link ReferenceOutput}).
+ *
+ * `input` is the example's input — the value fed to the task under evaluation.
+ * When the case is tracked, this becomes the dataset example's `input`.
+ */
+export interface TestParamsBase<Input extends KVMap = KVMap> {
+  /** Optional stable example id; used to upsert the example between runs. */
+  id?: string;
+  /** The example's input — fed to the task under evaluation. Required. */
+  input: Input;
+  /** Additional metadata stored on the example and its run. */
+  metadata?: KVMap;
+  /**
+   * Split assignment(s) for the example, used to slice the dataset and
+   * experiment in the Phoenix UI (e.g. `["factual_accuracy", "correct"]`).
+   */
+  splits?: string[];
+  /** Per-test config (tags + metadata recorded on the run). */
+  config?: TestConfig;
+  /**
+   * Number of times to run this test case. Each repetition becomes a
+   * separate experiment run against the same dataset example (carrying a
+   * distinct `repetition_number`). Overrides the suite-level `repetitions`.
+   * Defaults to the suite value, then `PHOENIX_TEST_REPETITIONS`, then `1`.
+   */
+  repetitions?: number;
+  /**
+   * When `true`, this test runs as an ordinary local test only — no dataset
+   * example is created and no experiment run or annotations are uploaded to
+   * Phoenix. Useful for scaffolding a case before it's ready to track.
+   */
+  dryRun?: boolean;
+}
+
+/**
+ * The full inline definition of a single `Example` under test.
+ *
+ * Combines {@link TestParamsBase} with a {@link ReferenceOutput}, so the
+ * example's expected output may be given under `expected`, `reference`, or
+ * `output` (at most one). All three resolve to the same canonical `expected`
+ * slot.
+ */
+export type TestParams<
+  Input extends KVMap = KVMap,
+  Expected extends KVMap = KVMap,
+> = TestParamsBase<Input> & ReferenceOutput<Expected>;
+
+/**
+ * Resolve an `Example`'s expected output from a value that may carry it
+ * under any of the `expected` / `reference` / `output` aliases (see
+ * {@link ReferenceOutput}). Returns the first one set, or `undefined` if none.
+ */
+export function resolveReference<Expected extends KVMap = KVMap>(
+  params: ReferenceOutput<Expected>
+): Expected | undefined {
+  return params.expected ?? params.reference ?? params.output;
+}
+
+/** Per-test runtime configuration. */
+export interface TestConfig {
+  /** Tags recorded on the experiment run for filtering in the Phoenix UI. */
+  tags?: string[];
+  /** Extra metadata recorded on the experiment run. */
+  metadata?: KVMap;
+}
+
+/**
+ * How a criterion aggregates an annotation's scores to gate the suite:
+ *
+ * - `"average"` — gate on overall quality: the **mean** score across all runs
+ *   must clear the criterion's `threshold`. A few weak runs are tolerated as
+ *   long as the mean holds.
+ * - `"passRate"` — gate on consistency: each run **passes** when the
+ *   criterion's `passFn` predicate returns `true` for its annotation, and the
+ *   suite passes when the **fraction** of runs that pass is at least
+ *   `minPassRate` (e.g. `minPassRate: 0.9` ⇒ 90% must pass; `1` ⇒ all).
+ */
+export type AcceptanceMetric = "average" | "passRate";
+
+/**
+ * Optimization direction for a criterion's scores: `"maximize"` (higher is
+ * better, the default) or `"minimize"` (lower is better). Controls every
+ * score comparison the criterion makes.
+ */
+export type OptimizationDirection = "maximize" | "minimize";
+
+/** Fields shared by every {@link AcceptanceCriterion} variant. */
+export interface AcceptanceCriterionBase {
+  /** Annotation name to aggregate across completed test runs. */
+  annotationName: string;
+}
+
+/**
+ * Gate the suite on the **mean** score: the average across all runs must clear
+ * `threshold` (compared in `direction`).
+ */
+export interface AverageAcceptanceCriterion extends AcceptanceCriterionBase {
+  metric: "average";
+  /**
+   * The bar the mean score must clear, compared in `direction`. Boolean scores
+   * average as `1` (`true`) / `0` (`false`).
+   */
+  threshold: number;
+  /**
+   * Optimization direction; defaults to `"maximize"`. `"maximize"` treats a
+   * higher mean as better (clears when `>= threshold`); `"minimize"` treats a
+   * lower mean as better (clears when `<= threshold`) — use it for cost,
+   * latency, or error-rate annotations.
+   */
+  direction?: OptimizationDirection;
+}
+
+/**
+ * Gate the suite on the **pass rate**: each run passes when `passFn` returns
+ * `true` for its annotation, and the suite passes when at least `minPassRate`
+ * of runs do. `passFn` decides what "passing" means, so any logic works — a
+ * score bar, a score range, a label match, a metadata check, etc.
+ */
+export interface PassRateAcceptanceCriterion extends AcceptanceCriterionBase {
+  metric: "passRate";
+  /**
+   * Predicate deciding whether a single run passes, given the run's last
+   * {@link Annotation} for `annotationName` (its `score`, `label`,
+   * `explanation`, `metadata`, …). Runs whose predicate returns `true` count
+   * toward the pass rate.
+   */
+  passFn: (annotation: Annotation) => boolean;
+  /**
+   * Minimum fraction of runs (`0`–`1`) that must pass for the suite to pass —
+   * e.g. `0.9` requires 90% of runs to satisfy `passFn`, `1` requires all of
+   * them. The suite passes when `passRate >= minPassRate`.
+   */
+  minPassRate: number;
+}
+
+/**
+ * One aggregate acceptance rule, evaluated once after every test in the suite
+ * has run. Each criterion aggregates a single annotation's scores with one
+ * {@link AcceptanceMetric} and fails the suite when the result misses its bar.
+ *
+ * Scoring notes shared by every metric:
+ * - Boolean scores count as `1` (`true`) / `0` (`false`).
+ * - If a run logs the same annotation more than once, the last one counts.
+ * - Skipped tests are excluded; dry-run tests are included (they still run).
+ * - A criterion whose annotation was never logged on any run fails (rather
+ *   than passing vacuously) — see {@link AcceptanceResultFields.failureReason}.
+ */
+export type AcceptanceCriterion =
+  | AverageAcceptanceCriterion
+  | PassRateAcceptanceCriterion;
+
+/** The computed fields added to an {@link AcceptanceCriterion} once evaluated. */
+export interface AcceptanceResultFields {
+  /**
+   * The aggregate the criterion gated on, or `null` when there were no runs to
+   * aggregate. For `"average"` this is the mean score; for `"passRate"` it is
+   * the fraction of runs that passed (so a fully-passing `"passRate"` criterion
+   * reports `1`).
+   */
+  value: number | null;
+  /** Number of runs included in the aggregate. */
+  sampleCount: number;
+  /** Whether the aggregate cleared the criterion. */
+  passed: boolean;
+  /** Human-readable failure reason for invalid or empty aggregates. */
+  failureReason?: string;
+}
+
+/** Computed result for one aggregate acceptance rule. */
+export type AcceptanceResult = AcceptanceCriterion & AcceptanceResultFields;
+
+/** Suite-level configuration accepted by `describe()`. */
+export interface SuiteConfig {
+  /** Override the dataset / experiment name used for the suite. */
+  datasetName?: string;
+  /** Description for the dataset and experiment. */
+  description?: string;
+  /** Suite-level metadata applied to every run in this experiment. */
+  metadata?: KVMap;
+  /** Override the Phoenix client used for syncing this suite. */
+  client?: PhoenixClient;
+  /**
+   * Number of times to run each test case in this suite. Individual tests
+   * may override this via `TestParams.repetitions`. Defaults to the
+   * `PHOENIX_TEST_REPETITIONS` env var, then `1`.
+   */
+  repetitions?: number;
+  /**
+   * When `true`, the whole suite runs as ordinary local tests — no dataset
+   * is uploaded and no experiment, runs, or annotations are created in
+   * Phoenix. Equivalent to `PHOENIX_TEST_TRACING=false` scoped to this
+   * suite. The reporter still prints a local summary.
+   */
+  dryRun?: boolean;
+  /**
+   * Aggregate annotation criteria that gate the suite after all tests run.
+   * Each criterion fails the suite when its scores miss the configured bar
+   * (see {@link AcceptanceCriterion}).
+   */
+  acceptanceCriteria?: AcceptanceCriterion[];
+}
+
+/**
+ * Arguments passed to a `test()` body: the `Example` under test, exposed
+ * as its `input`, `expected` output, and `metadata`. Read straight from the
+ * test's {@link TestParams} — the runner does not transform them.
+ */
+export interface TestArgs<
+  Input extends KVMap = KVMap,
+  Expected extends KVMap = KVMap,
+> {
+  /** The example's input under test. */
+  input: Input;
+  /** The example's expected (reference) output, when one was supplied. */
+  expected?: Expected;
+  /** Any metadata attached to the example. */
+  metadata?: KVMap;
+}
+
+/**
+ * Object form of an evaluator result. Reuses the shared experiment
+ * {@link ExperimentEvaluationResult} shape (label / explanation / metadata)
+ * but widens `score` to also accept booleans, which the testing API stores as
+ * `1` / `0`.
+ */
+export interface EvaluationResultObject extends Omit<
+  ExperimentEvaluationResult,
+  "score"
+> {
+  /** Numeric or boolean score; booleans are stored as `1` / `0`. */
+  score?: number | boolean | null;
+}
+
+/**
+ * One annotation recorded against a run. Extends the evaluator
+ * {@link EvaluationResultObject} with the `name` and `annotatorKind` carried
+ * on the evaluation body, plus an optional originating trace id.
+ */
+export interface Annotation extends EvaluationResultObject {
+  /** Phoenix evaluation name. Required, and unique per run (last write wins). */
+  name: string;
+  /** Who or what produced the annotation. Defaults to `"CODE"`. */
+  annotatorKind?: AnnotatorKind;
+  /** Trace id for this evaluation, when the annotation was produced by a traced evaluator. */
+  traceId?: string | null;
+}
+
+/** Result returned by `traceEvaluator` for any evaluator-shaped value. */
+export type EvaluatorResult = Annotation | (KVMap & { name: string });
+
+/** Result shape produced by evaluator objects used in eval tests. */
+export type EvaluationResult =
+  | number
+  | boolean
+  | string
+  | null
+  | EvaluationResultObject;
+
+/**
+ * Parameters passed to an evaluator when it runs inside a test. A relaxation of
+ * the shared {@link EvaluatorParams}: `input` is always present, while `output`
+ * (an evaluator may run before `logOutput()`) and the remaining fields are
+ * optional. Deriving from `EvaluatorParams` keeps this aligned with the
+ * experiment evaluator contract as that shape evolves.
+ */
+export type EvaluationParams = Partial<EvaluatorParams> & {
+  /** The example's input under test. */
+  input: KVMap;
+};
+
+/** Structural evaluator interface accepted by `evaluate()`. */
+export interface Evaluator<
+  Params extends KVMap = EvaluationParams & KVMap,
+  Result = EvaluationResult,
+> {
+  /** Annotation/evaluation name. */
+  name: string;
+  /** Who or what produced the result. Defaults to `"CODE"`. */
+  kind?: AnnotatorKind;
+  /** Compute the evaluation result. */
+  evaluate: (params: Params) => Result | Promise<Result>;
+}
+
+/** Test handler signature. */
+export type TestFn<
+  Input extends KVMap = KVMap,
+  Expected extends KVMap = KVMap,
+> = (args: TestArgs<Input, Expected>) => unknown | Promise<unknown>;
+
+/**
+ * Each-row shape accepted by `test.each(table)(name, fn)`; each row defines one
+ * `Example`.
+ *
+ * Like {@link TestParams}, the example's expected output is supplied via
+ * {@link ReferenceOutput} (`expected` / `reference` / `output`, at most one).
+ * The trailing index signature still permits arbitrary extra columns on a row
+ * (e.g. for `%j` name interpolation) without weakening that constraint.
+ */
+export type TestEachRow<
+  Input extends KVMap = KVMap,
+  Expected extends KVMap = KVMap,
+> = {
+  id?: string;
+  input: Input;
+  metadata?: KVMap;
+  /** Per-row split assignment(s); see `TestParams.splits`. */
+  splits?: string[];
+  /** Per-row repetition count; see `TestParams.repetitions`. */
+  repetitions?: number;
+  /** Per-row dry-run flag; see `TestParams.dryRun`. */
+  dryRun?: boolean;
+} & ReferenceOutput<Expected> &
+  Record<string, unknown>;

package/src/utils/channel.ts

@@ -74,7 +74,7 @@
  *
  * @see https://en.wikipedia.org/wiki/Communicating_sequential_processes
  *
- * @template T The type of values sent through the channel
+ * @template Message The type of values sent through the channel
  *
  * @example Safe Producer-Consumer Pattern
  * ```typescript
@@ -116,8 +116,8 @@
 /**
  * Internal type for blocked senders waiting to deliver values
  */
-interface Sender<T> {
-  readonly value: T;
+interface Sender<Message> {
+  readonly value: Message;
   readonly resolve: () => void;
   readonly reject: (error: Error) => void;
 }
@@ -125,8 +125,8 @@ interface Sender<T> {
 /**
  * Internal type for blocked receivers waiting for values
  */
-interface Receiver<T> {
-  readonly resolve: (value: T | typeof CLOSED) => void;
+interface Receiver<Message> {
+  readonly resolve: (value: Message | typeof CLOSED) => void;
   readonly reject: (error: Error) => void;
 }
 
@@ -149,10 +149,10 @@ const ERRORS = {
   NEGATIVE_CAPACITY: "Channel capacity must be non-negative",
 } as const satisfies Record<string, string>;
 
-export class Channel<T> {
-  #buffer: T[] = [];
-  #sendQueue: Sender<T>[] = [];
-  #receiveQueue: Receiver<T>[] = [];
+export class Channel<Message> {
+  #buffer: Message[] = [];
+  #sendQueue: Sender<Message>[] = [];
+  #receiveQueue: Receiver<Message>[] = [];
   #closed = false;
   readonly #capacity: number;
 
@@ -191,7 +191,7 @@ export class Channel<T> {
    * @param value - The value to send
    * @throws {ChannelError} If channel is closed
    */
-  async send(value: T): Promise<void> {
+  async send(value: Message): Promise<void> {
     if (this.#closed) {
       throw new ChannelError(ERRORS.SEND_TO_CLOSED);
     }
@@ -221,7 +221,7 @@ export class Channel<T> {
    *
    * @returns The received value, or CLOSED symbol if channel is closed and empty
    */
-  async receive(): Promise<T | typeof CLOSED> {
+  async receive(): Promise<Message | typeof CLOSED> {
     // Drain buffer first
     if (this.#buffer.length > 0) {
       const value = this.#buffer.shift()!;
@@ -249,7 +249,7 @@ export class Channel<T> {
     }
 
     // Block until value available
-    return new Promise<T | typeof CLOSED>((resolve, reject) => {
+    return new Promise<Message | typeof CLOSED>((resolve, reject) => {
       this.#receiveQueue.push({ resolve, reject });
     });
   }
@@ -271,7 +271,7 @@ export class Channel<T> {
    * }
    * ```
    */
-  tryReceive(): T | typeof CLOSED | undefined {
+  tryReceive(): Message | typeof CLOSED | undefined {
     // Drain buffer first
     if (this.#buffer.length > 0) {
       const value = this.#buffer.shift()!;
@@ -362,7 +362,7 @@ export class Channel<T> {
   /**
    * Async iterator support for for-await-of loops
    */
-  async *[Symbol.asyncIterator](): AsyncIterableIterator<T> {
+  async *[Symbol.asyncIterator](): AsyncIterableIterator<Message> {
     while (true) {
       const value = await this.receive();
       if (value === CLOSED) return;
@@ -392,6 +392,8 @@ export const CLOSED = Symbol("CLOSED");
  * }
  * ```
  */
-export function isClosed<T>(value: T | typeof CLOSED): value is typeof CLOSED {
+export function isClosed<Message>(
+  value: Message | typeof CLOSED
+): value is typeof CLOSED {
   return value === CLOSED;
 }

package/src/utils/promisifyResult.ts

@@ -2,12 +2,14 @@
  * If the incoming function returns a promise, return the promise.
  * Otherwise, return a promise that resolves to the incoming function's return value.
  */
-export function promisifyResult<T>(result: T) {
+export function promisifyResult<Result>(result: Result) {
   if (result instanceof Promise) {
-    return result as T extends Promise<infer U> ? Promise<U> : never;
+    return result as Result extends Promise<infer ResolvedValue>
+      ? Promise<ResolvedValue>
+      : never;
   }
 
-  return Promise.resolve(result) as T extends Promise<unknown>
+  return Promise.resolve(result) as Result extends Promise<unknown>
     ? never
-    : Promise<T>;
+    : Promise<Result>;
 }

package/src/utils/schemaMatches.ts

@@ -3,8 +3,10 @@ import type { z } from "zod";
 /**
  * Simple utility to check if two types are exactly equivalent
  */
-export type AssertEqual<T, U> =
-  (<V>() => V extends T ? 1 : 2) extends <V>() => V extends U ? 1 : 2
+export type AssertEqual<Actual, Expected> =
+  (<Argument>() => Argument extends Actual ? 1 : 2) extends <
+    Argument,
+  >() => Argument extends Expected ? 1 : 2
     ? true
     : false;
 
@@ -14,16 +16,16 @@ export type AssertEqual<T, U> =
  * @see https://github.com/colinhacks/zod/issues/372#issuecomment-2445439772
  */
 export const schemaMatches =
-  <T>() =>
-  <S extends z.ZodType<T, unknown>>(
-    schema: AssertEqual<S["_output"], T> extends true
-      ? S
-      : S & {
+  <Value>() =>
+  <Schema extends z.ZodType<Value, unknown>>(
+    schema: AssertEqual<Schema["_output"], Value> extends true
+      ? Schema
+      : Schema & {
           "types do not match": {
-            expected: T;
-            received: S["_output"];
+            expected: Value;
+            received: Schema["_output"];
           };
         }
-  ): S => {
+  ): Schema => {
     return schema;
   };

package/src/vitest/index.ts

@@ -0,0 +1,57 @@
+import {
+  afterAll as vitestAfterAll,
+  beforeAll as vitestBeforeAll,
+  describe as vitestDescribe,
+  test as vitestTest,
+} from "vitest";
+
+import { createTestApi } from "../testing/define-api";
+import type { RunnerHooks } from "../testing/runner";
+
+export type {
+  PhoenixDescribe,
+  PhoenixTest,
+  PhoenixTestApi,
+  PhoenixTestEach,
+} from "../testing/define-api";
+
+export type {
+  AcceptanceCriterion,
+  AcceptanceMetric,
+  AcceptanceResult,
+  Annotation,
+  AnnotatorKind,
+  EvaluationParams,
+  EvaluationResult,
+  Evaluator,
+  EvaluatorResult,
+  KVMap,
+  ReferenceOutput,
+  SuiteConfig,
+  TestArgs,
+  TestConfig,
+  TestEachRow,
+  TestFn,
+  TestParams,
+  TestParamsBase,
+} from "../testing/types";
+
+export {
+  evaluate,
+  logAnnotation,
+  logOutput,
+  traceEvaluator,
+} from "../testing/helpers";
+
+const hooks: RunnerHooks = {
+  describe: vitestDescribe,
+  describeOnly: vitestDescribe.only,
+  describeSkip: vitestDescribe.skip,
+  test: vitestTest,
+  testOnly: vitestTest.only,
+  testSkip: vitestTest.skip,
+  beforeAll: vitestBeforeAll,
+  afterAll: vitestAfterAll,
+};
+
+export const { describe, test, it } = createTestApi(() => hooks);

package/src/vitest/reporter.ts

@@ -0,0 +1,32 @@
+import { SuiteSummaryReportRun } from "../testing/report-run";
+import {
+  formatSuiteSummary,
+  printSuiteSummaries,
+} from "../testing/reporter-format";
+
+/**
+ * Vitest reporter for `@arizeai/phoenix-client/vitest`.
+ *
+ * The reporter does not replace Vitest's default test output. Instead it
+ * appends a Phoenix-flavored summary at the end of the run that lists
+ * outputs, annotations, and dataset/experiment links for each suite.
+ *
+ * The class implements the Vitest `Reporter` interface structurally rather
+ * than nominally so we don't have to import a CJS type from `vitest/reporters`.
+ */
+export default class PhoenixVitestReporter {
+  private readonly report = new SuiteSummaryReportRun();
+
+  onTestRunStart(): void {
+    this.report.begin();
+  }
+  onTestRunEnd(): void {
+    this.report.finish();
+  }
+  // Vitest also calls `onFinished` on legacy reporters; alias for safety.
+  onFinished(): void {
+    this.report.finish();
+  }
+}
+
+export { formatSuiteSummary, printSuiteSummaries };