@arizeai/phoenix-client 6.10.1 → 6.11.1

package/src/testing/acceptance.ts

@@ -0,0 +1,190 @@
+import type { TestResult } from "./state";
+import type {
+  AcceptanceCriterion,
+  AcceptanceResult,
+  Annotation,
+  OptimizationDirection,
+} from "./types";
+
+/**
+ * Evaluate all configured aggregate acceptance rules against completed runs.
+ * @param params - Evaluation parameters.
+ * @param params.criteria - Aggregate rules configured on the suite.
+ * @param params.results - Completed test results in the suite.
+ */
+export function evaluateAcceptanceCriteria({
+  criteria,
+  results,
+}: {
+  criteria: readonly AcceptanceCriterion[] | undefined;
+  results: readonly TestResult[];
+}): AcceptanceResult[] {
+  if (!criteria || criteria.length === 0) {
+    return [];
+  }
+  return criteria.map((criterion) =>
+    evaluateAcceptanceCriterion({ criterion, results })
+  );
+}
+
+/**
+ * Build one error containing every failed aggregate criterion.
+ * @param results - Computed acceptance results.
+ */
+export function createAcceptanceFailureError(
+  results: readonly AcceptanceResult[]
+): Error | undefined {
+  const failedResults = results.filter((result) => !result.passed);
+  if (failedResults.length === 0) {
+    return undefined;
+  }
+  return new Error(
+    [
+      "Acceptance criteria failed:",
+      ...failedResults.map((result) => `  ${formatAcceptanceResult(result)}`),
+    ].join("\n")
+  );
+}
+
+/** Format an acceptance result for reporters and thrown errors. */
+export function formatAcceptanceResult(result: AcceptanceResult): string {
+  const status = result.passed ? "PASS" : "FAIL";
+  const value = result.value === null ? "n/a" : result.value.toFixed(3);
+  const sampleLabel = result.sampleCount === 1 ? "sample" : "samples";
+  let requirement: string;
+  if (result.metric === "average") {
+    const cmp = (result.direction ?? "maximize") === "minimize" ? "<=" : ">=";
+    requirement = `mean ${cmp} ${result.threshold.toFixed(3)}`;
+  } else {
+    requirement = `pass rate >= ${result.minPassRate.toFixed(3)}`;
+  }
+  const reason = result.failureReason ? ` - ${result.failureReason}` : "";
+  return `${status} ${result.annotationName} ${result.metric} ${value} (need ${requirement}; ${result.sampleCount} ${sampleLabel})${reason}`;
+}
+
+function evaluateAcceptanceCriterion({
+  criterion,
+  results,
+}: {
+  criterion: AcceptanceCriterion;
+  results: readonly TestResult[];
+}): AcceptanceResult {
+  const annotations = collectAnnotations({ criterion, results });
+
+  if (criterion.metric === "average") {
+    // Only numeric / boolean scores can be averaged.
+    const scores = annotations
+      .map((annotation) => annotation.score)
+      .filter(isValidScore);
+    if (scores.length === 0) {
+      return {
+        ...criterion,
+        value: null,
+        sampleCount: 0,
+        passed: false,
+        failureReason: "no numeric or boolean scores found",
+      };
+    }
+    const direction = criterion.direction ?? "maximize";
+    const value = calculateAverage(scores);
+    return {
+      ...criterion,
+      value,
+      sampleCount: scores.length,
+      passed: meetsBar(value, criterion.threshold, direction),
+    };
+  }
+
+  // passRate: each run passes when `passFn` returns true for its annotation;
+  // the suite passes when the fraction of passing runs is at least
+  // `minPassRate`. The reported value is that fraction.
+  if (annotations.length === 0) {
+    return {
+      ...criterion,
+      value: null,
+      sampleCount: 0,
+      passed: false,
+      failureReason: "no matching annotations found",
+    };
+  }
+  const passed = annotations.filter((annotation) =>
+    criterion.passFn(annotation)
+  ).length;
+  const value = passed / annotations.length;
+  return {
+    ...criterion,
+    value,
+    sampleCount: annotations.length,
+    passed: value >= criterion.minPassRate,
+  };
+}
+
+/** Whether `value` clears `bar` in the given optimization direction. */
+function meetsBar(
+  value: number,
+  bar: number,
+  direction: OptimizationDirection
+): boolean {
+  return direction === "minimize" ? value <= bar : value >= bar;
+}
+
+/**
+ * The last annotation matching `annotationName` from each non-skipped run that
+ * logged it. One entry per run; runs that never logged the annotation are
+ * omitted.
+ */
+function collectAnnotations({
+  criterion,
+  results,
+}: {
+  criterion: AcceptanceCriterion;
+  results: readonly TestResult[];
+}): Annotation[] {
+  return results
+    .filter((result) => result.status !== "skipped")
+    .map((result) =>
+      findLastAnnotation({
+        annotations: result.annotations,
+        annotationName: criterion.annotationName,
+      })
+    )
+    .filter((annotation): annotation is Annotation => annotation !== undefined);
+}
+
+function findLastAnnotation({
+  annotations,
+  annotationName,
+}: {
+  annotations: readonly Annotation[];
+  annotationName: string;
+}): Annotation | undefined {
+  for (
+    let annotationIndex = annotations.length - 1;
+    annotationIndex >= 0;
+    annotationIndex--
+  ) {
+    const annotation = annotations[annotationIndex];
+    if (annotation?.name === annotationName) {
+      return annotation;
+    }
+  }
+  return undefined;
+}
+
+function isValidScore(score: Annotation["score"]): score is number | boolean {
+  return (
+    typeof score === "boolean" ||
+    (typeof score === "number" && Number.isFinite(score))
+  );
+}
+
+function calculateAverage(scores: readonly (number | boolean)[]): number {
+  const total = scores
+    .map(scoreToNumber)
+    .reduce((sum, score) => sum + score, 0);
+  return total / scores.length;
+}
+
+function scoreToNumber(score: number | boolean): number {
+  return typeof score === "boolean" ? (score ? 1 : 0) : score;
+}

package/src/testing/define-api.ts

@@ -0,0 +1,279 @@
+import { declareDescribe, declareTest, type RunnerHooks } from "./runner";
+import {
+  type KVMap,
+  resolveReference,
+  type SuiteConfig,
+  type TestEachRow,
+  type TestFn,
+  type TestParams,
+} from "./types";
+
+/**
+ * Declare Phoenix eval test suites.
+ *
+ * Drop-in replacement for the test runner's own `describe`. The suite name
+ * doubles as the dataset and experiment name on the Phoenix server, and the
+ * optional {@link SuiteConfig} controls dataset naming, repetitions, dry-run
+ * mode, and CI acceptance criteria.
+ *
+ * @example
+ * ```ts
+ * import * as px from "@arizeai/phoenix-client/vitest";
+ *
+ * px.describe("generate sql demo", () => {
+ *   px.test("offtopic input", { input: { question: "hi" } }, async ({ input }) => {
+ *     // ...
+ *   });
+ * }, { metadata: { model: "gpt-4o-mini" } });
+ * ```
+ */
+export interface PhoenixDescribe {
+  /**
+   * Declare a Phoenix eval test suite.
+   *
+   * @param name - Suite name; doubles as the dataset / experiment name on Phoenix.
+   * @param fn - Suite body that declares its `test` / `it` cases.
+   * @param config - Optional suite-level config (dataset name, repetitions, dry-run, acceptance criteria).
+   */
+  (name: string, fn: () => void, config?: SuiteConfig): void;
+  /**
+   * Run only this suite, skipping all sibling suites
+   * (matches the runner's `describe.only`).
+   *
+   * @param name - Suite name; doubles as the dataset / experiment name on Phoenix.
+   * @param fn - Suite body that declares its `test` / `it` cases.
+   * @param config - Optional suite-level config.
+   */
+  only(name: string, fn: () => void, config?: SuiteConfig): void;
+  /**
+   * Skip this suite entirely (matches the runner's `describe.skip`). No dataset
+   * or experiment is created on Phoenix.
+   *
+   * @param name - Suite name; doubles as the dataset / experiment name on Phoenix.
+   * @param fn - Suite body (not executed).
+   * @param config - Optional suite-level config.
+   */
+  skip(name: string, fn: () => void, config?: SuiteConfig): void;
+}
+
+/**
+ * The test body returned by {@link PhoenixTest.each} after a table is bound.
+ *
+ * @param name - Test name, or a template (`%i` / `%s` / `%j`), or a function
+ *   that derives the name from the row and its index.
+ * @param fn - The test handler, run once per row in the bound table.
+ * @param timeout - Optional per-test timeout in milliseconds.
+ */
+export type PhoenixTestEach<
+  Input extends KVMap = KVMap,
+  Expected extends KVMap = KVMap,
+> = (
+  name: string | ((row: TestEachRow<Input, Expected>, index: number) => string),
+  fn: TestFn<Input, Expected>,
+  timeout?: number
+) => void;
+
+/**
+ * Declare a single Phoenix eval test case.
+ *
+ * Drop-in replacement for the test runner's own `test` / `it`. The `params`
+ * argument carries the `input` and the reference output (`expected` /
+ * `reference` / `output`) that become the dataset example; whatever the handler
+ * returns (or passes to `logOutput()`) is recorded as the experiment run's
+ * output and made available to evaluators.
+ *
+ * `it` is the canonical alias for `test`; the two are identical.
+ *
+ * @example
+ * ```ts
+ * px.test(
+ *   "summarizes the article",
+ *   { input: { article }, expected: { summary } },
+ *   async ({ input, expected }) => {
+ *     const output = await summarize(input.article);
+ *     px.logOutput(output);
+ *     await px.evaluate({ name: "matches", evaluate: () => output === expected.summary });
+ *   }
+ * );
+ * ```
+ */
+export interface PhoenixTest {
+  /**
+   * Declare a single Phoenix eval test case.
+   *
+   * @param name - Test case name; doubles as the dataset example label.
+   * @param params - Inline `input` and reference output that become the dataset example.
+   * @param fn - Test handler; receives `{ input, expected, metadata }`.
+   * @param timeout - Optional per-test timeout in milliseconds.
+   */
+  <Input extends KVMap = KVMap, Expected extends KVMap = KVMap>(
+    name: string,
+    params: TestParams<Input, Expected>,
+    fn: TestFn<Input, Expected>,
+    timeout?: number
+  ): void;
+  /**
+   * Run only this test case, skipping its siblings
+   * (matches the runner's `test.only`).
+   *
+   * @param name - Test case name; doubles as the dataset example label.
+   * @param params - Inline `input` and reference output that become the dataset example.
+   * @param fn - Test handler; receives `{ input, expected, metadata }`.
+   * @param timeout - Optional per-test timeout in milliseconds.
+   */
+  only<Input extends KVMap = KVMap, Expected extends KVMap = KVMap>(
+    name: string,
+    params: TestParams<Input, Expected>,
+    fn: TestFn<Input, Expected>,
+    timeout?: number
+  ): void;
+  /**
+   * Skip this test case (matches the runner's `test.skip`). No dataset example
+   * or experiment run is created on Phoenix.
+   *
+   * @param name - Test case name; doubles as the dataset example label.
+   * @param params - Inline `input` and reference output (not used while skipped).
+   * @param fn - Test handler (not executed).
+   * @param timeout - Optional per-test timeout in milliseconds.
+   */
+  skip<Input extends KVMap = KVMap, Expected extends KVMap = KVMap>(
+    name: string,
+    params: TestParams<Input, Expected>,
+    fn: TestFn<Input, Expected>,
+    timeout?: number
+  ): void;
+  /**
+   * Run the same test handler across many examples. Returns a function that
+   * takes a name (or template / name-builder) and the shared test body; each
+   * row in `table` becomes its own dataset example and experiment run.
+   *
+   * @param table - Rows of `{ input, expected?, metadata?, ... }` to fan out over.
+   * @returns A {@link PhoenixTestEach} that binds the name and shared handler.
+   *
+   * @example
+   * ```ts
+   * px.test.each([
+   *   { input: { a: 1, b: 2 }, expected: { sum: 3 } },
+   *   { input: { a: 2, b: 2 }, expected: { sum: 4 } },
+   * ])("adds %j", async ({ input, expected }) => {
+   *   // ...
+   * });
+   * ```
+   */
+  each<Input extends KVMap, Expected extends KVMap>(
+    table: TestEachRow<Input, Expected>[]
+  ): PhoenixTestEach<Input, Expected>;
+}
+
+/** The public testing surface returned by {@link createTestApi}. */
+export interface PhoenixTestApi {
+  /** Declare a Phoenix eval test suite. See {@link PhoenixDescribe}. */
+  describe: PhoenixDescribe;
+  /** Declare a Phoenix eval test case. See {@link PhoenixTest}. */
+  test: PhoenixTest;
+  /** Canonical alias for {@link PhoenixTestApi.test}. */
+  it: PhoenixTest;
+}
+
+/**
+ * Build the public `describe`/`test`/`it` API for a runner adapter.
+ *
+ * Both the jest and vitest entrypoints expose the identical surface; the only
+ * thing that differs between them is how the {@link RunnerHooks} are obtained
+ * (vitest imports them statically, jest resolves them lazily from globals).
+ * That difference is captured by `getHooks`, which is invoked once per
+ * declaration so adapters are free to resolve hooks lazily.
+ *
+ * The JSDoc that surfaces in editors lives on the {@link PhoenixDescribe} and
+ * {@link PhoenixTest} interfaces rather than the implementations below, so the
+ * docs survive the `export const { describe, test, it } = createTestApi(...)`
+ * destructuring in each adapter.
+ */
+export function createTestApi(getHooks: () => RunnerHooks): PhoenixTestApi {
+  const describe = ((
+    name: string,
+    fn: () => void,
+    config?: SuiteConfig
+  ): void => {
+    declareDescribe(getHooks(), name, fn, config ?? {});
+  }) as PhoenixDescribe;
+  describe.only = (name, fn, config) => {
+    declareDescribe(getHooks(), name, fn, config ?? {}, "only");
+  };
+  describe.skip = (name, fn, config) => {
+    declareDescribe(getHooks(), name, fn, config ?? {}, "skip");
+  };
+
+  const test = (<Input extends KVMap = KVMap, Expected extends KVMap = KVMap>(
+    name: string,
+    params: TestParams<Input, Expected>,
+    fn: TestFn<Input, Expected>,
+    timeout?: number
+  ): void => {
+    declareTest(getHooks(), name, params, fn, "default", timeout);
+  }) as PhoenixTest;
+  test.only = (name, params, fn, timeout) => {
+    declareTest(getHooks(), name, params, fn, "only", timeout);
+  };
+  test.skip = (name, params, fn, timeout) => {
+    declareTest(getHooks(), name, params, fn, "skip", timeout);
+  };
+  test.each = <Input extends KVMap, Expected extends KVMap>(
+    table: TestEachRow<Input, Expected>[]
+  ): PhoenixTestEach<Input, Expected> => {
+    return (name, fn, timeout) => {
+      table.forEach((row, i) => {
+        const testName =
+          typeof name === "function"
+            ? name(row, i)
+            : interpolateName(name, row, i);
+        declareTest(
+          getHooks(),
+          testName,
+          {
+            id: row.id,
+            input: row.input,
+            expected: resolveReference(row),
+            metadata: row.metadata,
+            splits: row.splits,
+            repetitions: row.repetitions,
+            dryRun: row.dryRun,
+          },
+          fn,
+          "default",
+          timeout
+        );
+      });
+    };
+  };
+
+  // `it` is the canonical alias for `test`.
+  const it = test;
+
+  return { describe, test, it };
+}
+
+/**
+ * Interpolate a `test.each` name template for a single row. Supports the
+ * common `%i`/`%s`/`%j` placeholders for surface parity with the underlying
+ * runners; when no placeholder is present the 1-based row index is appended.
+ */
+function interpolateName(
+  name: string,
+  row: TestEachRow,
+  index: number
+): string {
+  if (!name.includes("%")) {
+    return `${name} #${index + 1}`;
+  }
+  const replacements: Array<[RegExp, string]> = [
+    [/%i/g, String(index)],
+    [/%s/g, JSON.stringify(row.input)],
+    [/%j/g, JSON.stringify(row)],
+  ];
+  let out = name;
+  for (const [pattern, value] of replacements) {
+    out = out.replace(pattern, value);
+  }
+  return out;
+}