@arizeai/phoenix-client 6.10.0 → 6.11.0

package/src/testing/define-api.ts

@@ -0,0 +1,279 @@
+import { declareDescribe, declareTest, type RunnerHooks } from "./runner";
+import {
+  type KVMap,
+  resolveReference,
+  type SuiteConfig,
+  type TestEachRow,
+  type TestFn,
+  type TestParams,
+} from "./types";
+
+/**
+ * Declare Phoenix eval test suites.
+ *
+ * Drop-in replacement for the test runner's own `describe`. The suite name
+ * doubles as the dataset and experiment name on the Phoenix server, and the
+ * optional {@link SuiteConfig} controls dataset naming, repetitions, dry-run
+ * mode, and CI acceptance criteria.
+ *
+ * @example
+ * ```ts
+ * import * as px from "@arizeai/phoenix-client/vitest";
+ *
+ * px.describe("generate sql demo", () => {
+ *   px.test("offtopic input", { input: { question: "hi" } }, async ({ input }) => {
+ *     // ...
+ *   });
+ * }, { metadata: { model: "gpt-4o-mini" } });
+ * ```
+ */
+export interface PhoenixDescribe {
+  /**
+   * Declare a Phoenix eval test suite.
+   *
+   * @param name - Suite name; doubles as the dataset / experiment name on Phoenix.
+   * @param fn - Suite body that declares its `test` / `it` cases.
+   * @param config - Optional suite-level config (dataset name, repetitions, dry-run, acceptance criteria).
+   */
+  (name: string, fn: () => void, config?: SuiteConfig): void;
+  /**
+   * Run only this suite, skipping all sibling suites
+   * (matches the runner's `describe.only`).
+   *
+   * @param name - Suite name; doubles as the dataset / experiment name on Phoenix.
+   * @param fn - Suite body that declares its `test` / `it` cases.
+   * @param config - Optional suite-level config.
+   */
+  only(name: string, fn: () => void, config?: SuiteConfig): void;
+  /**
+   * Skip this suite entirely (matches the runner's `describe.skip`). No dataset
+   * or experiment is created on Phoenix.
+   *
+   * @param name - Suite name; doubles as the dataset / experiment name on Phoenix.
+   * @param fn - Suite body (not executed).
+   * @param config - Optional suite-level config.
+   */
+  skip(name: string, fn: () => void, config?: SuiteConfig): void;
+}
+
+/**
+ * The test body returned by {@link PhoenixTest.each} after a table is bound.
+ *
+ * @param name - Test name, or a template (`%i` / `%s` / `%j`), or a function
+ *   that derives the name from the row and its index.
+ * @param fn - The test handler, run once per row in the bound table.
+ * @param timeout - Optional per-test timeout in milliseconds.
+ */
+export type PhoenixTestEach<
+  Input extends KVMap = KVMap,
+  Expected extends KVMap = KVMap,
+> = (
+  name: string | ((row: TestEachRow<Input, Expected>, index: number) => string),
+  fn: TestFn<Input, Expected>,
+  timeout?: number
+) => void;
+
+/**
+ * Declare a single Phoenix eval test case.
+ *
+ * Drop-in replacement for the test runner's own `test` / `it`. The `params`
+ * argument carries the `input` and the reference output (`expected` /
+ * `reference` / `output`) that become the dataset example; whatever the handler
+ * returns (or passes to `logOutput()`) is recorded as the experiment run's
+ * output and made available to evaluators.
+ *
+ * `it` is the canonical alias for `test`; the two are identical.
+ *
+ * @example
+ * ```ts
+ * px.test(
+ *   "summarizes the article",
+ *   { input: { article }, expected: { summary } },
+ *   async ({ input, expected }) => {
+ *     const output = await summarize(input.article);
+ *     px.logOutput(output);
+ *     await px.evaluate({ name: "matches", evaluate: () => output === expected.summary });
+ *   }
+ * );
+ * ```
+ */
+export interface PhoenixTest {
+  /**
+   * Declare a single Phoenix eval test case.
+   *
+   * @param name - Test case name; doubles as the dataset example label.
+   * @param params - Inline `input` and reference output that become the dataset example.
+   * @param fn - Test handler; receives `{ input, expected, metadata }`.
+   * @param timeout - Optional per-test timeout in milliseconds.
+   */
+  <Input extends KVMap = KVMap, Expected extends KVMap = KVMap>(
+    name: string,
+    params: TestParams<Input, Expected>,
+    fn: TestFn<Input, Expected>,
+    timeout?: number
+  ): void;
+  /**
+   * Run only this test case, skipping its siblings
+   * (matches the runner's `test.only`).
+   *
+   * @param name - Test case name; doubles as the dataset example label.
+   * @param params - Inline `input` and reference output that become the dataset example.
+   * @param fn - Test handler; receives `{ input, expected, metadata }`.
+   * @param timeout - Optional per-test timeout in milliseconds.
+   */
+  only<Input extends KVMap = KVMap, Expected extends KVMap = KVMap>(
+    name: string,
+    params: TestParams<Input, Expected>,
+    fn: TestFn<Input, Expected>,
+    timeout?: number
+  ): void;
+  /**
+   * Skip this test case (matches the runner's `test.skip`). No dataset example
+   * or experiment run is created on Phoenix.
+   *
+   * @param name - Test case name; doubles as the dataset example label.
+   * @param params - Inline `input` and reference output (not used while skipped).
+   * @param fn - Test handler (not executed).
+   * @param timeout - Optional per-test timeout in milliseconds.
+   */
+  skip<Input extends KVMap = KVMap, Expected extends KVMap = KVMap>(
+    name: string,
+    params: TestParams<Input, Expected>,
+    fn: TestFn<Input, Expected>,
+    timeout?: number
+  ): void;
+  /**
+   * Run the same test handler across many examples. Returns a function that
+   * takes a name (or template / name-builder) and the shared test body; each
+   * row in `table` becomes its own dataset example and experiment run.
+   *
+   * @param table - Rows of `{ input, expected?, metadata?, ... }` to fan out over.
+   * @returns A {@link PhoenixTestEach} that binds the name and shared handler.
+   *
+   * @example
+   * ```ts
+   * px.test.each([
+   *   { input: { a: 1, b: 2 }, expected: { sum: 3 } },
+   *   { input: { a: 2, b: 2 }, expected: { sum: 4 } },
+   * ])("adds %j", async ({ input, expected }) => {
+   *   // ...
+   * });
+   * ```
+   */
+  each<Input extends KVMap, Expected extends KVMap>(
+    table: TestEachRow<Input, Expected>[]
+  ): PhoenixTestEach<Input, Expected>;
+}
+
+/** The public testing surface returned by {@link createTestApi}. */
+export interface PhoenixTestApi {
+  /** Declare a Phoenix eval test suite. See {@link PhoenixDescribe}. */
+  describe: PhoenixDescribe;
+  /** Declare a Phoenix eval test case. See {@link PhoenixTest}. */
+  test: PhoenixTest;
+  /** Canonical alias for {@link PhoenixTestApi.test}. */
+  it: PhoenixTest;
+}
+
+/**
+ * Build the public `describe`/`test`/`it` API for a runner adapter.
+ *
+ * Both the jest and vitest entrypoints expose the identical surface; the only
+ * thing that differs between them is how the {@link RunnerHooks} are obtained
+ * (vitest imports them statically, jest resolves them lazily from globals).
+ * That difference is captured by `getHooks`, which is invoked once per
+ * declaration so adapters are free to resolve hooks lazily.
+ *
+ * The JSDoc that surfaces in editors lives on the {@link PhoenixDescribe} and
+ * {@link PhoenixTest} interfaces rather than the implementations below, so the
+ * docs survive the `export const { describe, test, it } = createTestApi(...)`
+ * destructuring in each adapter.
+ */
+export function createTestApi(getHooks: () => RunnerHooks): PhoenixTestApi {
+  const describe = ((
+    name: string,
+    fn: () => void,
+    config?: SuiteConfig
+  ): void => {
+    declareDescribe(getHooks(), name, fn, config ?? {});
+  }) as PhoenixDescribe;
+  describe.only = (name, fn, config) => {
+    declareDescribe(getHooks(), name, fn, config ?? {}, "only");
+  };
+  describe.skip = (name, fn, config) => {
+    declareDescribe(getHooks(), name, fn, config ?? {}, "skip");
+  };
+
+  const test = (<Input extends KVMap = KVMap, Expected extends KVMap = KVMap>(
+    name: string,
+    params: TestParams<Input, Expected>,
+    fn: TestFn<Input, Expected>,
+    timeout?: number
+  ): void => {
+    declareTest(getHooks(), name, params, fn, "default", timeout);
+  }) as PhoenixTest;
+  test.only = (name, params, fn, timeout) => {
+    declareTest(getHooks(), name, params, fn, "only", timeout);
+  };
+  test.skip = (name, params, fn, timeout) => {
+    declareTest(getHooks(), name, params, fn, "skip", timeout);
+  };
+  test.each = <Input extends KVMap, Expected extends KVMap>(
+    table: TestEachRow<Input, Expected>[]
+  ): PhoenixTestEach<Input, Expected> => {
+    return (name, fn, timeout) => {
+      table.forEach((row, i) => {
+        const testName =
+          typeof name === "function"
+            ? name(row, i)
+            : interpolateName(name, row, i);
+        declareTest(
+          getHooks(),
+          testName,
+          {
+            id: row.id,
+            input: row.input,
+            expected: resolveReference(row),
+            metadata: row.metadata,
+            splits: row.splits,
+            repetitions: row.repetitions,
+            dryRun: row.dryRun,
+          },
+          fn,
+          "default",
+          timeout
+        );
+      });
+    };
+  };
+
+  // `it` is the canonical alias for `test`.
+  const it = test;
+
+  return { describe, test, it };
+}
+
+/**
+ * Interpolate a `test.each` name template for a single row. Supports the
+ * common `%i`/`%s`/`%j` placeholders for surface parity with the underlying
+ * runners; when no placeholder is present the 1-based row index is appended.
+ */
+function interpolateName(
+  name: string,
+  row: TestEachRow,
+  index: number
+): string {
+  if (!name.includes("%")) {
+    return `${name} #${index + 1}`;
+  }
+  const replacements: Array<[RegExp, string]> = [
+    [/%i/g, String(index)],
+    [/%s/g, JSON.stringify(row.input)],
+    [/%j/g, JSON.stringify(row)],
+  ];
+  let out = name;
+  for (const [pattern, value] of replacements) {
+    out = out.replace(pattern, value);
+  }
+  return out;
+}

package/src/testing/helpers.ts

@@ -0,0 +1,251 @@
+import {
+  endTaskSpanForRun,
+  postAnnotation,
+  runEvaluatorWithTracing,
+} from "./phoenix-test-tracking";
+import { currentRun, type SuiteState } from "./state";
+import type {
+  Annotation,
+  EvaluationParams,
+  EvaluationResult,
+  EvaluationResultObject,
+  Evaluator,
+  KVMap,
+} from "./types";
+
+/**
+ * Log the output produced by the test for the current run.
+ *
+ * Calling this multiple times overwrites the previously recorded value.
+ * The argument can be any JSON-serializable value — typically an object
+ * matching the shape of the example's `expected` field.
+ */
+export function logOutput(output: unknown): void {
+  const run = currentRun();
+  if (!run) {
+    throw new Error(
+      "logOutput() must be called inside a Phoenix eval test body"
+    );
+  }
+  run.output = output;
+  run.outputSet = true;
+  endTaskSpanForRun(run);
+}
+
+/**
+ * Record an annotation on the current run.
+ *
+ * Annotations are collected during the test and posted to Phoenix as
+ * experiment evaluations after the test completes. The `name` is the
+ * Phoenix evaluation name; `score`, `label`, and `explanation` map to
+ * the standard Phoenix `EvaluationResult` fields.
+ *
+ * The annotation name `"pass"` is reserved — Phoenix eval tests always write
+ * a `pass` annotation derived from the test's assertion outcome, so a
+ * user-supplied annotation with that name would race / overwrite the
+ * built-in one. Such calls are silently ignored.
+ */
+export function logAnnotation(annotation: Annotation): void {
+  const run = currentRun();
+  if (!run) {
+    throw new Error(
+      "logAnnotation() must be called inside a Phoenix eval test body"
+    );
+  }
+  if (annotation.name === "pass") return;
+  run.annotations.push(annotation);
+}
+
+/**
+ * Run an evaluator object against the current test run and record the result.
+ *
+ * The evaluator may come from `@arizeai/phoenix-evals.createEvaluator`,
+ * `asExperimentEvaluator`, or any plain object with `{ name, evaluate }`.
+ * When `params` is omitted, the current test's `input`, recorded `output`,
+ * `expected`, `metadata`, and task `traceId` are supplied.
+ */
+export async function evaluate<
+  Params extends KVMap = EvaluationParams & KVMap,
+  Result = EvaluationResult,
+>(
+  evaluator: Evaluator<Params, Result>,
+  params?: Partial<Params> & KVMap
+): Promise<Result> {
+  const run = currentRun();
+  if (!run) {
+    return await evaluator.evaluate((params ?? {}) as Params);
+  }
+
+  if (!run.outputSet && !(params && "output" in params)) {
+    warnEvaluateBeforeOutput(run.suite, evaluator.name, run.testName);
+  }
+
+  const evaluatorParams = {
+    input: run.params.input,
+    // `run.output` is only ever set together with `outputSet`, so it is already
+    // `undefined` until a value is recorded.
+    output: run.output,
+    expected: run.params.expected,
+    metadata: run.params.metadata,
+    traceId: run.traceId ?? null,
+    ...(params ?? {}),
+  } as unknown as Params;
+
+  const { result, traceId } = await runEvaluatorWithTracing(
+    run.suite,
+    evaluator.name,
+    evaluatorParams,
+    (paramsToEvaluate) => evaluator.evaluate(paramsToEvaluate)
+  );
+  logAnnotation(
+    toAnnotation({
+      name: evaluator.name,
+      kind: evaluator.kind,
+      result,
+      traceId,
+    })
+  );
+  return result;
+}
+
+/**
+ * Trace an evaluator function so its execution shows up as a separate
+ * `EVALUATOR` span in Phoenix and any `{ name, score }`-shaped return
+ * value is automatically captured as an annotation on the current run.
+ *
+ * The annotation name defaults to the traced function's name, falling
+ * back to `"evaluator"`.
+ */
+export function traceEvaluator<EvaluatorParams extends KVMap, EvaluatorResult>(
+  fn: (params: EvaluatorParams) => EvaluatorResult | Promise<EvaluatorResult>,
+  options?: { name?: string }
+): (params: EvaluatorParams) => Promise<EvaluatorResult> {
+  const evaluatorName =
+    options?.name ?? (fn.name && fn.name !== "" ? fn.name : "evaluator");
+  return async (params: EvaluatorParams) => {
+    const run = currentRun();
+    if (!run) {
+      // outside a test context, just call the function plainly
+      return await fn(params);
+    }
+    const { result, traceId } = await runEvaluatorWithTracing(
+      run.suite,
+      evaluatorName,
+      params,
+      fn
+    );
+    if (isAnnotationShaped(result)) {
+      logAnnotation({ ...result, traceId });
+    }
+    return result;
+  };
+}
+
+/**
+ * Warn (at most once per suite) when an evaluator runs before any output was
+ * recorded and none was passed explicitly. Such an evaluator receives
+ * `output: undefined`, which silently scores against nothing — almost always a
+ * forgotten `logOutput()`. Harmless for evaluators that only read `input`.
+ */
+const warnedOutputSuites = new WeakSet<SuiteState>();
+function warnEvaluateBeforeOutput(
+  suite: SuiteState,
+  evaluatorName: string,
+  testName: string
+): void {
+  if (warnedOutputSuites.has(suite)) return;
+  warnedOutputSuites.add(suite);
+  // eslint-disable-next-line no-console
+  console.warn(
+    `[@arizeai/phoenix-client] evaluate("${evaluatorName}") ran before ` +
+      `logOutput() on test "${testName}", so the evaluator received ` +
+      `output=undefined. Call logOutput(...) first, or pass { output } ` +
+      `explicitly. (Ignore if this evaluator only needs input.)`
+  );
+}
+
+/**
+ * Normalize an evaluator's return value into an {@link Annotation}. The value
+ * is already typed as an {@link EvaluationResult}, so we only dispatch on its
+ * runtime shape: a string becomes a `label`, a number/boolean/null becomes a
+ * `score`, and an object contributes its `score`/`label`/`explanation`/
+ * `metadata` directly.
+ */
+function toAnnotation({
+  name,
+  kind,
+  result,
+  traceId,
+}: {
+  name: string;
+  kind?: Annotation["annotatorKind"];
+  result: unknown;
+  traceId?: string | null;
+}): Annotation {
+  const annotatorKind = kind ?? "CODE";
+  if (typeof result === "string") {
+    return { name, label: result, annotatorKind, traceId };
+  }
+  if (
+    typeof result === "number" ||
+    typeof result === "boolean" ||
+    result === null
+  ) {
+    return { name, score: result, annotatorKind, traceId };
+  }
+  if (typeof result === "object" && !Array.isArray(result)) {
+    const { score, label, explanation, metadata } =
+      result as EvaluationResultObject;
+    return {
+      name,
+      score,
+      label,
+      explanation,
+      metadata,
+      annotatorKind,
+      traceId,
+    };
+  }
+  return { name, annotatorKind, traceId };
+}
+
+function isAnnotationShaped(value: unknown): value is Annotation {
+  if (!value || typeof value !== "object") return false;
+  const v = value as { name?: unknown; score?: unknown };
+  if (typeof v.name !== "string") return false;
+  if (
+    v.score !== undefined &&
+    typeof v.score !== "number" &&
+    typeof v.score !== "boolean" &&
+    v.score !== null
+  ) {
+    return false;
+  }
+  return true;
+}
+
+/**
+ * Internal: persist all collected annotations for the run.
+ *
+ * Phoenix's `experiment_evaluations` endpoint is keyed by
+ * `(experiment_run_id, name)` so two annotations with the same name on
+ * the same run race each other. We collapse duplicates by name (last
+ * wins) up front, which makes the final state deterministic; the
+ * remaining writes target distinct names, so they post in parallel.
+ */
+export async function flushAnnotations(
+  runId: string | undefined,
+  annotations: Annotation[],
+  suite: SuiteState
+): Promise<void> {
+  if (!annotations.length) return;
+  const byName = new Map<string, Annotation>();
+  for (const annotation of annotations) {
+    byName.set(annotation.name, annotation);
+  }
+  await Promise.all(
+    Array.from(byName.values(), (annotation) =>
+      postAnnotation(suite, runId, annotation)
+    )
+  );
+}