@arizeai/phoenix-client 6.10.0 → 6.11.0

package/src/testing/runner.ts

@@ -0,0 +1,350 @@
+import {
+  createAcceptanceFailureError,
+  evaluateAcceptanceCriteria,
+} from "./acceptance";
+import { flushAnnotations } from "./helpers";
+import {
+  initializeSuite,
+  postExperimentRun,
+  resolveRepetitions,
+  runTaskWithTracing,
+  teardownSuite,
+} from "./phoenix-test-tracking";
+import { writeSuiteSummaryArtifact } from "./report-artifacts";
+import {
+  currentSuite,
+  popSuite,
+  pushSuite,
+  runStorage,
+  type RunState,
+  type SuiteState,
+  type TestResult,
+} from "./state";
+import {
+  type KVMap,
+  resolveReference,
+  type SuiteConfig,
+  type TestArgs,
+  type TestFn,
+  type TestParams,
+} from "./types";
+
+/**
+ * The minimum slice of a test runner (vitest or jest) the runner needs in
+ * order to declare suites, tests, and lifecycle hooks.
+ */
+export interface RunnerHooks {
+  describe: (name: string, fn: () => void) => void;
+  describeOnly?: (name: string, fn: () => void) => void;
+  describeSkip?: (name: string, fn: () => void) => void;
+  test: (
+    name: string,
+    fn: (...args: unknown[]) => unknown | Promise<unknown>,
+    timeout?: number
+  ) => void;
+  testOnly?: (
+    name: string,
+    fn: (...args: unknown[]) => unknown | Promise<unknown>,
+    timeout?: number
+  ) => void;
+  testSkip?: (
+    name: string,
+    fn: (...args: unknown[]) => unknown | Promise<unknown>,
+    timeout?: number
+  ) => void;
+  beforeAll: (fn: () => unknown | Promise<unknown>) => void;
+  afterAll: (fn: () => unknown | Promise<unknown>) => void;
+}
+
+/** Variants exposed by `test` (and `.skip`/`.only`). */
+type TestVariant = "default" | "only" | "skip";
+
+/** Globally accessible registry of all suites we created in this process. */
+const allSuites: SuiteState[] = [];
+
+/** Drive a `describe(name, fn, config?)` invocation. */
+export function declareDescribe(
+  hooks: RunnerHooks,
+  name: string,
+  fn: () => void,
+  config: SuiteConfig = {},
+  variant: TestVariant = "default"
+): void {
+  const describeFn =
+    variant === "only"
+      ? (hooks.describeOnly ?? hooks.describe)
+      : variant === "skip"
+        ? (hooks.describeSkip ?? hooks.describe)
+        : hooks.describe;
+
+  describeFn(name, () => {
+    const suite: SuiteState = {
+      name,
+      config,
+      registeredExamples: new Map(),
+      exampleIdsByTest: new Map(),
+      trackingDisabled: false,
+      results: [],
+      links: [],
+    };
+    pushSuite(suite);
+    try {
+      fn();
+    } finally {
+      popSuite();
+    }
+    allSuites.push(suite);
+
+    hooks.beforeAll(async () => {
+      await initializeSuite(suite);
+    });
+    hooks.afterAll(async () => {
+      let teardownError: unknown;
+      let acceptanceError: Error | undefined;
+      try {
+        await teardownSuite(suite);
+      } catch (err) {
+        teardownError = err;
+      }
+      try {
+        suite.acceptanceResults = evaluateAcceptanceCriteria({
+          criteria: suite.config.acceptanceCriteria,
+          results: suite.results,
+        });
+        acceptanceError = createAcceptanceFailureError(suite.acceptanceResults);
+      } finally {
+        writeSuiteSummaryArtifact(suite);
+      }
+      if (teardownError) {
+        throw teardownError;
+      }
+      if (acceptanceError) {
+        throw acceptanceError;
+      }
+    });
+  });
+}
+
+/** Drive a `test(name, params, fn)` invocation. */
+export function declareTest<Input extends KVMap, Expected extends KVMap>(
+  hooks: RunnerHooks,
+  name: string,
+  params: TestParams<Input, Expected>,
+  fn: TestFn<Input, Expected>,
+  variant: TestVariant = "default",
+  timeout?: number
+): void {
+  const suite = currentSuite();
+  if (!suite) {
+    throw new Error(
+      "Phoenix eval test() must be declared inside a describe() block"
+    );
+  }
+
+  // Collapse the `expected` / `reference` / `output` aliases into the single
+  // canonical `expected` slot up front, so every downstream consumer (dataset
+  // upload, evaluator params, test args) reads one field.
+  params = normalizeReferenceOutput(params);
+
+  // A per-test `dryRun` opts the case out of Phoenix entirely: no dataset
+  // example, no experiment run, no annotations — it runs as an ordinary
+  // local test. Suite-level dryRun (or PHOENIX_TEST_TRACING=false) is
+  // handled in `initializeSuite`, which never uploads anything anyway.
+  const isDryRun = !!params.dryRun;
+
+  // Avoid silently overwriting an example registration when the same test name
+  // is declared twice in the same suite.
+  const uniqueName = ensureUniqueName(suite, name);
+  // Skipped cases never run, and registering them would still upload their
+  // example to the tracked dataset via `initializeSuite()` — contradicting the
+  // skip contract and letting unfinished/flaky cases mutate the dataset. Dry-run
+  // cases opt out of Phoenix entirely. Neither should register an example.
+  if (!isDryRun && variant !== "skip") {
+    suite.registeredExamples.set(uniqueName, {
+      testName: uniqueName,
+      params: params as TestParams,
+    });
+  }
+
+  const repetitions = isDryRun
+    ? 1
+    : resolveRepetitions(params.repetitions, suite);
+  suite.maxRepetitions = Math.max(suite.maxRepetitions ?? 1, repetitions);
+
+  const testFn =
+    variant === "only"
+      ? (hooks.testOnly ?? hooks.test)
+      : variant === "skip"
+        ? (hooks.testSkip ?? hooks.test)
+        : hooks.test;
+
+  for (let rep = 1; rep <= repetitions; rep++) {
+    const runnerName =
+      repetitions > 1
+        ? `${uniqueName} [rep ${rep}/${repetitions}]`
+        : uniqueName;
+    testFn(
+      runnerName,
+      async () => {
+        await executeRun({
+          suite,
+          runnerName,
+          logicalName: uniqueName,
+          repetitionNumber: rep,
+          repetitions,
+          dryRun: isDryRun,
+          params: params as TestParams,
+          fn: fn as TestFn,
+        });
+      },
+      timeout
+    );
+  }
+}
+
+/** Execute a single (possibly-repeated) run of a declared test. */
+async function executeRun(opts: {
+  suite: SuiteState;
+  runnerName: string;
+  logicalName: string;
+  repetitionNumber: number;
+  repetitions: number;
+  dryRun: boolean;
+  params: TestParams;
+  fn: TestFn;
+}): Promise<void> {
+  const { suite, runnerName, logicalName, repetitionNumber, repetitions } =
+    opts;
+  const run: RunState = {
+    suite,
+    testName: runnerName,
+    logicalName,
+    repetitionNumber,
+    dryRun: opts.dryRun,
+    params: opts.params,
+    output: undefined,
+    outputSet: false,
+    annotations: [],
+    startTime: new Date(),
+    runMetadata: {},
+  };
+  const start = Date.now();
+  let status: "passed" | "failed" = "passed";
+  let thrown: unknown;
+  let testError: string | undefined;
+  await runStorage.run(run, async () => {
+    const taskOutcome = await runTaskWithTracing(
+      suite,
+      runnerName,
+      async () => {
+        const args: TestArgs = {
+          input: opts.params.input,
+          expected: opts.params.expected,
+          metadata: opts.params.metadata,
+        };
+        const result = await opts.fn(args);
+        // If the user returned a value and didn't call logOutput(),
+        // adopt the return value as the run's output.
+        if (
+          result !== undefined &&
+          !run.outputSet &&
+          isPlainObjectOrPrimitive(result)
+        ) {
+          run.output = result;
+          run.outputSet = true;
+        }
+        return result;
+      }
+    );
+    run.traceId = taskOutcome.traceId;
+    if ("error" in taskOutcome && taskOutcome.error) {
+      status = "failed";
+      testError = taskOutcome.error.message;
+      if (taskOutcome.isTaskError) {
+        run.error = taskOutcome.error.message;
+      }
+      thrown = taskOutcome.error;
+    }
+    run.endTime = new Date();
+
+    // The pass/fail annotation is recorded regardless of tracking mode
+    // so it shows up consistently in the reporter summary.
+    run.annotations.unshift({
+      name: "pass",
+      score: status === "passed",
+      annotatorKind: "CODE",
+    });
+
+    const runId = await postExperimentRun(suite, run);
+    run.runId = runId;
+    await flushAnnotations(runId, run.annotations, suite);
+  });
+
+  const result: TestResult = {
+    suiteName: suite.name,
+    testName: runnerName,
+    status,
+    output: run.outputSet ? run.output : undefined,
+    annotations: run.annotations,
+    error: testError,
+    durationMs: Date.now() - start,
+    repetitionNumber: repetitions > 1 ? repetitionNumber : undefined,
+    repetitions: repetitions > 1 ? repetitions : undefined,
+    dryRun: opts.dryRun || undefined,
+    traceId: run.traceId,
+    runId: run.runId,
+    exampleId: suite.exampleIdsByTest.get(logicalName)?.exampleId,
+  };
+  suite.results.push(result);
+
+  if (thrown) {
+    throw thrown;
+  }
+}
+
+/**
+ * Return a copy of `params` with the reference output collapsed onto the
+ * canonical `expected` key and the `reference` / `output` aliases dropped.
+ */
+function normalizeReferenceOutput<Input extends KVMap, Expected extends KVMap>(
+  params: TestParams<Input, Expected>
+): TestParams<Input, Expected> {
+  const expected = resolveReference(params);
+  const { reference: _reference, output: _output, ...rest } = params;
+  return { ...rest, expected };
+}
+
+function isPlainObjectOrPrimitive(value: unknown): boolean {
+  if (value === null) return true;
+  const type = typeof value;
+  return (
+    type === "string" ||
+    type === "number" ||
+    type === "boolean" ||
+    type === "object"
+  );
+}
+
+function ensureUniqueName(suite: SuiteState, name: string): string {
+  if (!suite.registeredExamples.has(name)) {
+    return name;
+  }
+  let i = 2;
+  while (suite.registeredExamples.has(`${name} (${i})`)) {
+    i++;
+  }
+  return `${name} (${i})`;
+}
+
+/** Suite registry exposed for reporters and the public summary helper. */
+export function getAllSuites(): readonly SuiteState[] {
+  return allSuites;
+}
+
+/**
+ * Reset the suite registry. Reporters call this at the start of every test
+ * run so module-cached state from previous watch invocations is released.
+ */
+export function clearAllSuites(): void {
+  allSuites.length = 0;
+}

package/src/testing/state.ts

@@ -0,0 +1,165 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import type {
+  GlobalTracerProviderRegistration,
+  NodeTracerProvider,
+  Tracer,
+} from "@arizeai/phoenix-otel";
+
+import type { PhoenixClient } from "../index";
+import type {
+  AcceptanceResult,
+  Annotation,
+  KVMap,
+  SuiteConfig,
+  TestParams,
+} from "./types";
+
+/**
+ * State for a single `describe()` block. Created during test collection
+ * and shared across all tests in the suite.
+ */
+export interface SuiteState {
+  /** The user-facing suite (and dataset/experiment) name. */
+  name: string;
+  config: SuiteConfig;
+  /**
+   * Map of test name → registered example metadata. Filled as tests are
+   * declared inside the describe callback. Keys are test names.
+   */
+  registeredExamples: Map<string, RegisteredExample>;
+  /**
+   * Map of test name → resolved server-side example id. Populated after the
+   * dataset is synced.
+   */
+  exampleIdsByTest: Map<string, { exampleId: string; nodeId: string }>;
+  /** Phoenix dataset id (after upload). */
+  datasetId?: string;
+  /** Phoenix experiment id (created at suite start). */
+  experimentId?: string;
+  /** Phoenix project name used for the experiment. */
+  projectName?: string;
+  /** Phoenix client used for this suite (resolved lazily). */
+  client?: PhoenixClient;
+  /** OpenInference tracer for task spans. */
+  tracer?: Tracer;
+  /** OpenInference tracer for evaluator spans. */
+  evaluatorTracer?: Tracer;
+  /** Tracer provider owned by this suite (cleaned up at suite end). */
+  tracerProvider?: NodeTracerProvider;
+  /** Global tracer registration owned by this suite (if any). */
+  globalRegistration?: GlobalTracerProviderRegistration | null;
+  /** Whether sync to Phoenix is disabled for this run. */
+  trackingDisabled: boolean;
+  /** Recorded test results for the reporter / summary. */
+  results: TestResult[];
+  /** Computed aggregate acceptance results for this suite. */
+  acceptanceResults?: AcceptanceResult[];
+  /** Links printed at the end of the suite (Phoenix dataset / experiment URLs). */
+  links: Array<{ label: string; url: string }>;
+  /** Reason the suite became inert (no tracking), if applicable. */
+  trackingDisabledReason?: string;
+  /** Setup error captured during suite initialization. */
+  setupError?: Error;
+  /** Count of best-effort POSTs (runs + annotations) that failed during upload. */
+  uploadFailureCount?: number;
+  /**
+   * Highest per-test repetition count seen in this suite. Used as the
+   * experiment's `repetitions` value (informational; runs are keyed by
+   * example + repetition number regardless).
+   */
+  maxRepetitions?: number;
+}
+
+/** Per-test registration captured when `test()` is declared. */
+export interface RegisteredExample {
+  testName: string;
+  params: TestParams;
+}
+
+/** Run-time state attached via AsyncLocalStorage to each running test. */
+export interface RunState {
+  suite: SuiteState;
+  /** Runner-facing test name (may include a `[rep i/N]` suffix). */
+  testName: string;
+  /** The logical test name; the key into `registeredExamples`. */
+  logicalName: string;
+  /** 1-based repetition index for this run. */
+  repetitionNumber: number;
+  /** When true, this run is local-only: no dataset example, no upload. */
+  dryRun: boolean;
+  params: TestParams;
+  output?: unknown;
+  outputSet: boolean;
+  annotations: Annotation[];
+  startTime: Date;
+  /** End time for the task span/run, which may be before the test body ends. */
+  taskEndTime?: Date;
+  endTime?: Date;
+  traceId?: string;
+  runId?: string;
+  error?: string;
+  /** Optional metadata accumulated during the test. */
+  runMetadata: KVMap;
+}
+
+/** A single test outcome surfaced to the reporter / summary printer. */
+export interface TestResult {
+  suiteName: string;
+  testName: string;
+  status: "passed" | "failed" | "skipped";
+  output?: unknown;
+  annotations: Annotation[];
+  error?: string;
+  durationMs: number;
+  /** 1-based repetition index, when the test ran more than once. */
+  repetitionNumber?: number;
+  /** Total repetitions configured for this test (≥ 1). */
+  repetitions?: number;
+  /** True when this test ran local-only (not uploaded to Phoenix). */
+  dryRun?: boolean;
+  /**
+   * OpenInference trace id for the task span, when the run was traced. Surfaced
+   * in the reporter's failure detail so an agent can pull the exact trace from
+   * Phoenix (e.g. `px`/GraphQL) to see what the task actually did.
+   */
+  traceId?: string;
+  /** Phoenix experiment run id, when the run was uploaded. */
+  runId?: string;
+  /** Phoenix dataset example id this run was scored against, when synced. */
+  exampleId?: string;
+}
+
+/**
+ * AsyncLocalStorage that lets `logOutput` / `logAnnotation` / `evaluate`
+ * reach the running test's state without threading it through arguments.
+ */
+export const runStorage = new AsyncLocalStorage<RunState>();
+
+/**
+ * Stack of currently active suites. Tests declared inside a `describe`
+ * callback consult the top of this stack to register themselves.
+ */
+const suiteStack: SuiteState[] = [];
+
+/** Push a suite onto the stack as we enter its `describe` callback. */
+export function pushSuite(suite: SuiteState): void {
+  suiteStack.push(suite);
+}
+
+/** Pop the current suite as we leave its `describe` callback. */
+export function popSuite(): SuiteState | undefined {
+  return suiteStack.pop();
+}
+
+/** The suite currently being declared, or `undefined` outside any `describe`. */
+export function currentSuite(): SuiteState | undefined {
+  return suiteStack[suiteStack.length - 1];
+}
+
+/**
+ * The state for the test currently executing, or `undefined` when called
+ * outside a Phoenix test body. Backed by {@link runStorage}.
+ */
+export function currentRun(): RunState | undefined {
+  return runStorage.getStore();
+}