@arizeai/phoenix-client 6.10.0 → 6.11.0

package/src/experiments/helpers/getExampleGlobalId.ts

@@ -0,0 +1,12 @@
+import type { ExampleWithId } from "../../types/datasets";
+
+/**
+ * The example's node GlobalID, which experiment runs record as their
+ * `dataset_example_id`. Servers that predate the `nodeId` field deliver the
+ * GlobalID in the `id` field instead.
+ */
+export function getExampleGlobalId(
+  example: Pick<ExampleWithId, "id"> & Partial<Pick<ExampleWithId, "nodeId">>
+): string {
+  return example.nodeId ?? example.id;
+}

package/src/experiments/resumeEvaluation.ts

@@ -29,6 +29,7 @@ import { ensureString } from "../utils/ensureString";
 import { toObjectHeaders } from "../utils/toObjectHeaders";
 import { getExperimentInfo } from "./getExperimentInfo.js";
 import { getExperimentEvaluators } from "./helpers";
+import { getExampleGlobalId } from "./helpers/getExampleGlobalId";
 import { logEvalResumeSummary, PROGRESS_PREFIX } from "./logging";
 import { cleanupOwnedTracerProvider } from "./tracing";
 
@@ -729,7 +730,7 @@ async function runSingleEvaluation({
         ...objectAsAttributes({
           experiment_id: experimentId,
           experiment_run_id: experimentRun.id,
-          dataset_example_id: datasetExample.nodeId,
+          dataset_example_id: getExampleGlobalId(datasetExample),
         }),
       });
 

package/src/experiments/resumeExperiment.ts

@@ -28,6 +28,7 @@ import { isHttpErrorWithStatus } from "../utils/isHttpError";
 import { toObjectHeaders } from "../utils/toObjectHeaders";
 import { getDatasetExperimentsUrl, getExperimentUrl } from "../utils/urlUtils";
 import { getExperimentInfo } from "./getExperimentInfo.js";
+import { getExampleGlobalId } from "./helpers/getExampleGlobalId";
 import {
   logExperimentResumeSummary,
   logLinks,
@@ -619,7 +620,7 @@ async function recordTaskResult({
         },
       },
       body: {
-        dataset_example_id: example.nodeId,
+        dataset_example_id: getExampleGlobalId(example),
         repetition_number: repetitionNumber,
         output: output as Record<string, unknown>,
         start_time: startTime.toISOString(),
@@ -695,7 +696,7 @@ async function runSingleTask({
         [SemanticConventions.INPUT_MIME_TYPE]: MimeType.JSON,
         ...objectAsAttributes({
           experiment_id: experimentId,
-          dataset_example_id: example.nodeId,
+          dataset_example_id: getExampleGlobalId(example),
           repetition_number: repetitionNumber,
         }),
       });

package/src/experiments/runExperiment.ts

@@ -47,6 +47,7 @@ import {
 } from "../utils/urlUtils";
 import { getExperimentInfo } from "./getExperimentInfo";
 import { getExperimentEvaluators } from "./helpers";
+import { getExampleGlobalId } from "./helpers/getExampleGlobalId";
 import {
   logEvalSummary,
   logLinks,
@@ -485,7 +486,7 @@ function runTaskWithExamples({
         id: localId(), // initialized with local id, will be replaced with server-assigned id when dry run is false
         traceId,
         experimentId,
-        datasetExampleId: example.id,
+        datasetExampleId: getExampleGlobalId(example),
         startTime: new Date(),
         endTime: new Date(), // will get replaced with actual end time
         output: null,
@@ -509,7 +510,7 @@ function runTaskWithExamples({
             },
           },
           body: {
-            dataset_example_id: example.nodeId,
+            dataset_example_id: getExampleGlobalId(example),
             output: thisRun.output,
             repetition_number: repetitionNumber,
             start_time: thisRun.startTime.toISOString(),
@@ -681,9 +682,11 @@ export async function evaluateExperiment({
     type EvaluationId = string;
     const evaluationRuns: Record<EvaluationId, ExperimentEvaluationRun> = {};
 
+    // Index examples by node GlobalID, matching how runs record
+    // datasetExampleId.
     const examplesById: Record<string, Example> = {};
     for (const example of dataset.examples) {
-      examplesById[example.id] = example;
+      examplesById[getExampleGlobalId(example)] = example;
     }
 
     const onEvaluationComplete = (run: ExperimentEvaluationRun) => {

package/src/jest/index.ts

@@ -0,0 +1,124 @@
+import { createTestApi } from "../testing/define-api";
+import type { RunnerHooks } from "../testing/runner";
+
+export type {
+  PhoenixDescribe,
+  PhoenixTest,
+  PhoenixTestApi,
+  PhoenixTestEach,
+} from "../testing/define-api";
+
+export type {
+  AcceptanceCriterion,
+  AcceptanceMetric,
+  AcceptanceResult,
+  Annotation,
+  AnnotatorKind,
+  EvaluationParams,
+  EvaluationResult,
+  Evaluator,
+  EvaluatorResult,
+  KVMap,
+  ReferenceOutput,
+  SuiteConfig,
+  TestArgs,
+  TestConfig,
+  TestEachRow,
+  TestFn,
+  TestParams,
+  TestParamsBase,
+} from "../testing/types";
+
+export {
+  evaluate,
+  logAnnotation,
+  logOutput,
+  traceEvaluator,
+} from "../testing/helpers";
+
+/**
+ * Resolve jest globals lazily. Jest injects `describe`, `test`, `it`,
+ * `beforeAll`, `afterAll` onto the global object during test runs; we read
+ * them off `globalThis` so importing this module outside of a jest run
+ * doesn't error.
+ */
+function getJestGlobals(): {
+  describe: JestDescribe;
+  test: JestTest;
+  beforeAll: (fn: () => unknown | Promise<unknown>) => void;
+  afterAll: (fn: () => unknown | Promise<unknown>) => void;
+} {
+  const g = globalThis as unknown as {
+    describe?: JestDescribe;
+    test?: JestTest;
+    it?: JestTest;
+    beforeAll?: (fn: () => unknown | Promise<unknown>) => void;
+    afterAll?: (fn: () => unknown | Promise<unknown>) => void;
+  };
+  if (!g.describe || !g.beforeAll || !g.afterAll) {
+    throw new Error(
+      "@arizeai/phoenix-client/jest could not find Jest globals. Make sure this module " +
+        "is imported from inside a Jest test file."
+    );
+  }
+  const test = g.test ?? g.it;
+  if (!test) {
+    throw new Error(
+      "@arizeai/phoenix-client/jest could not find Jest's test/it."
+    );
+  }
+  return {
+    describe: g.describe,
+    test,
+    beforeAll: g.beforeAll,
+    afterAll: g.afterAll,
+  };
+}
+
+interface JestDescribe {
+  (name: string, fn: () => void): void;
+  only: (name: string, fn: () => void) => void;
+  skip: (name: string, fn: () => void) => void;
+}
+
+interface JestTest {
+  (
+    name: string,
+    fn: (...args: unknown[]) => unknown | Promise<unknown>,
+    timeout?: number
+  ): void;
+  only: (
+    name: string,
+    fn: (...args: unknown[]) => unknown | Promise<unknown>,
+    timeout?: number
+  ) => void;
+  skip: (
+    name: string,
+    fn: (...args: unknown[]) => unknown | Promise<unknown>,
+    timeout?: number
+  ) => void;
+}
+
+/**
+ * Build the runner hooks from the (lazily-resolved) jest globals. Memoized so
+ * repeated declarations don't re-read `globalThis` on every call; jest injects
+ * its globals once per worker before any test file is evaluated.
+ */
+let cachedHooks: RunnerHooks | undefined;
+function getHooks(): RunnerHooks {
+  if (cachedHooks) return cachedHooks;
+  const g = getJestGlobals();
+  cachedHooks = {
+    describe: (name, fn) => g.describe(name, fn),
+    describeOnly: (name, fn) => g.describe.only(name, fn),
+    describeSkip: (name, fn) => g.describe.skip(name, fn),
+    test: (name, fn, timeout) => g.test(name, fn, timeout),
+    testOnly: (name, fn, timeout) => g.test.only(name, fn, timeout),
+    testSkip: (name, fn, timeout) => g.test.skip(name, fn, timeout),
+    beforeAll: (fn) => g.beforeAll(fn),
+    afterAll: (fn) => g.afterAll(fn),
+  };
+  return cachedHooks;
+}
+
+export const { describe, test, it } = createTestApi(getHooks);

package/src/jest/reporter.ts

@@ -0,0 +1,22 @@
+import { SuiteSummaryReportRun } from "../testing/report-run";
+
+/**
+ * Jest reporter for `@arizeai/phoenix-client/jest`.
+ *
+ * Like the Vitest reporter, this does not replace Jest's default output. It
+ * appends a Phoenix summary block at the end of the run.
+ */
+export default class PhoenixJestReporter {
+  private readonly report = new SuiteSummaryReportRun();
+
+  // jest passes globalConfig + reporterOptions; we ignore both
+  constructor(_globalConfig?: unknown, _reporterOptions?: unknown) {}
+
+  onRunStart(): void {
+    this.report.begin();
+  }
+
+  onRunComplete(): void {
+    this.report.finish();
+  }
+}

package/src/prompts/sdks/toAI.ts

@@ -40,7 +40,8 @@ The tool choice strategy. Default: 'auto'.
   toolChoice?: ToolChoice<ToolSet>;
 };
 
-export type ToAIParams<V extends Variables> = toSDKParamsBase<V>;
+export type ToAIParams<PromptVariables extends Variables> =
+  toSDKParamsBase<PromptVariables>;
 
 /**
  * Converts a Phoenix prompt to Vercel AI sdk params.
@@ -48,10 +49,10 @@ export type ToAIParams<V extends Variables> = toSDKParamsBase<V>;
  * - note: To use response format, you must pass `prompt.response_format.json_schema.schema` to generateObject or streamObject
  *   via `jsonSchema()`, through the `schema` argument.
  */
-export const toAI = <V extends Variables>({
+export const toAI = <PromptVariables extends Variables>({
   prompt,
   variables,
-}: ToAIParams<V>): PartialAIParams | null => {
+}: ToAIParams<PromptVariables>): PartialAIParams | null => {
   // eslint-disable-next-line no-console
   console.warn(
     "Prompt invocation parameters not currently supported in AI SDK, falling back to default invocation parameters"

package/src/prompts/sdks/toAnthropic.ts

@@ -14,15 +14,16 @@ import type { toSDKParamsBase, Variables } from "./types";
 // We must re-export these types so that they are included in the phoenix-client distribution
 export type { MessageCreateParams };
 
-export type ToAnthropicParams<V extends Variables> = toSDKParamsBase<V>;
+export type ToAnthropicParams<PromptVariables extends Variables> =
+  toSDKParamsBase<PromptVariables>;
 
 /**
  * Convert a Phoenix prompt to Anthropic client sdk's message create parameters
  */
-export const toAnthropic = <V extends Variables = Variables>({
+export const toAnthropic = <PromptVariables extends Variables = Variables>({
   prompt,
   variables,
-}: ToAnthropicParams<V>): MessageCreateParams | null => {
+}: ToAnthropicParams<PromptVariables>): MessageCreateParams | null => {
   try {
     let invocationParameters: { max_tokens: number } | undefined;
     if (prompt.invocation_parameters.type === "anthropic") {

package/src/prompts/sdks/toOpenAI.ts

@@ -23,17 +23,18 @@ export type {
   ResponseFormatJSONSchema,
 };
 
-export type ToOpenAIParams<V extends Variables> = toSDKParamsBase<V>;
+export type ToOpenAIParams<PromptVariables extends Variables> =
+  toSDKParamsBase<PromptVariables>;
 
 /**
  * Convert a Phoenix prompt to OpenAI client sdk's chat completion parameters
  *
  * @returns The converted chat completion parameters
  */
-export const toOpenAI = <V extends Variables = Variables>({
+export const toOpenAI = <PromptVariables extends Variables = Variables>({
   prompt,
   variables,
-}: ToOpenAIParams<V>): ChatCompletionCreateParams | null => {
+}: ToOpenAIParams<PromptVariables>): ChatCompletionCreateParams | null => {
   try {
     let invocationParameters: Partial<ChatCompletionCreateParams>;
     switch (prompt.invocation_parameters.type) {

package/src/prompts/sdks/toSDK.ts

@@ -15,10 +15,10 @@ import type { SupportedSDK, toSDKParamsBase, Variables } from "./types";
  * toOpenAI(params)
  * ```
  */
-export type SDKParams<T extends SupportedSDK> = Parameters<
-  (typeof PROVIDER_TO_SDK)[T]
+export type SDKParams<SDK extends SupportedSDK> = Parameters<
+  (typeof PROVIDER_TO_SDK)[SDK]
 >[number] extends toSDKParamsBase
-  ? Parameters<(typeof PROVIDER_TO_SDK)[T]>[number]
+  ? Parameters<(typeof PROVIDER_TO_SDK)[SDK]>[number]
   : never;
 
 /**
@@ -33,7 +33,7 @@ export const PROVIDER_TO_SDK = {
 /**
  * Get the conversion function for a specific SDK name
  */
-const getTargetSDK = <T extends SupportedSDK>(sdk: T) => {
+const getTargetSDK = <SDK extends SupportedSDK>(sdk: SDK) => {
   switch (sdk) {
     case "openai":
       return PROVIDER_TO_SDK.openai;
@@ -50,19 +50,19 @@ const getTargetSDK = <T extends SupportedSDK>(sdk: T) => {
  * Parameters specific to the toSDK function
  */
 export type ToSDKParams<
-  T extends SupportedSDK,
-  V extends Variables = Variables,
+  SDK extends SupportedSDK,
+  PromptVariables extends Variables = Variables,
 > = {
   /**
    * String representing the SDK to convert to
    */
-  sdk: T;
+  sdk: SDK;
   /**
    * Optional variables to format the prompt with
    * Keys are the variable names, values are the variable values
    * The variable format is determined via prompt.template_format
    */
-  variables?: V;
+  variables?: PromptVariables;
 };
 
 /**
@@ -86,11 +86,16 @@ export type ToSDKParams<
  * @param params - The parameters to convert a prompt to an SDK's parameters
  * @returns The SDK's parameters
  */
-export const toSDK = <T extends SupportedSDK, V extends Variables = Variables>({
+export const toSDK = <
+  SDK extends SupportedSDK,
+  PromptVariables extends Variables = Variables,
+>({
   sdk: _sdk,
   ...rest
-}: ToSDKParams<T, V> & SDKParams<T>) => {
+}: ToSDKParams<SDK, PromptVariables> & SDKParams<SDK>) => {
   const sdk = getTargetSDK(_sdk);
   invariant(sdk, `No SDK found for provider ${_sdk}`);
-  return sdk<V>(rest) as ReturnType<(typeof PROVIDER_TO_SDK)[T]>;
+  return sdk<PromptVariables>(rest) as ReturnType<
+    (typeof PROVIDER_TO_SDK)[SDK]
+  >;
 };

package/src/prompts/sdks/types.ts

@@ -11,7 +11,7 @@ export type Variables = Record<string, string | { toString: () => string }>;
 /**
  * Base parameters for an SDK conversion function
  */
-export type toSDKParamsBase<V extends Variables = Variables> = {
+export type toSDKParamsBase<PromptVariables extends Variables = Variables> = {
   /**
    * The Phoenix prompt to convert
    */
@@ -19,7 +19,7 @@ export type toSDKParamsBase<V extends Variables = Variables> = {
   /**
    * The variables to use in the prompt
    */
-  variables?: V;
+  variables?: PromptVariables;
 };
 
 /**

package/src/testing/acceptance.ts

@@ -0,0 +1,190 @@
+import type { TestResult } from "./state";
+import type {
+  AcceptanceCriterion,
+  AcceptanceResult,
+  Annotation,
+  OptimizationDirection,
+} from "./types";
+
+/**
+ * Evaluate all configured aggregate acceptance rules against completed runs.
+ * @param params - Evaluation parameters.
+ * @param params.criteria - Aggregate rules configured on the suite.
+ * @param params.results - Completed test results in the suite.
+ */
+export function evaluateAcceptanceCriteria({
+  criteria,
+  results,
+}: {
+  criteria: readonly AcceptanceCriterion[] | undefined;
+  results: readonly TestResult[];
+}): AcceptanceResult[] {
+  if (!criteria || criteria.length === 0) {
+    return [];
+  }
+  return criteria.map((criterion) =>
+    evaluateAcceptanceCriterion({ criterion, results })
+  );
+}
+
+/**
+ * Build one error containing every failed aggregate criterion.
+ * @param results - Computed acceptance results.
+ */
+export function createAcceptanceFailureError(
+  results: readonly AcceptanceResult[]
+): Error | undefined {
+  const failedResults = results.filter((result) => !result.passed);
+  if (failedResults.length === 0) {
+    return undefined;
+  }
+  return new Error(
+    [
+      "Acceptance criteria failed:",
+      ...failedResults.map((result) => `  ${formatAcceptanceResult(result)}`),
+    ].join("\n")
+  );
+}
+
+/** Format an acceptance result for reporters and thrown errors. */
+export function formatAcceptanceResult(result: AcceptanceResult): string {
+  const status = result.passed ? "PASS" : "FAIL";
+  const value = result.value === null ? "n/a" : result.value.toFixed(3);
+  const sampleLabel = result.sampleCount === 1 ? "sample" : "samples";
+  let requirement: string;
+  if (result.metric === "average") {
+    const cmp = (result.direction ?? "maximize") === "minimize" ? "<=" : ">=";
+    requirement = `mean ${cmp} ${result.threshold.toFixed(3)}`;
+  } else {
+    requirement = `pass rate >= ${result.minPassRate.toFixed(3)}`;
+  }
+  const reason = result.failureReason ? ` - ${result.failureReason}` : "";
+  return `${status} ${result.annotationName} ${result.metric} ${value} (need ${requirement}; ${result.sampleCount} ${sampleLabel})${reason}`;
+}
+
+function evaluateAcceptanceCriterion({
+  criterion,
+  results,
+}: {
+  criterion: AcceptanceCriterion;
+  results: readonly TestResult[];
+}): AcceptanceResult {
+  const annotations = collectAnnotations({ criterion, results });
+
+  if (criterion.metric === "average") {
+    // Only numeric / boolean scores can be averaged.
+    const scores = annotations
+      .map((annotation) => annotation.score)
+      .filter(isValidScore);
+    if (scores.length === 0) {
+      return {
+        ...criterion,
+        value: null,
+        sampleCount: 0,
+        passed: false,
+        failureReason: "no numeric or boolean scores found",
+      };
+    }
+    const direction = criterion.direction ?? "maximize";
+    const value = calculateAverage(scores);
+    return {
+      ...criterion,
+      value,
+      sampleCount: scores.length,
+      passed: meetsBar(value, criterion.threshold, direction),
+    };
+  }
+
+  // passRate: each run passes when `passFn` returns true for its annotation;
+  // the suite passes when the fraction of passing runs is at least
+  // `minPassRate`. The reported value is that fraction.
+  if (annotations.length === 0) {
+    return {
+      ...criterion,
+      value: null,
+      sampleCount: 0,
+      passed: false,
+      failureReason: "no matching annotations found",
+    };
+  }
+  const passed = annotations.filter((annotation) =>
+    criterion.passFn(annotation)
+  ).length;
+  const value = passed / annotations.length;
+  return {
+    ...criterion,
+    value,
+    sampleCount: annotations.length,
+    passed: value >= criterion.minPassRate,
+  };
+}
+
+/** Whether `value` clears `bar` in the given optimization direction. */
+function meetsBar(
+  value: number,
+  bar: number,
+  direction: OptimizationDirection
+): boolean {
+  return direction === "minimize" ? value <= bar : value >= bar;
+}
+
+/**
+ * The last annotation matching `annotationName` from each non-skipped run that
+ * logged it. One entry per run; runs that never logged the annotation are
+ * omitted.
+ */
+function collectAnnotations({
+  criterion,
+  results,
+}: {
+  criterion: AcceptanceCriterion;
+  results: readonly TestResult[];
+}): Annotation[] {
+  return results
+    .filter((result) => result.status !== "skipped")
+    .map((result) =>
+      findLastAnnotation({
+        annotations: result.annotations,
+        annotationName: criterion.annotationName,
+      })
+    )
+    .filter((annotation): annotation is Annotation => annotation !== undefined);
+}
+
+function findLastAnnotation({
+  annotations,
+  annotationName,
+}: {
+  annotations: readonly Annotation[];
+  annotationName: string;
+}): Annotation | undefined {
+  for (
+    let annotationIndex = annotations.length - 1;
+    annotationIndex >= 0;
+    annotationIndex--
+  ) {
+    const annotation = annotations[annotationIndex];
+    if (annotation?.name === annotationName) {
+      return annotation;
+    }
+  }
+  return undefined;
+}
+
+function isValidScore(score: Annotation["score"]): score is number | boolean {
+  return (
+    typeof score === "boolean" ||
+    (typeof score === "number" && Number.isFinite(score))
+  );
+}
+
+function calculateAverage(scores: readonly (number | boolean)[]): number {
+  const total = scores
+    .map(scoreToNumber)
+    .reduce((sum, score) => sum + score, 0);
+  return total / scores.length;
+}
+
+function scoreToNumber(score: number | boolean): number {
+  return typeof score === "boolean" ? (score ? 1 : 0) : score;
+}