@arizeai/phoenix-client 6.10.1 → 6.11.1

package/dist/esm/testing/helpers.js

@@ -0,0 +1,179 @@
+import { endTaskSpanForRun, postAnnotation, runEvaluatorWithTracing, } from "./phoenix-test-tracking.js";
+import { currentRun } from "./state.js";
+/**
+ * Log the output produced by the test for the current run.
+ *
+ * Calling this multiple times overwrites the previously recorded value.
+ * The argument can be any JSON-serializable value — typically an object
+ * matching the shape of the example's `expected` field.
+ */
+export function logOutput(output) {
+    const run = currentRun();
+    if (!run) {
+        throw new Error("logOutput() must be called inside a Phoenix eval test body");
+    }
+    run.output = output;
+    run.outputSet = true;
+    endTaskSpanForRun(run);
+}
+/**
+ * Record an annotation on the current run.
+ *
+ * Annotations are collected during the test and posted to Phoenix as
+ * experiment evaluations after the test completes. The `name` is the
+ * Phoenix evaluation name; `score`, `label`, and `explanation` map to
+ * the standard Phoenix `EvaluationResult` fields.
+ *
+ * The annotation name `"pass"` is reserved — Phoenix eval tests always write
+ * a `pass` annotation derived from the test's assertion outcome, so a
+ * user-supplied annotation with that name would race / overwrite the
+ * built-in one. Such calls are silently ignored.
+ */
+export function logAnnotation(annotation) {
+    const run = currentRun();
+    if (!run) {
+        throw new Error("logAnnotation() must be called inside a Phoenix eval test body");
+    }
+    if (annotation.name === "pass")
+        return;
+    run.annotations.push(annotation);
+}
+/**
+ * Run an evaluator object against the current test run and record the result.
+ *
+ * The evaluator may come from `@arizeai/phoenix-evals.createEvaluator`,
+ * `asExperimentEvaluator`, or any plain object with `{ name, evaluate }`.
+ * When `params` is omitted, the current test's `input`, recorded `output`,
+ * `expected`, `metadata`, and task `traceId` are supplied.
+ */
+export async function evaluate(evaluator, params) {
+    const run = currentRun();
+    if (!run) {
+        return await evaluator.evaluate((params ?? {}));
+    }
+    if (!run.outputSet && !(params && "output" in params)) {
+        warnEvaluateBeforeOutput(run.suite, evaluator.name, run.testName);
+    }
+    const evaluatorParams = {
+        input: run.params.input,
+        // `run.output` is only ever set together with `outputSet`, so it is already
+        // `undefined` until a value is recorded.
+        output: run.output,
+        expected: run.params.expected,
+        metadata: run.params.metadata,
+        traceId: run.traceId ?? null,
+        ...(params ?? {}),
+    };
+    const { result, traceId } = await runEvaluatorWithTracing(run.suite, evaluator.name, evaluatorParams, (paramsToEvaluate) => evaluator.evaluate(paramsToEvaluate));
+    logAnnotation(toAnnotation({
+        name: evaluator.name,
+        kind: evaluator.kind,
+        result,
+        traceId,
+    }));
+    return result;
+}
+/**
+ * Trace an evaluator function so its execution shows up as a separate
+ * `EVALUATOR` span in Phoenix and any `{ name, score }`-shaped return
+ * value is automatically captured as an annotation on the current run.
+ *
+ * The annotation name defaults to the traced function's name, falling
+ * back to `"evaluator"`.
+ */
+export function traceEvaluator(fn, options) {
+    const evaluatorName = options?.name ?? (fn.name && fn.name !== "" ? fn.name : "evaluator");
+    return async (params) => {
+        const run = currentRun();
+        if (!run) {
+            // outside a test context, just call the function plainly
+            return await fn(params);
+        }
+        const { result, traceId } = await runEvaluatorWithTracing(run.suite, evaluatorName, params, fn);
+        if (isAnnotationShaped(result)) {
+            logAnnotation({ ...result, traceId });
+        }
+        return result;
+    };
+}
+/**
+ * Warn (at most once per suite) when an evaluator runs before any output was
+ * recorded and none was passed explicitly. Such an evaluator receives
+ * `output: undefined`, which silently scores against nothing — almost always a
+ * forgotten `logOutput()`. Harmless for evaluators that only read `input`.
+ */
+const warnedOutputSuites = new WeakSet();
+function warnEvaluateBeforeOutput(suite, evaluatorName, testName) {
+    if (warnedOutputSuites.has(suite))
+        return;
+    warnedOutputSuites.add(suite);
+    // eslint-disable-next-line no-console
+    console.warn(`[@arizeai/phoenix-client] evaluate("${evaluatorName}") ran before ` +
+        `logOutput() on test "${testName}", so the evaluator received ` +
+        `output=undefined. Call logOutput(...) first, or pass { output } ` +
+        `explicitly. (Ignore if this evaluator only needs input.)`);
+}
+/**
+ * Normalize an evaluator's return value into an {@link Annotation}. The value
+ * is already typed as an {@link EvaluationResult}, so we only dispatch on its
+ * runtime shape: a string becomes a `label`, a number/boolean/null becomes a
+ * `score`, and an object contributes its `score`/`label`/`explanation`/
+ * `metadata` directly.
+ */
+function toAnnotation({ name, kind, result, traceId, }) {
+    const annotatorKind = kind ?? "CODE";
+    if (typeof result === "string") {
+        return { name, label: result, annotatorKind, traceId };
+    }
+    if (typeof result === "number" ||
+        typeof result === "boolean" ||
+        result === null) {
+        return { name, score: result, annotatorKind, traceId };
+    }
+    if (typeof result === "object" && !Array.isArray(result)) {
+        const { score, label, explanation, metadata } = result;
+        return {
+            name,
+            score,
+            label,
+            explanation,
+            metadata,
+            annotatorKind,
+            traceId,
+        };
+    }
+    return { name, annotatorKind, traceId };
+}
+function isAnnotationShaped(value) {
+    if (!value || typeof value !== "object")
+        return false;
+    const v = value;
+    if (typeof v.name !== "string")
+        return false;
+    if (v.score !== undefined &&
+        typeof v.score !== "number" &&
+        typeof v.score !== "boolean" &&
+        v.score !== null) {
+        return false;
+    }
+    return true;
+}
+/**
+ * Internal: persist all collected annotations for the run.
+ *
+ * Phoenix's `experiment_evaluations` endpoint is keyed by
+ * `(experiment_run_id, name)` so two annotations with the same name on
+ * the same run race each other. We collapse duplicates by name (last
+ * wins) up front, which makes the final state deterministic; the
+ * remaining writes target distinct names, so they post in parallel.
+ */
+export async function flushAnnotations(runId, annotations, suite) {
+    if (!annotations.length)
+        return;
+    const byName = new Map();
+    for (const annotation of annotations) {
+        byName.set(annotation.name, annotation);
+    }
+    await Promise.all(Array.from(byName.values(), (annotation) => postAnnotation(suite, runId, annotation)));
+}
+//# sourceMappingURL=helpers.js.map

package/dist/esm/testing/helpers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"helpers.js","sourceRoot":"","sources":["../../../src/testing/helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,cAAc,EACd,uBAAuB,GACxB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,UAAU,EAAmB,MAAM,SAAS,CAAC;AAUtD;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CAAC,MAAe;IACvC,MAAM,GAAG,GAAG,UAAU,EAAE,CAAC;IACzB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CACb,4DAA4D,CAC7D,CAAC;IACJ,CAAC;IACD,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC;IACpB,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC;IACrB,iBAAiB,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,aAAa,CAAC,UAAsB;IAClD,MAAM,GAAG,GAAG,UAAU,EAAE,CAAC;IACzB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CACb,gEAAgE,CACjE,CAAC;IACJ,CAAC;IACD,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO;IACvC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;AACnC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAI5B,SAAoC,EACpC,MAAgC;IAEhC,MAAM,GAAG,GAAG,UAAU,EAAE,CAAC;IACzB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,MAAM,SAAS,CAAC,QAAQ,CAAC,CAAC,MAAM,IAAI,EAAE,CAAW,CAAC,CAAC;IAC5D,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ,IAAI,MAAM,CAAC,EAAE,CAAC;QACtD,wBAAwB,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,eAAe,GAAG;QACtB,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK;QACvB,4EAA4E;QAC5E,yCAAyC;QACzC,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ;QAC7B,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ;QAC7B,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,IAAI;QAC5B,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;KACG,CAAC;IAEvB,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,uBAAuB,CACvD,GAAG,CAAC,KAAK,EACT,SAAS,CAAC,IAAI,EACd,eAAe,EACf,CAAC,gBAAgB,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAC3D,CAAC;IACF,aAAa,CACX,YAAY,CAAC;QACX,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,MAAM;QACN,OAAO;KACR,CAAC,CACH,CAAC;IACF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAC5B,EAA2E,EAC3E,OAA2B;IAE3B,MAAM,aAAa,GACjB,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IACvE,OAAO,KAAK,EAAE,MAAuB,EAAE,EAAE;QACvC,MAAM,GAAG,GAAG,UAAU,EAAE,CAAC;QACzB,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,yDAAyD;YACzD,OAAO,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,uBAAuB,CACvD,GAAG,CAAC,KAAK,EACT,aAAa,EACb,MAAM,EACN,EAAE,CACH,CAAC;QACF,IAAI,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/B,aAAa,CAAC,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QACxC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,kBAAkB,GAAG,IAAI,OAAO,EAAc,CAAC;AACrD,SAAS,wBAAwB,CAC/B,KAAiB,EACjB,aAAqB,EACrB,QAAgB;IAEhB,IAAI,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC;QAAE,OAAO;IAC1C,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC9B,sCAAsC;IACtC,OAAO,CAAC,IAAI,CACV,uCAAuC,aAAa,gBAAgB;QAClE,wBAAwB,QAAQ,+BAA+B;QAC/D,kEAAkE;QAClE,0DAA0D,CAC7D,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,SAAS,YAAY,CAAC,EACpB,IAAI,EACJ,IAAI,EACJ,MAAM,EACN,OAAO,GAMR;IACC,MAAM,aAAa,GAAG,IAAI,IAAI,MAAM,CAAC;IACrC,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;IACzD,CAAC;IACD,IACE,OAAO,MAAM,KAAK,QAAQ;QAC1B,OAAO,MAAM,KAAK,SAAS;QAC3B,MAAM,KAAK,IAAI,EACf,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;IACzD,CAAC;IACD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACzD,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAC3C,MAAgC,CAAC;QACnC,OAAO;YACL,IAAI;YACJ,KAAK;YACL,KAAK;YACL,WAAW;YACX,QAAQ;YACR,aAAa;YACb,OAAO;SACR,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACtD,MAAM,CAAC,GAAG,KAA4C,CAAC;IACvD,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC7C,IACE,CAAC,CAAC,KAAK,KAAK,SAAS;QACrB,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ;QAC3B,OAAO,CAAC,CAAC,KAAK,KAAK,SAAS;QAC5B,CAAC,CAAC,KAAK,KAAK,IAAI,EAChB,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,KAAyB,EACzB,WAAyB,EACzB,KAAiB;IAEjB,IAAI,CAAC,WAAW,CAAC,MAAM;QAAE,OAAO;IAChC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,EAAE,EAAE,CACzC,cAAc,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,CAAC,CACzC,CACF,CAAC;AACJ,CAAC"}

package/dist/esm/testing/phoenix-test-tracking.d.ts

@@ -0,0 +1,68 @@
+import { type GlobalTracerProviderRegistration, type NodeTracerProvider } from "@arizeai/phoenix-otel";
+import { type RunState, type SuiteState } from "./state.js";
+import type { Annotation, KVMap } from "./types.js";
+/**
+ * Decide whether tests should sync to Phoenix.
+ *
+ * Tracing is enabled by default. It can be disabled globally by setting
+ * `PHOENIX_TEST_TRACKING=false`, or per suite via `SuiteConfig.dryRun`.
+ */
+export declare function isTrackingEnabled(suite?: SuiteState): {
+    enabled: boolean;
+    reason?: string;
+};
+/**
+ * Resolve the repetition count for a test: per-test value, else suite-level,
+ * else the `PHOENIX_TEST_REPETITIONS` env var, else `1`. Non-positive or
+ * non-finite values fall back to `1`.
+ */
+export declare function resolveRepetitions(perTest: number | undefined, suite: SuiteState): number;
+/**
+ * Initialize the suite: upload the dataset, create the experiment, and
+ * register the OpenInference tracer.
+ *
+ * If tracing is disabled (no Phoenix env vars, or PHOENIX_TEST_TRACKING=false),
+ * this populates a no-op tracer and exits without making any network calls.
+ */
+export declare function initializeSuite(suite: SuiteState): Promise<void>;
+/**
+ * Wrap the user's test body in an OpenInference task span and return the
+ * trace id so we can submit it with the experiment run.
+ */
+export declare function runTaskWithTracing<Result>(suite: SuiteState, testName: string, fn: () => Promise<Result>): Promise<{
+    traceId: string;
+    result: Result;
+} | {
+    traceId: string;
+    error: Error;
+    isTaskError: boolean;
+}>;
+/**
+ * End the current run's task span. `logOutput()` calls this immediately so
+ * evaluator work that follows is not included in the task span duration.
+ */
+export declare function endTaskSpanForRun(run: RunState): void;
+/**
+ * POST a single experiment run for one test case to Phoenix.
+ *
+ * Best-effort: failures are captured and surfaced via the test reporter but
+ * do not fail the test itself.
+ */
+export declare function postExperimentRun(suite: SuiteState, run: RunState): Promise<string | undefined>;
+/**
+ * POST one annotation (an "experiment_evaluation") for a run.
+ */
+export declare function postAnnotation(suite: SuiteState, runId: string | undefined, annotation: Annotation): Promise<void>;
+/** Run an evaluator in an OpenInference evaluator span. */
+export declare function runEvaluatorWithTracing<EvaluatorParams extends KVMap, EvaluatorResult>(suite: SuiteState, name: string, params: EvaluatorParams, fn: (params: EvaluatorParams) => EvaluatorResult | Promise<EvaluatorResult>): Promise<{
+    result: EvaluatorResult;
+    traceId: string | null;
+}>;
+/**
+ * Tear down the tracer provider created for the suite, flushing any pending
+ * spans before the process exits.
+ */
+export declare function teardownSuite(suite: SuiteState): Promise<void>;
+/** Re-export the ones the entrypoint files consume. */
+export type { GlobalTracerProviderRegistration, NodeTracerProvider };
+//# sourceMappingURL=phoenix-test-tracking.d.ts.map

package/dist/esm/testing/phoenix-test-tracking.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"phoenix-test-tracking.d.ts","sourceRoot":"","sources":["../../../src/testing/phoenix-test-tracking.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,gCAAgC,EAErC,KAAK,kBAAkB,EAQxB,MAAM,uBAAuB,CAAC;AAa/B,OAAO,EAAc,KAAK,QAAQ,EAAE,KAAK,UAAU,EAAE,MAAM,SAAS,CAAC;AACrE,OAAO,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAOjD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,CAAC,EAAE,UAAU,GAAG;IACrD,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAQA;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,KAAK,EAAE,UAAU,GAChB,MAAM,CAaR;AAoGD;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAgKtE;AAeD;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,MAAM,EAC7C,KAAK,EAAE,UAAU,EACjB,QAAQ,EAAE,MAAM,EAChB,EAAE,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,GACxB,OAAO,CACN;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,KAAK,CAAC;IAAC,WAAW,EAAE,OAAO,CAAA;CAAE,CAC1D,CAiCA;AAMD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,QAAQ,GAAG,IAAI,CAErD;AAsDD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,UAAU,EACjB,GAAG,EAAE,QAAQ,GACZ,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAwC7B;AAED;;GAEG;AACH,wBAAsB,cAAc,CAClC,KAAK,EAAE,UAAU,EACjB,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,UAAU,EAAE,UAAU,GACrB,OAAO,CAAC,IAAI,CAAC,CAgCf;AAED,2DAA2D;AAC3D,wBAAsB,uBAAuB,CAC3C,eAAe,SAAS,KAAK,EAC7B,eAAe,EAEf,KAAK,EAAE,UAAU,EACjB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,eAAe,EACvB,EAAE,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,GAC1E,OAAO,CAAC;IAAE,MAAM,EAAE,eAAe,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAiC9D;AAED;;;GAGG;AACH,wBAAsB,aAAa,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAYpE;AAgBD,uDAAuD;AACvD,YAAY,EAAE,gCAAgC,EAAE,kBAAkB,EAAE,CAAC"}