@arizeai/phoenix-client 6.11.1 → 6.12.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arizeai/phoenix-client",
-  "version": "6.11.1",
+  "version": "6.12.0",
   "description": "A client for the Phoenix API",
   "keywords": [
     "arize",
@@ -96,25 +96,25 @@
   },
   "dependencies": {
     "@arizeai/openinference-semantic-conventions": "^2.5.0",
-    "@arizeai/openinference-vercel": "^2.7.8",
+    "@arizeai/openinference-vercel": "^2.8.1",
     "async": "^3.2.6",
     "openapi-fetch": "^0.17.0",
     "tiny-invariant": "^1.3.3",
     "zod": "^4.4.3",
-    "@arizeai/phoenix-config": "0.1.4",
-    "@arizeai/phoenix-otel": "1.0.2"
+    "@arizeai/phoenix-otel": "1.0.2",
+    "@arizeai/phoenix-config": "0.1.4"
   },
   "devDependencies": {
-    "@ai-sdk/openai": "^3.0.73",
+    "@ai-sdk/openai": "^3.0.75",
     "@anthropic-ai/sdk": "^0.102.0",
     "@opentelemetry/api": "^1.9.1",
     "@opentelemetry/sdk-trace-node": "^2.8.0",
     "@types/async": "^3.2.25",
     "@types/node": "^25.9.4",
-    "ai": "^6.0.208",
-    "dotenv": "^16.4.7",
+    "ai": "^6.0.211",
+    "dotenv": "^16.6.1",
     "jest": "^29.7.0",
-    "openai": "^6.44.0",
+    "openai": "^6.45.0",
     "openapi-typescript": "^7.13.0",
     "tsx": "^4.22.4",
     "vitest": "^4.1.9",

package/src/__generated__/api/v1.ts

@@ -1307,7 +1307,24 @@ export interface paths {
         };
         get?: never;
         put?: never;
-        /** Run Server Agent */
+        /**
+         * Run Server Agent
+         * @description Stream a chat turn from the GraphQL server agent.
+         *
+         *     This is the endpoint the PXI CLI talks to directly (no pre-configured
+         *     agent record): it builds a fresh server agent per request from the
+         *     caller-supplied model and contexts, then streams the reply back as
+         *     Vercel-AI chunks.
+         *
+         *     The request contexts gate capabilities — GraphQL mutations, web access,
+         *     and subagents — and mutations are refused for viewer users. When trace
+         *     recording is enabled (and permitted by system settings), the run is
+         *     traced; locally ingested traces are persisted to the agent's project
+         *     once the stream completes.
+         *
+         *     Returns ``403`` if agents or the server agent are disabled, or if a
+         *     viewer requests mutations.
+         */
         post: operations["run_server_agent_agents_server_sessions__session_id__chat_post"];
         delete?: never;
         options?: never;

package/src/spans/index.ts

@@ -7,3 +7,4 @@ export * from "./getSpans";
 export type { SpanKindFilter, SpanStatusCode } from "../types/spans";
 export * from "./getSpanAnnotations";
 export * from "./deleteSpan";
+export * from "./logSpans";

package/src/spans/logSpans.ts

@@ -0,0 +1,328 @@
+import type { paths } from "../__generated__/api/v1";
+import { createClient } from "../client";
+import type { ClientFn } from "../types/core";
+import type { ProjectIdentifier } from "../types/projects";
+import { resolveProjectIdentifier } from "../types/projects";
+import { formatApiError } from "../utils/apiErrorUtils";
+import { isObject } from "../utils/isObject";
+import { safelyParseJSON } from "../utils/safelyParseJSON";
+
+type CreateSpansRequestData =
+  paths["/v1/projects/{project_identifier}/spans"]["post"]["requestBody"]["content"]["application/json"]["data"];
+
+/**
+ * A span in Phoenix's simplified span structure, as accepted by {@link logSpans}.
+ * This is the same shape returned by `getSpans`, which makes it possible to read
+ * spans from one project and log them into another.
+ */
+export type Span = CreateSpansRequestData[number];
+
+/** Information about a span that failed validation and was not queued. */
+export interface InvalidSpanInfo {
+  spanId: string;
+  traceId: string;
+  error: string;
+}
+
+/** Information about a span that was rejected because it already exists. */
+export interface DuplicateSpanInfo {
+  spanId: string;
+  traceId: string;
+}
+
+/**
+ * Raised by {@link logSpans} when one or more spans in the request are invalid
+ * or duplicates. If any span in a request fails, none of the spans in that
+ * request are queued.
+ */
+export class SpanCreationError extends Error {
+  readonly invalidSpans: InvalidSpanInfo[];
+  readonly duplicateSpans: DuplicateSpanInfo[];
+  readonly totalReceived: number;
+  readonly totalQueued: number;
+
+  constructor(params: {
+    message: string;
+    invalidSpans?: InvalidSpanInfo[];
+    duplicateSpans?: DuplicateSpanInfo[];
+    totalReceived?: number;
+    totalQueued?: number;
+  }) {
+    super(params.message);
+    this.name = "SpanCreationError";
+    this.invalidSpans = params.invalidSpans ?? [];
+    this.duplicateSpans = params.duplicateSpans ?? [];
+    this.totalReceived = params.totalReceived ?? 0;
+    this.totalQueued = params.totalQueued ?? 0;
+  }
+
+  /** Number of spans rejected as invalid. */
+  get totalInvalid(): number {
+    return this.invalidSpans.length;
+  }
+
+  /** Number of spans rejected as duplicates. */
+  get totalDuplicates(): number {
+    return this.duplicateSpans.length;
+  }
+}
+
+/**
+ * Parameters to log spans to a project
+ */
+export interface LogSpansParams extends ClientFn {
+  /** The project to log spans into */
+  project: ProjectIdentifier;
+  /** The spans to log */
+  spans: Span[];
+}
+
+/**
+ * Statistics about a {@link logSpans} call. When successful, `totalQueued`
+ * equals `totalReceived`.
+ */
+export interface LogSpansResult {
+  totalReceived: number;
+  totalQueued: number;
+}
+
+const MAX_ERRORS_TO_SHOW = 5;
+
+function formatLogSpansErrorMessage({
+  invalidSpans,
+  duplicateSpans,
+}: {
+  invalidSpans: InvalidSpanInfo[];
+  duplicateSpans: DuplicateSpanInfo[];
+}): string {
+  const parts: string[] = [];
+
+  if (invalidSpans.length > 0) {
+    parts.push(`Failed to queue ${invalidSpans.length} invalid spans:`);
+    for (const span of invalidSpans.slice(0, MAX_ERRORS_TO_SHOW)) {
+      parts.push(`  - Span ${span.spanId}: ${span.error}`);
+    }
+    if (invalidSpans.length > MAX_ERRORS_TO_SHOW) {
+      parts.push(
+        `  ... and ${invalidSpans.length - MAX_ERRORS_TO_SHOW} more invalid spans`
+      );
+    }
+  }
+
+  if (duplicateSpans.length > 0) {
+    if (parts.length > 0) parts.push("");
+    parts.push(`Found ${duplicateSpans.length} duplicate spans:`);
+    for (const span of duplicateSpans.slice(0, MAX_ERRORS_TO_SHOW)) {
+      parts.push(`  - Span ${span.spanId}`);
+    }
+    if (duplicateSpans.length > MAX_ERRORS_TO_SHOW) {
+      parts.push(
+        `  ... and ${duplicateSpans.length - MAX_ERRORS_TO_SHOW} more duplicates`
+      );
+    }
+  }
+
+  return parts.join("\n");
+}
+
+function toSpanIdAndTraceId(item: unknown): {
+  spanId: string;
+  traceId: string;
+} {
+  const record = isObject(item) ? (item as Record<string, unknown>) : {};
+  return {
+    spanId: typeof record.span_id === "string" ? record.span_id : "unknown",
+    traceId: typeof record.trace_id === "string" ? record.trace_id : "unknown",
+  };
+}
+
+function toInvalidSpanInfoList(value: unknown): InvalidSpanInfo[] {
+  if (!Array.isArray(value)) return [];
+  return value.map((item) => {
+    const record = isObject(item) ? (item as Record<string, unknown>) : {};
+    return {
+      ...toSpanIdAndTraceId(item),
+      error:
+        typeof record.error === "string" ? record.error : "Validation error",
+    };
+  });
+}
+
+function toDuplicateSpanInfoList(value: unknown): DuplicateSpanInfo[] {
+  if (!Array.isArray(value)) return [];
+  return value.map(toSpanIdAndTraceId);
+}
+
+/**
+ * Builds a {@link SpanCreationError} from already-normalized invalid/duplicate
+ * span lists, computing the shared message once.
+ */
+function makeSpanCreationError(params: {
+  invalidSpans: InvalidSpanInfo[];
+  duplicateSpans: DuplicateSpanInfo[];
+  totalReceived: number;
+  totalQueued: number;
+}): SpanCreationError {
+  return new SpanCreationError({
+    message: formatLogSpansErrorMessage(params),
+    ...params,
+  });
+}
+
+/**
+ * Builds a {@link SpanCreationError} from a parsed error payload matching the
+ * shape returned by the server for invalid/duplicate spans:
+ * `{ total_received, total_queued, total_invalid, total_duplicates, invalid_spans, duplicate_spans }`.
+ */
+function buildSpanCreationError(
+  payload: Record<string, unknown>
+): SpanCreationError {
+  return makeSpanCreationError({
+    invalidSpans: toInvalidSpanInfoList(payload.invalid_spans),
+    duplicateSpans: toDuplicateSpanInfoList(payload.duplicate_spans),
+    totalReceived:
+      typeof payload.total_received === "number" ? payload.total_received : 0,
+    totalQueued:
+      typeof payload.total_queued === "number" ? payload.total_queued : 0,
+  });
+}
+
+function isSpanCreationErrorPayload(
+  value: unknown
+): value is Record<string, unknown> {
+  return (
+    isObject(value) && ("invalid_spans" in value || "duplicate_spans" in value)
+  );
+}
+
+/**
+ * Extracts invalid span info from a FastAPI request-validation error array
+ * (returned with a 422 status when a span in the payload is malformed).
+ */
+function extractInvalidSpansFromValidationErrors(
+  errors: unknown[],
+  spans: Span[]
+): InvalidSpanInfo[] {
+  const invalidSpans: InvalidSpanInfo[] = [];
+
+  for (const rawError of errors) {
+    if (!isObject(rawError)) continue;
+    const loc = (rawError as { loc?: unknown }).loc;
+    if (
+      !Array.isArray(loc) ||
+      loc.length < 3 ||
+      loc[0] !== "body" ||
+      loc[1] !== "data" ||
+      typeof loc[2] !== "number"
+    ) {
+      continue;
+    }
+
+    const span = spans[loc[2]];
+    if (!span) continue;
+
+    const msg = (rawError as { msg?: unknown }).msg;
+    invalidSpans.push({
+      spanId: span.context?.span_id ?? "unknown",
+      traceId: span.context?.trace_id ?? "unknown",
+      error: typeof msg === "string" ? msg : "Validation error",
+    });
+  }
+
+  return invalidSpans;
+}
+
+function parseLogSpansError(error: unknown, spans: Span[]): Error {
+  if (isObject(error)) {
+    const detail = (error as { detail?: unknown }).detail;
+
+    if (typeof detail === "string") {
+      const { json: parsed } = safelyParseJSON(detail);
+      if (isSpanCreationErrorPayload(parsed)) {
+        return buildSpanCreationError(parsed);
+      }
+    } else if (Array.isArray(detail)) {
+      const invalidSpans = extractInvalidSpansFromValidationErrors(
+        detail,
+        spans
+      );
+      if (invalidSpans.length > 0) {
+        return makeSpanCreationError({
+          invalidSpans,
+          duplicateSpans: [],
+          totalReceived: spans.length,
+          // A FastAPI 422 rejects the entire request body, so no spans are
+          // queued — matching the all-or-nothing contract and the 400 path.
+          totalQueued: 0,
+        });
+      }
+    } else if (isSpanCreationErrorPayload(error)) {
+      return buildSpanCreationError(error);
+    }
+  }
+
+  return new Error(`Failed to log spans: ${formatApiError(error)}`);
+}
+
+/**
+ * Log spans to a project using Phoenix's simplified span structure.
+ *
+ * If any span in the request is invalid or a duplicate of a span that already
+ * exists, none of the spans in the request are queued and a
+ * {@link SpanCreationError} is thrown with details about the failures.
+ *
+ * @experimental this function is experimental and may change in the future
+ *
+ * @param params - The parameters to log spans
+ * @returns Statistics about the operation. When successful, `totalQueued`
+ * equals `totalReceived`.
+ *
+ * @example
+ * ```ts
+ * const result = await logSpans({
+ *   project: { projectName: "my-project" },
+ *   spans: [
+ *     {
+ *       name: "test",
+ *       context: { trace_id: "123", span_id: "456" },
+ *       span_kind: "CHAIN",
+ *       start_time: "2024-01-01T00:00:00Z",
+ *       end_time: "2024-01-01T00:00:01Z",
+ *       status_code: "OK",
+ *     },
+ *   ],
+ * });
+ * console.log(`Queued ${result.totalQueued} spans`);
+ * ```
+ */
+export async function logSpans({
+  client: _client,
+  project,
+  spans,
+}: LogSpansParams): Promise<LogSpansResult> {
+  const client = _client ?? createClient();
+  const projectIdentifier = resolveProjectIdentifier(project);
+
+  const { data, error } = await client.POST(
+    "/v1/projects/{project_identifier}/spans",
+    {
+      params: {
+        path: {
+          project_identifier: projectIdentifier,
+        },
+      },
+      body: {
+        data: spans,
+      },
+    }
+  );
+
+  if (error) {
+    throw parseLogSpansError(error, spans);
+  }
+
+  return {
+    totalReceived: data?.total_received ?? spans.length,
+    totalQueued: data?.total_queued ?? 0,
+  };
+}

package/src/testing/phoenix-test-tracking.ts

@@ -28,22 +28,66 @@ import {
 import { currentRun, type RunState, type SuiteState } from "./state";
 import type { Annotation, KVMap } from "./types";
 
-function isFalsyFlag(value: string | undefined): boolean {
-  const v = (value ?? "").toLowerCase();
+export function isFalsyFlag(value: string | undefined): boolean {
+  // Tolerate surrounding whitespace and matching quotes that survive some
+  // shells and `.env` loaders (e.g. `PHOENIX_TEST_TRACKING="false"` or a value
+  // with a trailing newline). Without this, such a value reads as truthy and
+  // silently re-enables recording even though the user asked to disable it.
+  const v = (value ?? "")
+    .trim()
+    .replace(/^(['"])(.*)\1$/, "$2")
+    .trim()
+    .toLowerCase();
   return v === "false" || v === "0" || v === "off" || v === "no";
 }
 
+/**
+ * Snapshot of `PHOENIX_TEST_TRACKING` as it was when this module first loaded.
+ *
+ * When the flag is exported on the command line (the documented `eval:offline`
+ * workflow), this captures the user's intent at process start and is immune to
+ * any later in-process mutation of `process.env` by a sibling suite or setup
+ * file. That mutation is what made tracking leak across suites in #13930: one
+ * suite flipped the env var and re-enabled recording for the others.
+ */
+const trackingDisabledAtLoad = isFalsyFlag(process.env.PHOENIX_TEST_TRACKING);
+
+/**
+ * Latches to `true` the first time tracking is observed disabled in this
+ * process. Recording is opt-out and shared process-wide, so a single falsy
+ * reading turns the whole run off and keeps it off — later suites cannot
+ * re-enable recording regardless of declaration or execution order.
+ */
+let trackingLatchedOff = trackingDisabledAtLoad;
+
+/**
+ * Reset the process-level tracking latch. Test-only seam so unit tests can
+ * exercise the enable/disable transitions in isolation; not part of the
+ * public API.
+ *
+ * @internal
+ */
+export function __resetTrackingLatchForTests(): void {
+  trackingLatchedOff = isFalsyFlag(process.env.PHOENIX_TEST_TRACKING);
+}
+
 /**
  * Decide whether tests should sync to Phoenix.
  *
  * Tracing is enabled by default. It can be disabled globally by setting
  * `PHOENIX_TEST_TRACKING=false`, or per suite via `SuiteConfig.dryRun`.
+ *
+ * The global disable is sticky for the lifetime of the process: once the flag
+ * is seen falsy — at load time or on any later call — tracking stays off for
+ * every suite. This keeps offline mode deterministic regardless of which
+ * suites are included in a run, or the order they execute in.
  */
 export function isTrackingEnabled(suite?: SuiteState): {
   enabled: boolean;
   reason?: string;
 } {
-  if (isFalsyFlag(process.env.PHOENIX_TEST_TRACKING)) {
+  if (trackingLatchedOff || isFalsyFlag(process.env.PHOENIX_TEST_TRACKING)) {
+    trackingLatchedOff = true;
     return { enabled: false, reason: "PHOENIX_TEST_TRACKING is disabled" };
   }
   if (suite?.config.dryRun) {
@@ -478,6 +522,7 @@ export async function postExperimentRun(
   if (
     run.dryRun ||
     suite.trackingDisabled ||
+    !isTrackingEnabled(suite).enabled ||
     !suite.client ||
     !suite.experimentId
   ) {
@@ -524,7 +569,13 @@ export async function postAnnotation(
   runId: string | undefined,
   annotation: Annotation
 ): Promise<void> {
-  if (suite.trackingDisabled || !suite.client || !runId) return;
+  if (
+    suite.trackingDisabled ||
+    !isTrackingEnabled(suite).enabled ||
+    !suite.client ||
+    !runId
+  )
+    return;
   const start = new Date();
   const end = new Date();
   try {