@arizeai/phoenix-client 1.1.0 → 1.3.0

package/src/spans/addSpanAnnotation.ts

@@ -0,0 +1,59 @@
+import { createClient } from "../client";
+import { ClientFn } from "../types/core";
+import { SpanAnnotation, toSpanAnnotationData } from "./types";
+
+/**
+ * Parameters to add a span annotation
+ */
+interface AddSpanAnnotationParams extends ClientFn {
+  spanAnnotation: SpanAnnotation;
+}
+
+/**
+ * Add an annotation to a span.
+ *
+ * The annotation can be of type "LLM", "CODE", or "HUMAN" and can include a label, score, and metadata.
+ * If an identifier is provided and an annotation with that identifier already exists, it will be updated.
+ *
+ * @param params - The parameters to add a span annotation
+ * @returns The ID of the created or updated annotation
+ *
+ * @example
+ * ```ts
+ * const result = await addSpanAnnotation({
+ *   spanAnnotation: {
+ *     spanId: "123abc",
+ *     name: "quality_score",
+ *     label: "good",
+ *     score: 0.95,
+ *     annotatorKind: "LLM",
+ *     identifier: "custom_id_123",
+ *     metadata: {
+ *       model: "gpt-4"
+ *     }
+ *   }
+ * });
+ * ```
+ */
+export async function addSpanAnnotation({
+  client: _client,
+  spanAnnotation,
+}: AddSpanAnnotationParams): Promise<{ id: string }> {
+  const client = _client ?? createClient();
+
+  const { data, error } = await client.POST("/v1/span_annotations", {
+    body: {
+      data: [toSpanAnnotationData(spanAnnotation)],
+    },
+  });
+
+  if (error) {
+    throw new Error(`Failed to add span annotation: ${error}`);
+  }
+
+  if (!data?.data?.[0]?.id) {
+    throw new Error("No annotation ID returned from server");
+  }
+
+  return data.data[0];
+}

package/src/spans/index.ts

@@ -0,0 +1,2 @@
+export * from "./addSpanAnnotation";
+export * from "./logSpanAnnotations";

package/src/spans/logSpanAnnotations.ts

@@ -0,0 +1,71 @@
+import { createClient } from "../client";
+import { ClientFn } from "../types/core";
+import { SpanAnnotation, toSpanAnnotationData } from "./types";
+
+/**
+ * Parameters to log multiple span annotations
+ */
+interface LogSpanAnnotationsParams extends ClientFn {
+  /**
+   * The span annotations to log
+   */
+  spanAnnotations: SpanAnnotation[];
+}
+
+/**
+ * Log multiple span annotations in a single request.
+ *
+ * Each annotation can be of type "LLM", "CODE", or "HUMAN" and can include a label, score, and metadata.
+ * If an identifier is provided and an annotation with that identifier already exists, it will be updated.
+ *
+ * @param params - The parameters to log span annotations
+ * @returns The IDs of the created or updated annotations
+ *
+ * @example
+ * ```ts
+ * const results = await logSpanAnnotations({
+ *   spanAnnotations: [
+ *     {
+ *       spanId: "123abc",
+ *       name: "quality_score",
+ *       label: "good",
+ *       score: 0.95,
+ *       annotatorKind: "LLM",
+ *       identifier: "custom_id_123",
+ *       metadata: {
+ *         model: "gpt-4"
+ *       }
+ *     },
+ *     {
+ *       spanId: "456def",
+ *       name: "sentiment",
+ *       label: "positive",
+ *       score: 0.8,
+ *       annotatorKind: "CODE"
+ *     }
+ *   ]
+ * });
+ * ```
+ */
+export async function logSpanAnnotations({
+  client: _client,
+  spanAnnotations,
+}: LogSpanAnnotationsParams): Promise<{ id: string }[]> {
+  const client = _client ?? createClient();
+
+  const { data, error } = await client.POST("/v1/span_annotations", {
+    body: {
+      data: spanAnnotations.map(toSpanAnnotationData),
+    },
+  });
+
+  if (error) {
+    throw new Error(`Failed to log span annotations: ${error}`);
+  }
+
+  if (!data?.data?.length) {
+    throw new Error("No annotation IDs returned from server");
+  }
+
+  return data.data;
+}

package/src/spans/types.ts

@@ -0,0 +1,60 @@
+import { paths } from "../__generated__/api/v1";
+
+type SpanAnnotationData =
+  paths["/v1/span_annotations"]["post"]["requestBody"]["content"]["application/json"]["data"][0];
+
+/**
+ * Parameters for a single span annotation
+ */
+export interface SpanAnnotation {
+  /**
+   * The OpenTelemetry Span ID (hex format without 0x prefix)
+   */
+  spanId: string;
+  /**
+   * The name of the annotation
+   */
+  name: string;
+  /**
+   * The label assigned by the annotation
+   */
+  label?: string;
+  /**
+   * The score assigned by the annotation
+   */
+  score?: number;
+  /**
+   * The identifier of the annotation. If provided, the annotation will be updated if it already exists.
+   */
+  identifier?: string;
+  /**
+   * Metadata for the annotation
+   */
+  metadata?: Record<string, unknown>;
+  /**
+   * The kind of annotator used for the annotation
+   * Can be "HUMAN", "LLM", or "CODE"
+   * @default "HUMAN"
+   */
+  annotatorKind?: SpanAnnotationData["annotator_kind"];
+}
+
+/**
+ * Convert a SpanAnnotation to the API format
+ */
+export function toSpanAnnotationData(
+  annotation: SpanAnnotation
+): SpanAnnotationData {
+  return {
+    span_id: annotation.spanId,
+    name: annotation.name,
+    annotator_kind: annotation.annotatorKind ?? "HUMAN",
+    result: {
+      label: annotation.label ?? null,
+      score: annotation.score ?? null,
+      explanation: null,
+    },
+    metadata: annotation.metadata ?? null,
+    identifier: annotation.identifier ?? "",
+  };
+}

package/src/types/annotations.ts

@@ -1 +1,4 @@
-export type AnnotatorKind = "HUMAN" | "LLM";
+import { components } from "../__generated__/api/v1";
+
+export type AnnotatorKind =
+  components["schemas"]["SpanAnnotationData"]["annotator_kind"];

package/src/types/experiments.ts

@@ -8,7 +8,6 @@ import { Example } from "./datasets";
 export interface Experiment extends Node {
   datasetId: string;
   datasetVersionId: string;
-  repetitions: number;
   /**
    * The project under which the experiment task traces are recorded
    */
@@ -16,7 +15,6 @@ export interface Experiment extends Node {
 }
 
 export interface RanExperiment extends Experiment {
-  params: ExperimentParameters;
   runs: Record<string, ExperimentRun>;
   evaluationRuns?: ExperimentEvaluationRun[];
 }
@@ -32,7 +30,6 @@ export interface ExperimentRun extends Node {
    */
   experimentId: string;
   datasetExampleId: string;
-  repetitionNumber: number;
   output?: string | Record<string, unknown> | null;
   error: string | null;
   traceId: string | null;
@@ -59,6 +56,7 @@ export type EvaluatorParams = {
 
 export type Evaluator = {
   name: string;
+  kind: AnnotatorKind;
   evaluate: (
     args: EvaluatorParams
   ) => Promise<EvaluationResult> | EvaluationResult;
@@ -100,9 +98,4 @@ export interface ExperimentParameters {
    * The number of examples to run the experiment on
    */
   nExamples: number;
-  /**
-   * The number of repetitions to run the experiment
-   * e.g. the number of times to run the task
-   */
-  nRepetitions: number;
 }

package/src/utils/ensureString.ts

@@ -0,0 +1,14 @@
+import { safelyStringifyJSON } from "./safelyStringifyJSON";
+
+/**
+ * Ensures that a value is a string.
+ * If the value is not a string, it will be converted to a string using `safelyStringifyJSON`.
+ * @param value - The value to ensure is a string.
+ * @returns The value as a string.
+ */
+export function ensureString(value: unknown): string {
+  if (typeof value === "string") {
+    return value;
+  }
+  return safelyStringifyJSON(value)?.json ?? "";
+}

package/src/utils/objectAsAttributes.ts

@@ -0,0 +1,9 @@
+import { AttributeValue } from "@opentelemetry/api";
+
+export function objectAsAttributes<T extends Record<string, unknown>>(
+  obj: T
+): Record<string, AttributeValue> {
+  return Object.fromEntries(
+    Object.entries(obj).filter(([_, value]) => value !== null)
+  ) as Record<string, AttributeValue>;
+}