@ai-sdk/otel 1.0.110 → 1.0.111

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @ai-sdk/otel
 
+## 1.0.111
+
+### Patch Changes
+
+- 31d24ce: feat: add telemetry support to `experimental_evaluate`
+- Updated dependencies [31d24ce]
+- Updated dependencies [a65bfd9]
+  - ai@7.0.111
+
 ## 1.0.110
 
 ### Patch Changes

package/dist/index.d.ts

@@ -1,9 +1,9 @@
-import { Telemetry, InferTelemetryEvent, GenerateTextStartEvent, GenerateObjectStartEvent, EmbedStartEvent, RerankStartEvent, GenerateObjectStepStartEvent, GenerateObjectStepEndEvent, GenerateTextStepStartEvent, ToolSet, LanguageModelCallStartEvent, LanguageModelCallEndEvent, ToolExecutionStartEvent, ToolExecutionEndEvent, GenerateTextStepEndEvent, GenerateTextEndEvent, GenerateObjectEndEvent, EmbedEndEvent, RerankEndEvent, EmbeddingModelCallStartEvent, EmbeddingModelCallEndEvent, RerankingModelCallStartEvent, RerankingModelCallEndEvent, GenerateTextAbortEvent, Output } from 'ai';
+import { Telemetry, InferTelemetryEvent, GenerateTextStartEvent, GenerateObjectStartEvent, EmbedStartEvent, RerankStartEvent, GenerateObjectStepStartEvent, GenerateObjectStepEndEvent, GenerateTextStepStartEvent, ToolSet, LanguageModelCallStartEvent, LanguageModelCallEndEvent, ToolExecutionStartEvent, ToolExecutionEndEvent, GenerateTextStepEndEvent, GenerateTextEndEvent, GenerateObjectEndEvent, EmbedEndEvent, RerankEndEvent, EmbeddingModelCallStartEvent, EmbeddingModelCallEndEvent, RerankingModelCallStartEvent, RerankingModelCallEndEvent, Experimental_EvaluateStartEvent, Experimental_EvaluateEndEvent, Experimental_EvaluationModelCallStartEvent, Experimental_EvaluationModelCallEndEvent, GenerateTextAbortEvent, Output } from 'ai';
 import { Attributes, Tracer } from '@opentelemetry/api';
 import { LanguageModelV4Prompt } from '@ai-sdk/provider';
 import { Context } from '@ai-sdk/provider-utils';
 
-type OpenTelemetrySpanType = 'operation' | 'step' | 'languageModel' | 'tool' | 'embedding' | 'reranking';
+type OpenTelemetrySpanType = 'operation' | 'step' | 'languageModel' | 'tool' | 'embedding' | 'reranking' | 'experimental_evaluation';
 type EnrichSpan = (options: {
     spanType: OpenTelemetrySpanType;
     operationId: string;
@@ -36,6 +36,10 @@ type OpenTelemetryOptions = {
      * Emit reranking input documents and output ranking.
      */
     reranking?: boolean;
+    /**
+     * Emit evaluation state, questions, and answers.
+     */
+    experimental_evaluation?: boolean;
     /**
      * Emit runtime context values.
      */
@@ -105,6 +109,12 @@ declare class OpenTelemetry implements Telemetry {
     private onRerankOperationEnd;
     onRerankStart(event: RerankingModelCallStartEvent): void;
     onRerankEnd(event: RerankingModelCallEndEvent): void;
+    private onEvaluateOperationStart;
+    private onEvaluateOperationEnd;
+    experimental_onEvaluateStart(event: InferTelemetryEvent<Experimental_EvaluateStartEvent>): void;
+    experimental_onEvaluateEnd(event: Experimental_EvaluateEndEvent): void;
+    experimental_onEvaluationModelCallStart(event: Experimental_EvaluationModelCallStartEvent): void;
+    experimental_onEvaluationModelCallEnd(event: Experimental_EvaluationModelCallEndEvent): void;
     onAbort(event: GenerateTextAbortEvent<ToolSet>): void;
     onError(error: unknown): void;
 }
@@ -162,6 +172,12 @@ declare class LegacyOpenTelemetry implements Telemetry {
     private onRerankOperationEnd;
     onRerankStart(event: RerankingModelCallStartEvent): void;
     onRerankEnd(event: RerankingModelCallEndEvent): void;
+    private onEvaluateOperationStart;
+    private onEvaluateOperationEnd;
+    experimental_onEvaluateStart(event: InferTelemetryEvent<Experimental_EvaluateStartEvent>): void;
+    experimental_onEvaluateEnd(event: Experimental_EvaluateEndEvent): void;
+    experimental_onEvaluationModelCallStart(event: Experimental_EvaluationModelCallStartEvent): void;
+    experimental_onEvaluationModelCallEnd(event: Experimental_EvaluationModelCallEndEvent): void;
     onAbort(event: GenerateTextAbortEvent<ToolSet>): void;
     onError(error: unknown): void;
 }

package/dist/index.js

@@ -569,23 +569,25 @@ var disabledSupplementalAttributes = {
   providerMetadata: false,
   embedding: false,
   reranking: false,
+  experimental_evaluation: false,
   runtimeContext: false,
   headers: false,
   toolChoice: false,
   schema: false
 };
 function normalizeSupplementalAttributes(options) {
-  var _a, _b, _c, _d, _e, _f, _g, _h;
+  var _a, _b, _c, _d, _e, _f, _g, _h, _i;
   return {
     ...disabledSupplementalAttributes,
     usage: (_a = options.usage) != null ? _a : false,
     providerMetadata: (_b = options.providerMetadata) != null ? _b : false,
     embedding: (_c = options.embedding) != null ? _c : false,
     reranking: (_d = options.reranking) != null ? _d : false,
-    runtimeContext: (_e = options.runtimeContext) != null ? _e : false,
-    headers: (_f = options.headers) != null ? _f : false,
-    toolChoice: (_g = options.toolChoice) != null ? _g : false,
-    schema: (_h = options.schema) != null ? _h : false
+    experimental_evaluation: (_e = options.experimental_evaluation) != null ? _e : false,
+    runtimeContext: (_f = options.runtimeContext) != null ? _f : false,
+    headers: (_g = options.headers) != null ? _g : false,
+    toolChoice: (_h = options.toolChoice) != null ? _h : false,
+    schema: (_i = options.schema) != null ? _i : false
   };
 }
 function getRuntimeContextAttributes(context4) {
@@ -810,6 +812,7 @@ var OpenTelemetry = class {
       inferenceContext: void 0,
       embedSpans: /* @__PURE__ */ new Map(),
       rerankSpan: void 0,
+      evaluationSpan: void 0,
       toolSpans: /* @__PURE__ */ new Map(),
       settings,
       provider: event.provider,
@@ -901,6 +904,7 @@ var OpenTelemetry = class {
       inferenceContext: void 0,
       embedSpans: /* @__PURE__ */ new Map(),
       rerankSpan: void 0,
+      evaluationSpan: void 0,
       toolSpans: /* @__PURE__ */ new Map(),
       settings,
       provider: event.provider,
@@ -1063,6 +1067,7 @@ var OpenTelemetry = class {
       inferenceContext: void 0,
       embedSpans: /* @__PURE__ */ new Map(),
       rerankSpan: void 0,
+      evaluationSpan: void 0,
       toolSpans: /* @__PURE__ */ new Map(),
       settings: { maxRetries: event.maxRetries },
       provider: event.provider,
@@ -1602,6 +1607,7 @@ var OpenTelemetry = class {
       inferenceContext: void 0,
       embedSpans: /* @__PURE__ */ new Map(),
       rerankSpan: void 0,
+      evaluationSpan: void 0,
       toolSpans: /* @__PURE__ */ new Map(),
       settings: { maxRetries: event.maxRetries },
       provider: event.provider,
@@ -1670,6 +1676,162 @@ var OpenTelemetry = class {
     span.end();
     state.rerankSpan = void 0;
   }
+  onEvaluateOperationStart(event) {
+    const telemetry = {
+      recordInputs: event.recordInputs,
+      recordOutputs: event.recordOutputs,
+      functionId: event.functionId
+    };
+    const runtimeContext = event.runtimeContext;
+    const baseSupplementalAttributes = selectSupplementalAttributes(
+      telemetry,
+      this.supplementalAttributes,
+      {
+        runtimeContext: getRuntimeContextAttributes(runtimeContext),
+        headers: getHeaderAttributes(event.headers)
+      }
+    );
+    const attributes = selectAttributes(telemetry, {
+      "gen_ai.operation.name": "evaluate",
+      "gen_ai.provider.name": mapProviderName(event.provider),
+      "gen_ai.request.model": event.modelId,
+      ...baseSupplementalAttributes,
+      ...selectSupplementalAttributes(telemetry, this.supplementalAttributes, {
+        experimental_evaluation: {
+          "ai.evaluation.state": {
+            input: () => JSON.stringify(event.state)
+          },
+          "ai.evaluation.questions": {
+            input: () => JSON.stringify(event.questions)
+          }
+        }
+      })
+    });
+    const rootSpan = this.tracer.startSpan(`evaluate ${event.modelId}`, {
+      attributes: this.getSpanAttributes({
+        attributes,
+        spanType: "operation",
+        operationId: event.operationId,
+        callId: event.callId,
+        runtimeContext
+      }),
+      kind: SpanKind.CLIENT
+    });
+    const rootContext = trace.setSpan(context2.active(), rootSpan);
+    this.callStates.set(event.callId, {
+      operationId: event.operationId,
+      telemetry,
+      rootSpan,
+      rootContext,
+      stepSpan: void 0,
+      stepContext: void 0,
+      inferenceSpan: void 0,
+      inferenceContext: void 0,
+      embedSpans: /* @__PURE__ */ new Map(),
+      rerankSpan: void 0,
+      evaluationSpan: void 0,
+      toolSpans: /* @__PURE__ */ new Map(),
+      settings: { maxRetries: event.maxRetries },
+      provider: event.provider,
+      modelId: event.modelId,
+      runtimeContext,
+      baseSupplementalAttributes
+    });
+  }
+  onEvaluateOperationEnd(event) {
+    const state = this.getCallState(event.callId);
+    if (!(state == null ? void 0 : state.rootSpan)) return;
+    state.rootSpan.setAttributes(
+      selectSupplementalAttributes(
+        state.telemetry,
+        this.supplementalAttributes,
+        {
+          experimental_evaluation: {
+            "ai.evaluation.answers": {
+              output: () => JSON.stringify(event.answers)
+            }
+          }
+        }
+      )
+    );
+    state.rootSpan.end();
+    this.cleanupCallState(event.callId);
+  }
+  experimental_onEvaluateStart(event) {
+    this.onEvaluateOperationStart(event);
+  }
+  experimental_onEvaluateEnd(event) {
+    this.onEvaluateOperationEnd(event);
+  }
+  experimental_onEvaluationModelCallStart(event) {
+    const state = this.getCallState(event.callId);
+    if (!(state == null ? void 0 : state.rootSpan) || !state.rootContext) return;
+    const attributes = selectAttributes(state.telemetry, {
+      "gen_ai.operation.name": "evaluate",
+      "gen_ai.provider.name": mapProviderName(state.provider),
+      "gen_ai.request.model": state.modelId,
+      ...state.baseSupplementalAttributes,
+      ...selectSupplementalAttributes(
+        state.telemetry,
+        this.supplementalAttributes,
+        {
+          experimental_evaluation: {
+            "ai.evaluation.state": {
+              input: () => JSON.stringify(event.state)
+            },
+            "ai.evaluation.questions": {
+              input: () => JSON.stringify(event.questions)
+            }
+          }
+        }
+      )
+    });
+    const span = this.tracer.startSpan(
+      `evaluate ${state.modelId}`,
+      {
+        attributes: this.getSpanAttributes({
+          attributes,
+          spanType: "experimental_evaluation",
+          operationId: event.operationId,
+          callId: event.callId,
+          runtimeContext: state.runtimeContext
+        }),
+        kind: SpanKind.CLIENT
+      },
+      state.rootContext
+    );
+    state.evaluationSpan = {
+      span,
+      context: trace.setSpan(state.rootContext, span)
+    };
+  }
+  experimental_onEvaluationModelCallEnd(event) {
+    var _a, _b;
+    const state = this.getCallState(event.callId);
+    if (!(state == null ? void 0 : state.evaluationSpan)) return;
+    state.evaluationSpan.span.setAttributes(
+      selectAttributes(state.telemetry, {
+        "gen_ai.usage.input_tokens": (_a = event.usage) == null ? void 0 : _a.inputTokens,
+        "gen_ai.usage.output_tokens": (_b = event.usage) == null ? void 0 : _b.outputTokens,
+        ...selectSupplementalAttributes(
+          state.telemetry,
+          this.supplementalAttributes,
+          {
+            experimental_evaluation: {
+              "ai.evaluation.answers": {
+                output: () => JSON.stringify(event.answers)
+              }
+            },
+            providerMetadata: {
+              "ai.response.providerMetadata": event.providerMetadata ? JSON.stringify(event.providerMetadata) : void 0
+            }
+          }
+        )
+      })
+    );
+    state.evaluationSpan.span.end();
+    state.evaluationSpan = void 0;
+  }
   onAbort(event) {
     const state = this.getCallState(event.callId);
     if (!(state == null ? void 0 : state.rootSpan)) return;
@@ -1695,6 +1857,10 @@ var OpenTelemetry = class {
       state.rerankSpan.span.end();
       state.rerankSpan = void 0;
     }
+    if (state.evaluationSpan) {
+      state.evaluationSpan.span.end();
+      state.evaluationSpan = void 0;
+    }
     state.rootSpan.end();
     this.cleanupCallState(event.callId);
   }
@@ -1732,6 +1898,11 @@ var OpenTelemetry = class {
       state.rerankSpan.span.end();
       state.rerankSpan = void 0;
     }
+    if (state.evaluationSpan) {
+      recordErrorOnSpan(state.evaluationSpan.span, actualError);
+      state.evaluationSpan.span.end();
+      state.evaluationSpan = void 0;
+    }
     recordErrorOnSpan(state.rootSpan, actualError);
     state.rootSpan.end();
     this.cleanupCallState(event.callId);
@@ -1971,6 +2142,7 @@ var LegacyOpenTelemetry = class {
       stepContext: void 0,
       embedSpans: /* @__PURE__ */ new Map(),
       rerankSpan: void 0,
+      evaluationSpan: void 0,
       toolSpans: /* @__PURE__ */ new Map(),
       baseTelemetryAttributes,
       settings
@@ -2027,6 +2199,7 @@ var LegacyOpenTelemetry = class {
       stepContext: void 0,
       embedSpans: /* @__PURE__ */ new Map(),
       rerankSpan: void 0,
+      evaluationSpan: void 0,
       toolSpans: /* @__PURE__ */ new Map(),
       baseTelemetryAttributes,
       settings
@@ -2154,6 +2327,7 @@ var LegacyOpenTelemetry = class {
       stepContext: void 0,
       embedSpans: /* @__PURE__ */ new Map(),
       rerankSpan: void 0,
+      evaluationSpan: void 0,
       toolSpans: /* @__PURE__ */ new Map(),
       baseTelemetryAttributes,
       settings
@@ -2529,6 +2703,7 @@ var LegacyOpenTelemetry = class {
       stepContext: void 0,
       embedSpans: /* @__PURE__ */ new Map(),
       rerankSpan: void 0,
+      evaluationSpan: void 0,
       toolSpans: /* @__PURE__ */ new Map(),
       baseTelemetryAttributes,
       settings
@@ -2578,6 +2753,110 @@ var LegacyOpenTelemetry = class {
     span.end();
     state.rerankSpan = void 0;
   }
+  onEvaluateOperationStart(event) {
+    const telemetry = {
+      recordInputs: event.recordInputs,
+      recordOutputs: event.recordOutputs,
+      functionId: event.functionId
+    };
+    const settings = {
+      maxRetries: event.maxRetries
+    };
+    const baseTelemetryAttributes = getBaseTelemetryAttributes({
+      model: { provider: event.provider, modelId: event.modelId },
+      headers: event.headers,
+      settings,
+      context: event.runtimeContext
+    });
+    const attributes = selectAttributes2(telemetry, {
+      ...assembleOperationName({ operationId: event.operationId, telemetry }),
+      ...baseTelemetryAttributes,
+      "ai.evaluation.state": {
+        input: () => JSON.stringify(event.state)
+      },
+      "ai.evaluation.questions": {
+        input: () => JSON.stringify(event.questions)
+      }
+    });
+    const rootSpan = this.tracer.startSpan(event.operationId, { attributes });
+    const rootContext = trace2.setSpan(context3.active(), rootSpan);
+    this.callStates.set(event.callId, {
+      operationId: event.operationId,
+      telemetry,
+      rootSpan,
+      rootContext,
+      stepSpan: void 0,
+      stepContext: void 0,
+      embedSpans: /* @__PURE__ */ new Map(),
+      rerankSpan: void 0,
+      evaluationSpan: void 0,
+      toolSpans: /* @__PURE__ */ new Map(),
+      baseTelemetryAttributes,
+      settings
+    });
+  }
+  onEvaluateOperationEnd(event) {
+    const state = this.getCallState(event.callId);
+    if (!(state == null ? void 0 : state.rootSpan)) return;
+    state.rootSpan.setAttributes(
+      selectAttributes2(state.telemetry, {
+        "ai.evaluation.answers": {
+          output: () => JSON.stringify(event.answers)
+        }
+      })
+    );
+    state.rootSpan.end();
+    this.cleanupCallState(event.callId);
+  }
+  experimental_onEvaluateStart(event) {
+    this.onEvaluateOperationStart(event);
+  }
+  experimental_onEvaluateEnd(event) {
+    this.onEvaluateOperationEnd(event);
+  }
+  experimental_onEvaluationModelCallStart(event) {
+    const state = this.getCallState(event.callId);
+    if (!(state == null ? void 0 : state.rootSpan) || !state.rootContext) return;
+    const attributes = selectAttributes2(state.telemetry, {
+      ...assembleOperationName({
+        operationId: event.operationId,
+        telemetry: state.telemetry
+      }),
+      ...state.baseTelemetryAttributes,
+      "ai.evaluation.state": {
+        input: () => JSON.stringify(event.state)
+      },
+      "ai.evaluation.questions": {
+        input: () => JSON.stringify(event.questions)
+      }
+    });
+    const span = this.tracer.startSpan(
+      event.operationId,
+      { attributes },
+      state.rootContext
+    );
+    state.evaluationSpan = {
+      span,
+      context: trace2.setSpan(state.rootContext, span)
+    };
+  }
+  experimental_onEvaluationModelCallEnd(event) {
+    var _a, _b;
+    const state = this.getCallState(event.callId);
+    if (!(state == null ? void 0 : state.evaluationSpan)) return;
+    state.evaluationSpan.span.setAttributes(
+      selectAttributes2(state.telemetry, {
+        "ai.evaluation.answers": {
+          output: () => JSON.stringify(event.answers)
+        },
+        "ai.usage.inputTokens": (_a = event.usage) == null ? void 0 : _a.inputTokens,
+        "ai.usage.outputTokens": (_b = event.usage) == null ? void 0 : _b.outputTokens,
+        "ai.response.providerMetadata": event.providerMetadata ? JSON.stringify(event.providerMetadata) : void 0
+      })
+    );
+    state.evaluationSpan.span.end();
+    state.evaluationSpan = void 0;
+  }
   onAbort(event) {
     const state = this.getCallState(event.callId);
     if (!(state == null ? void 0 : state.rootSpan)) return;
@@ -2598,6 +2877,10 @@ var LegacyOpenTelemetry = class {
       state.rerankSpan.span.end();
       state.rerankSpan = void 0;
     }
+    if (state.evaluationSpan) {
+      state.evaluationSpan.span.end();
+      state.evaluationSpan = void 0;
+    }
     state.rootSpan.end();
     this.cleanupCallState(event.callId);
   }
@@ -2622,6 +2905,11 @@ var LegacyOpenTelemetry = class {
       state.rerankSpan.span.end();
       state.rerankSpan = void 0;
     }
+    if (state.evaluationSpan) {
+      recordSpanError(state.evaluationSpan.span, actualError);
+      state.evaluationSpan.span.end();
+      state.evaluationSpan = void 0;
+    }
     recordSpanError(state.rootSpan, actualError);
     state.rootSpan.end();
     this.cleanupCallState(event.callId);