@latitude-data/telemetry 2.0.1 → 3.0.0-beta.0

package/dist/index.d.ts

@@ -18,23 +18,6 @@ type LatitudeInstrumentationOptions = {
     completions?: boolean;
 };
 
-declare enum LogSources {
-    API = "api",
-    AgentAsTool = "agent_as_tool",
-    Copilot = "copilot",
-    EmailTrigger = "email_trigger",
-    Evaluation = "evaluation",
-    Experiment = "experiment",
-    IntegrationTrigger = "integration_trigger",
-    Playground = "playground",
-    ScheduledTrigger = "scheduled_trigger",
-    SharedPrompt = "shared_prompt",
-    ShadowTest = "shadow_test",
-    ABTestChallenger = "ab_test_challenger",
-    User = "user",
-    Optimization = "optimization"
-}
-
 declare const traceContextSchema: z.ZodObject<{
     traceparent: z.ZodString;
     tracestate: z.ZodOptional<z.ZodString>;
@@ -52,12 +35,13 @@ type EndSpanOptions = {
 type ErrorOptions = {
     attributes?: otel.Attributes;
 };
-type StartToolSpanOptions = StartSpanOptions & {
+type StartToolSpanOptions = {
     name: string;
     call: {
         id: string;
         arguments: Record<string, unknown>;
     };
+    attributes?: otel.Attributes;
 };
 type EndToolSpanOptions = EndSpanOptions & {
     result: {
@@ -65,14 +49,13 @@ type EndToolSpanOptions = EndSpanOptions & {
         isError: boolean;
     };
 };
-type StartCompletionSpanOptions = StartSpanOptions & {
+type StartCompletionSpanOptions = {
+    name?: string;
     provider: string;
     model: string;
     configuration?: Record<string, unknown>;
     input?: string | Record<string, unknown>[];
-    versionUuid?: string;
-    promptUuid?: string;
-    experimentUuid?: string;
+    attributes?: otel.Attributes;
 };
 type EndCompletionSpanOptions = EndSpanOptions & {
     output?: string | Record<string, unknown>[];
@@ -84,13 +67,15 @@ type EndCompletionSpanOptions = EndSpanOptions & {
     };
     finishReason?: string;
 };
-type StartHttpSpanOptions = StartSpanOptions & {
+type StartHttpSpanOptions = {
+    name?: string;
     request: {
         method: string;
         url: string;
         headers: Record<string, string>;
         body: string | Record<string, unknown>;
     };
+    attributes?: otel.Attributes;
 };
 type EndHttpSpanOptions = EndSpanOptions & {
     response: {
@@ -99,37 +84,28 @@ type EndHttpSpanOptions = EndSpanOptions & {
         body: string | Record<string, unknown>;
     };
 };
-type PromptSpanOptions = StartSpanOptions & {
-    documentLogUuid: string;
-    versionUuid?: string;
-    promptUuid: string;
-    projectId?: number;
-    experimentUuid?: string;
-    testDeploymentId?: number;
-    externalId?: string;
+type PromptSpanOptions = {
+    name?: string;
     template: string;
     parameters?: Record<string, unknown>;
-    source?: LogSources;
+    attributes?: otel.Attributes;
 };
-type ChatSpanOptions = StartSpanOptions & {
-    documentLogUuid: string;
-    previousTraceId: string;
-    source?: LogSources;
-    versionUuid?: string;
-    promptUuid?: string;
+type ChatSpanOptions = {
+    name?: string;
+    attributes?: otel.Attributes;
 };
-type ExternalSpanOptions = StartSpanOptions & {
-    promptUuid: string;
-    documentLogUuid: string;
-    source?: LogSources;
-    versionUuid?: string;
+type ExternalSpanOptions = {
+    name?: string;
     externalId?: string;
+    attributes?: otel.Attributes;
 };
-type CaptureOptions = StartSpanOptions & {
+type CaptureOptions = {
+    name?: string;
     path: string;
     projectId: number;
     versionUuid?: string;
     conversationUuid?: string;
+    attributes?: otel.Attributes;
 };
 type ManualInstrumentationOptions = {
     provider?: Provider;
@@ -172,22 +148,22 @@ declare class ManualInstrumentation implements BaseInstrumentation {
         context: otel.Context;
         fail: (_error: Error, _options?: ErrorOptions) => void;
     };
-    prompt(ctx: otel.Context, { documentLogUuid, versionUuid, promptUuid, projectId, experimentUuid, testDeploymentId, externalId, template, parameters, name, source, ...rest }: PromptSpanOptions): {
+    prompt(ctx: otel.Context, options: PromptSpanOptions): {
         context: otel.Context;
         end: (_options?: EndSpanOptions) => void;
         fail: (_error: Error, _options?: ErrorOptions) => void;
     };
-    chat(ctx: otel.Context, { documentLogUuid, previousTraceId, source, name, versionUuid, promptUuid, ...rest }: ChatSpanOptions): {
+    chat(ctx: otel.Context, options: ChatSpanOptions): {
         context: otel.Context;
         end: (_options?: EndSpanOptions) => void;
         fail: (_error: Error, _options?: ErrorOptions) => void;
     };
-    external(ctx: otel.Context, { promptUuid, documentLogUuid, source, versionUuid, externalId, name, ...rest }: ExternalSpanOptions): {
+    external(ctx: otel.Context, options: ExternalSpanOptions): {
         context: otel.Context;
         end: (_options?: EndSpanOptions) => void;
         fail: (_error: Error, _options?: ErrorOptions) => void;
     };
-    unresolvedExternal(ctx: otel.Context, { path, projectId, versionUuid, conversationUuid, name, ...rest }: CaptureOptions): {
+    unresolvedExternal(ctx: otel.Context, options: CaptureOptions): {
         context: otel.Context;
         end: (_options?: EndSpanOptions) => void;
         fail: (_error: Error, _options?: ErrorOptions) => void;
@@ -262,6 +238,40 @@ declare class ContextManager {
     resume(ctx: TraceContext): otel.Context;
     active(): otel.Context;
     with<A extends unknown[], F extends (...args: A) => ReturnType<F>>(ctx: TelemetryContext, fn: F, thisArg?: ThisParameterType<F>, ...args: A): ReturnType<F>;
+    /**
+     * Sets custom attributes in the OpenTelemetry baggage that will be
+     * automatically propagated to all child spans in the trace.
+     *
+     * This is useful for setting trace-level metadata that should be
+     * inherited by all spans without manually passing them to each span.
+     *
+     * Example:
+     * ```typescript
+     * const ctx = telemetry.context.setAttributes(
+     *   telemetry.context.active(),
+     *   {
+     *     'latitude.documentLogUuid': 'uuid-123',
+     *     'latitude.documentUuid': 'prompt-456',
+     *     'latitude.commitUuid': 'commit-789',
+     *     'latitude.projectId': '123',
+     *     'custom.attribute': 'value'
+     *   }
+     * )
+     *
+     * // Latitude keys are normalized to snake_case in baggage:
+     * // latitude.document_log_uuid, latitude.document_uuid, ...
+     *
+     * // Use context.with() to make the context active
+     * // All spans created within this context will automatically
+     * // inherit the baggage attributes without explicitly passing ctx
+     * telemetry.context.with(ctx, () => {
+     *   telemetry.span.tool({ name: 'my-tool', call: { id: '1', arguments: {} } })
+     *   // ... other spans
+     * })
+     * ```
+     */
+    setAttributes(ctx: TelemetryContext, attributes: Record<string, string>): TelemetryContext;
+    private normalizeBaggageKey;
 }
 declare class InstrumentationManager {
     private readonly instrumentations;

package/dist/index.js

@@ -4,7 +4,6 @@ import { ATTR_HTTP_REQUEST_METHOD, ATTR_HTTP_RESPONSE_STATUS_CODE, ATTR_SERVICE_
 import * as otel from '@opentelemetry/api';
 import { trace, propagation, context } from '@opentelemetry/api';
 import { Translator, Provider } from 'rosetta-ai';
-import { v4 } from 'uuid';
 import { BaggageSpanProcessor, ALLOW_ALL_BAGGAGE_KEYS } from '@opentelemetry/baggage-span-processor';
 import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
 import { CompositePropagator, W3CTraceContextPropagator, W3CBaggagePropagator } from '@opentelemetry/core';
@@ -198,17 +197,23 @@ const expectedOutputConfiguration = z.object({
     fieldAccessor: z.string().optional(), // Field accessor to get the output from if it's a key-value format
 });
 const EVALUATION_TRIGGER_TARGETS = ['first', 'every', 'last'];
-const DEFAULT_LAST_INTERACTION_DEBOUNCE_SECONDS = 120;
 const LAST_INTERACTION_DEBOUNCE_MIN_SECONDS = 30;
 const LAST_INTERACTION_DEBOUNCE_MAX_SECONDS = 60 * 60 * 24; // 1 day
+const MIN_EVALUATION_SAMPLE_RATE = 0; // 0%
+const MAX_EVALUATION_SAMPLE_RATE = 100; // 100%
 const triggerConfiguration = z.object({
     target: z.enum(EVALUATION_TRIGGER_TARGETS),
     lastInteractionDebounce: z
         .number()
         .min(LAST_INTERACTION_DEBOUNCE_MIN_SECONDS)
         .max(LAST_INTERACTION_DEBOUNCE_MAX_SECONDS)
-        .optional()
-        .default(DEFAULT_LAST_INTERACTION_DEBOUNCE_SECONDS),
+        .optional(),
+    sampleRate: z
+        .number()
+        .int()
+        .min(MIN_EVALUATION_SAMPLE_RATE)
+        .max(MAX_EVALUATION_SAMPLE_RATE)
+        .optional(),
 });
 const baseEvaluationConfiguration = z.object({
     reverseScale: z.boolean(), // If true, lower is better, otherwise, higher is better
@@ -921,16 +926,11 @@ const ATTRIBUTES = {
     // Custom attributes added and used by Latitude spans (Prompt / External / Chat)
     LATITUDE: {
         type: 'latitude.type',
-        documentUuid: 'latitude.document_uuid',
         promptPath: 'latitude.prompt_path',
         commitUuid: 'latitude.commit_uuid',
         documentLogUuid: 'latitude.document_log_uuid',
         projectId: 'latitude.project_id',
-        experimentUuid: 'latitude.experiment_uuid',
-        source: 'latitude.source',
         externalId: 'latitude.external_id',
-        testDeploymentId: 'latitude.test_deployment_id',
-        previousTraceId: 'latitude.previous_trace_id',
         // Custom additions to the GenAI semantic conventions (deprecated)
         request: {
             _root: 'gen_ai.request',
@@ -1151,6 +1151,12 @@ const OptimizationBudgetSchema = z.object({
     tokens: z.number().min(0).optional(),
 });
 z.object({
+    dataset: z
+        .object({
+        target: z.number().min(0).optional(), // Note: number of rows to curate when not provided by the user
+        label: z.string().optional(), // Note: expected output column when using a labeled evaluation
+    })
+        .optional(),
     parameters: z
         .record(z.string(), z.object({
         column: z.string().optional(), // Note: corresponding column in the user-provided trainset and testset
@@ -1168,7 +1174,6 @@ z.object({
 });
 
 // TODO(tracing): deprecated
-const HEAD_COMMIT = 'live';
 var Providers;
 (function (Providers) {
     Providers["OpenAI"] = "openai";
@@ -1323,6 +1328,7 @@ class ManualInstrumentation {
             jsonArguments = '{}';
         }
         const span = this.span(ctx, start.name, SpanType.Tool, {
+            ...start,
             attributes: {
                 [ATTRIBUTES.OPENTELEMETRY.GEN_AI._deprecated.tool.name]: start.name,
                 [ATTRIBUTES.OPENTELEMETRY.GEN_AI._deprecated.tool.type]: VALUES.OPENTELEMETRY.GEN_AI.tool.type.function,
@@ -1410,6 +1416,7 @@ class ManualInstrumentation {
             jsonInput = '[]';
         }
         const span = this.span(ctx, start.name || `${start.provider} / ${start.model}`, SpanType.Completion, {
+            ...start,
             attributes: {
                 [ATTRIBUTES.OPENTELEMETRY.GEN_AI._deprecated.system]: start.provider,
                 [ATTRIBUTES.LATITUDE.request.configuration]: jsonConfiguration,
@@ -1417,9 +1424,6 @@ class ManualInstrumentation {
                 [ATTRIBUTES.OPENTELEMETRY.GEN_AI.systemInstructions]: jsonSystem,
                 [ATTRIBUTES.OPENTELEMETRY.GEN_AI.input.messages]: jsonInput,
                 ...(start.attributes || {}),
-                [ATTRIBUTES.LATITUDE.commitUuid]: start.versionUuid,
-                [ATTRIBUTES.LATITUDE.documentUuid]: start.promptUuid,
-                [ATTRIBUTES.LATITUDE.experimentUuid]: start.experimentUuid,
             },
         });
         return {
@@ -1502,6 +1506,7 @@ class ManualInstrumentation {
             }
         }
         const span = this.span(ctx, start.name || `${method} ${start.request.url}`, SpanType.Http, {
+            ...start,
             attributes: {
                 [ATTRIBUTES.OPENTELEMETRY.HTTP.request.method]: method,
                 [ATTRIBUTES.OPENTELEMETRY.HTTP.request.url]: start.request.url,
@@ -1539,7 +1544,8 @@ class ManualInstrumentation {
             },
         };
     }
-    prompt(ctx, { documentLogUuid, versionUuid, promptUuid, projectId, experimentUuid, testDeploymentId, externalId, template, parameters, name, source, ...rest }) {
+    prompt(ctx, options) {
+        const { template, parameters, name, attributes: userAttributes } = options;
         let jsonParameters = '';
         try {
             jsonParameters = JSON.stringify(parameters || {});
@@ -1550,51 +1556,27 @@ class ManualInstrumentation {
         const attributes = {
             [ATTRIBUTES.LATITUDE.request.template]: template,
             [ATTRIBUTES.LATITUDE.request.parameters]: jsonParameters,
-            [ATTRIBUTES.LATITUDE.commitUuid]: versionUuid || HEAD_COMMIT,
-            [ATTRIBUTES.LATITUDE.documentUuid]: promptUuid,
-            [ATTRIBUTES.LATITUDE.projectId]: projectId,
-            [ATTRIBUTES.LATITUDE.documentLogUuid]: documentLogUuid,
-            ...(experimentUuid && {
-                [ATTRIBUTES.LATITUDE.experimentUuid]: experimentUuid,
-            }),
-            ...(testDeploymentId && {
-                [ATTRIBUTES.LATITUDE.testDeploymentId]: testDeploymentId,
-            }),
-            ...(externalId && { [ATTRIBUTES.LATITUDE.externalId]: externalId }),
-            ...(source && { [ATTRIBUTES.LATITUDE.source]: source }),
-            ...(rest.attributes || {}),
+            ...(userAttributes || {}),
         };
-        return this.span(ctx, name || `prompt-${promptUuid}`, SpanType.Prompt, {
-            attributes,
-        });
+        return this.span(ctx, name || 'prompt', SpanType.Prompt, { attributes });
     }
-    chat(ctx, { documentLogUuid, previousTraceId, source, name, versionUuid, promptUuid, ...rest }) {
+    chat(ctx, options) {
+        const { name, attributes: userAttributes } = options;
         const attributes = {
-            [ATTRIBUTES.LATITUDE.documentLogUuid]: documentLogUuid,
-            [ATTRIBUTES.LATITUDE.previousTraceId]: previousTraceId,
-            ...(versionUuid && { [ATTRIBUTES.LATITUDE.commitUuid]: versionUuid }),
-            ...(promptUuid && { [ATTRIBUTES.LATITUDE.documentUuid]: promptUuid }),
-            ...(source && { [ATTRIBUTES.LATITUDE.source]: source }),
-            ...(rest.attributes || {}),
+            ...(userAttributes || {}),
         };
-        return this.span(ctx, name || `chat-${documentLogUuid}`, SpanType.Chat, {
-            attributes,
-        });
+        return this.span(ctx, name || 'chat', SpanType.Chat, { attributes });
     }
-    external(ctx, { promptUuid, documentLogUuid, source, versionUuid, externalId, name, ...rest }) {
+    external(ctx, options) {
+        const { externalId, name, attributes: userAttributes } = options;
         const attributes = {
-            [ATTRIBUTES.LATITUDE.documentUuid]: promptUuid,
-            [ATTRIBUTES.LATITUDE.documentLogUuid]: documentLogUuid,
-            [ATTRIBUTES.LATITUDE.source]: source ?? LogSources.API,
-            ...(versionUuid && { [ATTRIBUTES.LATITUDE.commitUuid]: versionUuid }),
             ...(externalId && { [ATTRIBUTES.LATITUDE.externalId]: externalId }),
-            ...(rest.attributes || {}),
+            ...(userAttributes || {}),
         };
-        return this.span(ctx, name || `external-${promptUuid}`, SpanType.External, {
-            attributes,
-        });
+        return this.span(ctx, name || 'external', SpanType.External, { attributes });
     }
-    unresolvedExternal(ctx, { path, projectId, versionUuid, conversationUuid, name, ...rest }) {
+    unresolvedExternal(ctx, options) {
+        const { path, projectId, versionUuid, conversationUuid, name, attributes: userAttributes, } = options;
         const attributes = {
             [ATTRIBUTES.LATITUDE.promptPath]: path,
             [ATTRIBUTES.LATITUDE.projectId]: projectId,
@@ -1602,7 +1584,7 @@ class ManualInstrumentation {
             ...(conversationUuid && {
                 [ATTRIBUTES.LATITUDE.documentLogUuid]: conversationUuid,
             }),
-            ...(rest.attributes || {}),
+            ...(userAttributes || {}),
         };
         return this.span(ctx, name || `capture-${path}`, SpanType.UnresolvedExternal, { attributes });
     }
@@ -1648,9 +1630,6 @@ class LatitudeInstrumentation {
     async wrapRenderChain(fn, ...args) {
         const { prompt, parameters } = args[0];
         const $prompt = this.manualTelemetry.prompt(context.active(), {
-            documentLogUuid: v4(),
-            versionUuid: prompt.versionUuid,
-            promptUuid: prompt.uuid,
             template: prompt.content,
             parameters: parameters,
         });
@@ -1854,6 +1833,51 @@ class ContextManager {
     with(ctx, fn, thisArg, ...args) {
         return context.with(ctx, fn, thisArg, ...args);
     }
+    /**
+     * Sets custom attributes in the OpenTelemetry baggage that will be
+     * automatically propagated to all child spans in the trace.
+     *
+     * This is useful for setting trace-level metadata that should be
+     * inherited by all spans without manually passing them to each span.
+     *
+     * Example:
+     * ```typescript
+     * const ctx = telemetry.context.setAttributes(
+     *   telemetry.context.active(),
+     *   {
+     *     'latitude.documentLogUuid': 'uuid-123',
+     *     'latitude.documentUuid': 'prompt-456',
+     *     'latitude.commitUuid': 'commit-789',
+     *     'latitude.projectId': '123',
+     *     'custom.attribute': 'value'
+     *   }
+     * )
+     *
+     * // Latitude keys are normalized to snake_case in baggage:
+     * // latitude.document_log_uuid, latitude.document_uuid, ...
+     *
+     * // Use context.with() to make the context active
+     * // All spans created within this context will automatically
+     * // inherit the baggage attributes without explicitly passing ctx
+     * telemetry.context.with(ctx, () => {
+     *   telemetry.span.tool({ name: 'my-tool', call: { id: '1', arguments: {} } })
+     *   // ... other spans
+     * })
+     * ```
+     */
+    setAttributes(ctx, attributes) {
+        const baggage = propagation.getBaggage(ctx) || propagation.createBaggage({});
+        const newBaggage = Object.entries(attributes).reduce((bag, [key, value]) => bag.setEntry(this.normalizeBaggageKey(key), { value }), baggage);
+        return propagation.setBaggage(ctx, newBaggage);
+    }
+    normalizeBaggageKey(key) {
+        if (!key.startsWith('latitude.'))
+            return key;
+        const [namespace, ...path] = key.split('.');
+        if (!namespace || path.length === 0)
+            return key;
+        return [namespace, ...path.map(toSnakeCase)].join('.');
+    }
 }
 class InstrumentationManager {
     instrumentations;