@latitude-data/telemetry 2.0.1 → 2.0.4

package/dist/index.cjs

@@ -218,17 +218,23 @@ const expectedOutputConfiguration = zod.z.object({
     fieldAccessor: zod.z.string().optional(), // Field accessor to get the output from if it's a key-value format
 });
 const EVALUATION_TRIGGER_TARGETS = ['first', 'every', 'last'];
-const DEFAULT_LAST_INTERACTION_DEBOUNCE_SECONDS = 120;
 const LAST_INTERACTION_DEBOUNCE_MIN_SECONDS = 30;
 const LAST_INTERACTION_DEBOUNCE_MAX_SECONDS = 60 * 60 * 24; // 1 day
+const MIN_EVALUATION_SAMPLE_RATE = 0; // 0%
+const MAX_EVALUATION_SAMPLE_RATE = 100; // 100%
 const triggerConfiguration = zod.z.object({
     target: zod.z.enum(EVALUATION_TRIGGER_TARGETS),
     lastInteractionDebounce: zod.z
         .number()
         .min(LAST_INTERACTION_DEBOUNCE_MIN_SECONDS)
         .max(LAST_INTERACTION_DEBOUNCE_MAX_SECONDS)
-        .optional()
-        .default(DEFAULT_LAST_INTERACTION_DEBOUNCE_SECONDS),
+        .optional(),
+    sampleRate: zod.z
+        .number()
+        .int()
+        .min(MIN_EVALUATION_SAMPLE_RATE)
+        .max(MAX_EVALUATION_SAMPLE_RATE)
+        .optional(),
 });
 const baseEvaluationConfiguration = zod.z.object({
     reverseScale: zod.z.boolean(), // If true, lower is better, otherwise, higher is better
@@ -259,6 +265,7 @@ const compositeEvaluationResultMetadata = baseEvaluationResultMetadata.extend({
         score: zod.z.number(), // Normalized score
         reason: zod.z.string(),
         passed: zod.z.boolean(),
+        tokens: zod.z.number().optional(), // Optional llm evaluation usage
     })),
 });
 const compositeEvaluationResultError = baseEvaluationResultError.extend({
@@ -1171,6 +1178,12 @@ const OptimizationBudgetSchema = zod.z.object({
     tokens: zod.z.number().min(0).optional(),
 });
 zod.z.object({
+    dataset: zod.z
+        .object({
+        target: zod.z.number().min(0).optional(), // Note: number of rows to curate when not provided by the user
+        label: zod.z.string().optional(), // Note: expected output column when using a labeled evaluation
+    })
+        .optional(),
     parameters: zod.z
         .record(zod.z.string(), zod.z.object({
         column: zod.z.string().optional(), // Note: corresponding column in the user-provided trainset and testset
@@ -1614,6 +1627,7 @@ class ManualInstrumentation {
             attributes,
         });
     }
+    // TODO(tracing): deprecate
     unresolvedExternal(ctx, { path, projectId, versionUuid, conversationUuid, name, ...rest }) {
         const attributes = {
             [ATTRIBUTES.LATITUDE.promptPath]: path,
@@ -2062,7 +2076,22 @@ class LatitudeTelemetry {
         if (!DOCUMENT_PATH_REGEXP.test(options.path)) {
             throw new BadRequestError("Invalid path, no spaces. Only letters, numbers, '.', '-' and '_'");
         }
-        const span = this.manualInstrumentation.unresolvedExternal(BACKGROUND(), options);
+        const captureBaggageEntries = {
+            [ATTRIBUTES.LATITUDE.promptPath]: { value: options.path },
+            [ATTRIBUTES.LATITUDE.projectId]: { value: String(options.projectId) },
+        };
+        if (options.versionUuid) {
+            captureBaggageEntries[ATTRIBUTES.LATITUDE.commitUuid] = {
+                value: options.versionUuid,
+            };
+        }
+        if (options.conversationUuid) {
+            captureBaggageEntries[ATTRIBUTES.LATITUDE.documentLogUuid] = {
+                value: options.conversationUuid,
+            };
+        }
+        const captureContext = otel.propagation.setBaggage(BACKGROUND(), otel.propagation.createBaggage(captureBaggageEntries));
+        const span = this.manualInstrumentation.unresolvedExternal(captureContext, options);
         let result;
         try {
             result = await otel.context.with(span.context, async () => await fn(span.context));