@latitude-data/telemetry 2.0.2 → 2.0.4

package/dist/index.js

@@ -198,17 +198,23 @@ const expectedOutputConfiguration = z.object({
     fieldAccessor: z.string().optional(), // Field accessor to get the output from if it's a key-value format
 });
 const EVALUATION_TRIGGER_TARGETS = ['first', 'every', 'last'];
-const DEFAULT_LAST_INTERACTION_DEBOUNCE_SECONDS = 120;
 const LAST_INTERACTION_DEBOUNCE_MIN_SECONDS = 30;
 const LAST_INTERACTION_DEBOUNCE_MAX_SECONDS = 60 * 60 * 24; // 1 day
+const MIN_EVALUATION_SAMPLE_RATE = 0; // 0%
+const MAX_EVALUATION_SAMPLE_RATE = 100; // 100%
 const triggerConfiguration = z.object({
     target: z.enum(EVALUATION_TRIGGER_TARGETS),
     lastInteractionDebounce: z
         .number()
         .min(LAST_INTERACTION_DEBOUNCE_MIN_SECONDS)
         .max(LAST_INTERACTION_DEBOUNCE_MAX_SECONDS)
-        .optional()
-        .default(DEFAULT_LAST_INTERACTION_DEBOUNCE_SECONDS),
+        .optional(),
+    sampleRate: z
+        .number()
+        .int()
+        .min(MIN_EVALUATION_SAMPLE_RATE)
+        .max(MAX_EVALUATION_SAMPLE_RATE)
+        .optional(),
 });
 const baseEvaluationConfiguration = z.object({
     reverseScale: z.boolean(), // If true, lower is better, otherwise, higher is better
@@ -239,6 +245,7 @@ const compositeEvaluationResultMetadata = baseEvaluationResultMetadata.extend({
         score: z.number(), // Normalized score
         reason: z.string(),
         passed: z.boolean(),
+        tokens: z.number().optional(), // Optional llm evaluation usage
     })),
 });
 const compositeEvaluationResultError = baseEvaluationResultError.extend({
@@ -1151,6 +1158,12 @@ const OptimizationBudgetSchema = z.object({
     tokens: z.number().min(0).optional(),
 });
 z.object({
+    dataset: z
+        .object({
+        target: z.number().min(0).optional(), // Note: number of rows to curate when not provided by the user
+        label: z.string().optional(), // Note: expected output column when using a labeled evaluation
+    })
+        .optional(),
     parameters: z
         .record(z.string(), z.object({
         column: z.string().optional(), // Note: corresponding column in the user-provided trainset and testset
@@ -1594,6 +1607,7 @@ class ManualInstrumentation {
             attributes,
         });
     }
+    // TODO(tracing): deprecate
     unresolvedExternal(ctx, { path, projectId, versionUuid, conversationUuid, name, ...rest }) {
         const attributes = {
             [ATTRIBUTES.LATITUDE.promptPath]: path,
@@ -2042,7 +2056,22 @@ class LatitudeTelemetry {
         if (!DOCUMENT_PATH_REGEXP.test(options.path)) {
             throw new BadRequestError("Invalid path, no spaces. Only letters, numbers, '.', '-' and '_'");
         }
-        const span = this.manualInstrumentation.unresolvedExternal(BACKGROUND(), options);
+        const captureBaggageEntries = {
+            [ATTRIBUTES.LATITUDE.promptPath]: { value: options.path },
+            [ATTRIBUTES.LATITUDE.projectId]: { value: String(options.projectId) },
+        };
+        if (options.versionUuid) {
+            captureBaggageEntries[ATTRIBUTES.LATITUDE.commitUuid] = {
+                value: options.versionUuid,
+            };
+        }
+        if (options.conversationUuid) {
+            captureBaggageEntries[ATTRIBUTES.LATITUDE.documentLogUuid] = {
+                value: options.conversationUuid,
+            };
+        }
+        const captureContext = propagation.setBaggage(BACKGROUND(), propagation.createBaggage(captureBaggageEntries));
+        const span = this.manualInstrumentation.unresolvedExternal(captureContext, options);
         let result;
         try {
             result = await context.with(span.context, async () => await fn(span.context));