braintrust 0.0.89 → 0.0.91

package/dist/browser.js

@@ -73,6 +73,10 @@ var v4_default = v4;
 // ../core/js/dist/index.mjs
 var TRANSACTION_ID_FIELD = "_xact_id";
 var IS_MERGE_FIELD = "_is_merge";
+var AUDIT_SOURCE_FIELD = "_audit_source";
+var AUDIT_METADATA_FIELD = "_audit_metadata";
+var VALID_SOURCES = ["app", "api", "external"];
+var PARENT_ID_FIELD = "_parent_id";
 function mergeDicts(mergeInto, mergeFrom) {
   for (const [k, mergeFromV] of Object.entries(mergeFrom)) {
     const mergeIntoV = mergeInto[k];
@@ -85,6 +89,7 @@ function mergeDicts(mergeInto, mergeFrom) {
       mergeInto[k] = mergeFromV;
     }
   }
+  return mergeInto;
 }
 function generateMergedRowKey(row) {
   return JSON.stringify(
@@ -126,6 +131,15 @@ function mergeRowBatch(rows) {
   out.push(...Object.values(rowGroups));
   return out;
 }
+var SpanTypeAttribute = /* @__PURE__ */ ((SpanTypeAttribute2) => {
+  SpanTypeAttribute2["LLM"] = "llm";
+  SpanTypeAttribute2["SCORE"] = "score";
+  SpanTypeAttribute2["FUNCTION"] = "function";
+  SpanTypeAttribute2["EVAL"] = "eval";
+  SpanTypeAttribute2["TASK"] = "task";
+  SpanTypeAttribute2["TOOL"] = "tool";
+  return SpanTypeAttribute2;
+})(SpanTypeAttribute || {});
 
 // src/util.ts
 var GLOBAL_PROJECT = "Global";
@@ -148,6 +162,9 @@ function runFinally(f, finallyF) {
 function getCurrentUnixTimestamp() {
   return (/* @__PURE__ */ new Date()).getTime() / 1e3;
 }
+function isEmpty(a) {
+  return a === void 0 || a === null;
+}
 
 // src/logger.ts
 var NoopSpan = class {
@@ -159,6 +176,8 @@ var NoopSpan = class {
   }
   log(_) {
   }
+  logFeedback(event) {
+  }
   traced(callback, _1) {
     return callback(this);
   }
@@ -333,6 +352,70 @@ var HTTPConnection = class _HTTPConnection {
     return await resp.json();
   }
 };
+function logFeedbackImpl(bgLogger, parentIds, {
+  id,
+  expected,
+  scores,
+  metadata: inputMetadata,
+  comment,
+  source: inputSource
+}) {
+  const source = inputSource ?? "external";
+  if (!VALID_SOURCES.includes(source)) {
+    throw new Error(`source must be one of ${VALID_SOURCES}`);
+  }
+  if (isEmpty(scores) && isEmpty(expected) && isEmpty(comment)) {
+    throw new Error(
+      "At least one of scores, expected, or comment must be specified"
+    );
+  }
+  const validatedEvent = validateAndSanitizeExperimentLogPartialArgs({
+    scores,
+    metadata: inputMetadata,
+    expected
+  });
+  let { metadata, ...updateEvent } = validatedEvent;
+  updateEvent = Object.fromEntries(
+    Object.entries(updateEvent).filter(([_, v]) => !isEmpty(v))
+  );
+  const trueParentIds = (async () => {
+    const { kind, ...ids } = await parentIds;
+    return ids;
+  })();
+  if (Object.keys(updateEvent).length > 0) {
+    const record = (async () => {
+      return {
+        id,
+        ...updateEvent,
+        ...await trueParentIds,
+        [AUDIT_SOURCE_FIELD]: source,
+        [AUDIT_METADATA_FIELD]: metadata,
+        [IS_MERGE_FIELD]: true
+      };
+    })();
+    bgLogger.log([record]);
+  }
+  if (!isEmpty(comment)) {
+    const record = (async () => {
+      return {
+        id: v4_default(),
+        created: (/* @__PURE__ */ new Date()).toISOString(),
+        origin: {
+          // NOTE: We do not know (or care?) what the transaction id of the row that
+          // we're commenting on is here, so we omit it.
+          id
+        },
+        comment: {
+          text: comment
+        },
+        ...await trueParentIds,
+        [AUDIT_SOURCE_FIELD]: source,
+        [AUDIT_METADATA_FIELD]: metadata
+      };
+    })();
+    bgLogger.log([record]);
+  }
+}
 var Logger = class {
   constructor(lazyMetadata, logOptions = {}) {
     // For type identification.
@@ -411,6 +494,14 @@ var Logger = class {
       })();
     }
   }
+  async lazyParentIds() {
+    return {
+      kind: "project_log",
+      org_id: await this.org_id,
+      project_id: (await this.project).id,
+      log_id: "g"
+    };
+  }
   /**
    * Lower-level alternative to `traced`, which does not automatically end the span or mark it as current.
    *
@@ -418,19 +509,27 @@ var Logger = class {
    */
   startSpan(args) {
     const { name, ...argsRest } = args ?? {};
-    const parentIds = (async () => ({
-      kind: "project_log",
-      org_id: await this.org_id,
-      project_id: (await this.project).id,
-      log_id: "g"
-    }))();
     return new SpanImpl({
-      parentIds,
+      parentIds: this.lazyParentIds(),
       bgLogger: this.bgLogger,
       name: name ?? "root",
       ...argsRest
     });
   }
+  /**
+   * Log feedback to an event. Feedback is used to save feedback scores, set an expected value, or add a comment.
+   *
+   * @param event
+   * @param event.id The id of the event to log feedback for. This is the `id` returned by `log` or accessible as the `id` field of a span.
+   * @param event.scores (Optional) a dictionary of numeric values (between 0 and 1) to log. These scores will be merged into the existing scores for the event.
+   * @param event.expected (Optional) the ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not.
+   * @param event.comment (Optional) an optional comment string to log about the event.
+   * @param event.metadata (Optional) a dictionary with additional data about the feedback. If you have a `user_id`, you can log it here and access it in the Braintrust UI.
+   * @param event.source (Optional) the source of the feedback. Must be one of "external" (default), "app", or "api".
+   */
+  logFeedback(event) {
+    logFeedbackImpl(this.bgLogger, this.lazyParentIds(), event);
+  }
   /*
    * Flush any pending logs to the server.
    */
@@ -1022,6 +1121,13 @@ var Experiment = class {
       () => span.end()
     );
   }
+  async lazyParentIds() {
+    return {
+      kind: "experiment",
+      project_id: (await this.project).id,
+      experiment_id: await this.id
+    };
+  }
   /**
    * Lower-level alternative to `traced`, which does not automatically end the span or mark it as current.
    *
@@ -1029,13 +1135,8 @@ var Experiment = class {
    */
   startSpan(args) {
     const { name, ...argsRest } = args ?? {};
-    const parentIds = (async () => ({
-      kind: "experiment",
-      project_id: (await this.project).id,
-      experiment_id: await this.id
-    }))();
     return new SpanImpl({
-      parentIds,
+      parentIds: this.lazyParentIds(),
       bgLogger: this.bgLogger,
       name: name ?? "root",
       ...argsRest
@@ -1097,6 +1198,20 @@ var Experiment = class {
       metrics
     };
   }
+  /**
+   * Log feedback to an event in the experiment. Feedback is used to save feedback scores, set an expected value, or add a comment.
+   *
+   * @param event
+   * @param event.id The id of the event to log feedback for. This is the `id` returned by `log` or accessible as the `id` field of a span.
+   * @param event.scores (Optional) a dictionary of numeric values (between 0 and 1) to log. These scores will be merged into the existing scores for the event.
+   * @param event.expected (Optional) the ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not.
+   * @param event.comment (Optional) an optional comment string to log about the event.
+   * @param event.metadata (Optional) a dictionary with additional data about the feedback. If you have a `user_id`, you can log it here and access it in the Braintrust UI.
+   * @param event.source (Optional) the source of the feedback. Must be one of "external" (default), "app", or "api".
+   */
+  logFeedback(event) {
+    logFeedbackImpl(this.bgLogger, this.lazyParentIds(), event);
+  }
   /**
    * Flush any pending rows to the server.
    */
@@ -1147,10 +1262,12 @@ var SpanImpl = class _SpanImpl {
     this.rowIds = {
       id,
       span_id,
-      root_span_id: args.parentSpanInfo?.root_span_id ?? span_id
+      root_span_id: "parentSpanInfo" in args && args.parentSpanInfo?.root_span_id ? args.parentSpanInfo.root_span_id : span_id
     };
-    if (args.parentSpanInfo) {
+    if ("parentSpanInfo" in args && args.parentSpanInfo?.span_id) {
       this.internalData.span_parents = [args.parentSpanInfo.span_id];
+    } else if ("parentId" in args && !isEmpty(args.parentId)) {
+      this.rowIds[PARENT_ID_FIELD] = args.parentId;
     }
     this.isMerge = false;
     const { id: _id, ...eventRest } = args.event ?? {};
@@ -1174,16 +1291,26 @@ var SpanImpl = class _SpanImpl {
     if (sanitizedAndInternalData.metrics?.end) {
       this.loggedEndTime = sanitizedAndInternalData.metrics?.end;
     }
+    const parentIds = (async () => {
+      const { kind, ...ids } = await this.parentIds;
+      return ids;
+    })();
     const record = (async () => {
       return {
         ...sanitizedAndInternalData,
         ...this.rowIds,
-        ...await this.parentIds,
+        ...await parentIds,
         [IS_MERGE_FIELD]: this.isMerge
       };
     })();
     this.bgLogger.log([record]);
   }
+  logFeedback(event) {
+    logFeedbackImpl(this.bgLogger, this.parentIds, {
+      ...event,
+      id: this.id
+    });
+  }
   traced(callback, args) {
     const { setCurrent, ...argsRest } = args ?? {};
     const span = this.startSpan(argsRest);
@@ -1451,6 +1578,7 @@ function wrapOpenAI(openai) {
     return openai;
   }
 }
+globalThis.__inherited_braintrust_wrap_openai = wrapOpenAI;
 function wrapOpenAIv4(openai) {
   let completionProxy = new Proxy(openai.chat.completions, {
     get(target, name, receiver) {
@@ -1469,6 +1597,15 @@ function wrapOpenAIv4(openai) {
       return Reflect.get(target, name, receiver);
     }
   });
+  let embeddingProxy = new Proxy(openai.embeddings, {
+    get(target, name, receiver) {
+      const baseVal = Reflect.get(target, name, receiver);
+      if (name === "create") {
+        return wrapEmbeddings(baseVal.bind(target));
+      }
+      return baseVal;
+    }
+  });
   let betaProxy;
   if (openai.beta?.chat?.completions?.stream) {
     let betaChatCompletionProxy = new Proxy(openai?.beta?.chat.completions, {
@@ -1502,6 +1639,9 @@ function wrapOpenAIv4(openai) {
       if (name === "chat") {
         return chatProxy;
       }
+      if (name === "embeddings") {
+        return embeddingProxy;
+      }
       if (name === "beta" && betaProxy) {
         return betaProxy;
       }
@@ -1511,10 +1651,13 @@ function wrapOpenAIv4(openai) {
   return proxy;
 }
 function wrapBetaChatCompletion(completion) {
-  return async (params) => {
+  return (params) => {
     const { messages, ...rest } = params;
     const span = startSpan({
       name: "OpenAI Chat Completion",
+      spanAttributes: {
+        type: SpanTypeAttribute.LLM
+      },
       event: {
         input: messages,
         metadata: {
@@ -1523,7 +1666,7 @@ function wrapBetaChatCompletion(completion) {
       }
     });
     const startTime = getCurrentUnixTimestamp();
-    const ret = await completion(params);
+    const ret = completion(params);
     let first = true;
     ret.on("chunk", (_chunk) => {
       if (first) {
@@ -1548,10 +1691,13 @@ function wrapBetaChatCompletion(completion) {
   };
 }
 function wrapChatCompletion(completion) {
-  return async (params) => {
+  return async (params, options) => {
     const { messages, ...rest } = params;
     const span = startSpan({
       name: "OpenAI Chat Completion",
+      spanAttributes: {
+        type: SpanTypeAttribute.LLM
+      },
       event: {
         input: messages,
         metadata: {
@@ -1561,11 +1707,14 @@ function wrapChatCompletion(completion) {
     });
     if (params.stream) {
       const startTime = getCurrentUnixTimestamp();
-      const ret = await completion(params);
+      const ret = await completion(params, options);
       return new WrapperStream(span, startTime, ret);
     } else {
       try {
-        const ret = await completion(params);
+        const ret = await completion(
+          params,
+          options
+        );
         const { messages: messages2, ...rest2 } = params;
         span.log({
           input: messages2,
@@ -1586,6 +1735,37 @@ function wrapChatCompletion(completion) {
     }
   };
 }
+function wrapEmbeddings(create) {
+  return async (params, options) => {
+    const { input, ...rest } = params;
+    return traced(
+      async (span) => {
+        const result = await create(params, options);
+        const output = result.data[0];
+        span.log({
+          output,
+          metrics: {
+            tokens: result.usage?.total_tokens,
+            prompt_tokens: result.usage?.prompt_tokens
+          }
+        });
+        return result;
+      },
+      {
+        name: "OpenAI Embedding",
+        spanAttributes: {
+          type: SpanTypeAttribute.LLM
+        },
+        event: {
+          input,
+          metadata: {
+            ...rest
+          }
+        }
+      }
+    );
+  };
+}
 var WrapperStream = class {
   constructor(span, startTime, iter) {
     this.span = span;

package/dist/cli.js

@@ -9065,7 +9065,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "braintrust",
-      version: "0.0.89",
+      version: "0.0.91",
       description: "SDK for integrating Braintrust",
       main: "./dist/index.js",
       browser: {
@@ -9108,7 +9108,7 @@ var require_package = __commonJS({
         typescript: "^5.3.3"
       },
       dependencies: {
-        "@braintrust/core": "^0.0.8",
+        "@braintrust/core": "^0.0.12",
         argparse: "^2.0.1",
         chalk: "^4.1.2",
         "cli-progress": "^3.12.0",
@@ -10511,6 +10511,10 @@ var import_pluralize2 = __toESM(require_pluralize());
 // ../core/js/dist/index.mjs
 var TRANSACTION_ID_FIELD = "_xact_id";
 var IS_MERGE_FIELD = "_is_merge";
+var AUDIT_SOURCE_FIELD = "_audit_source";
+var AUDIT_METADATA_FIELD = "_audit_metadata";
+var VALID_SOURCES = ["app", "api", "external"];
+var PARENT_ID_FIELD = "_parent_id";
 function mergeDicts(mergeInto, mergeFrom) {
   for (const [k, mergeFromV] of Object.entries(mergeFrom)) {
     const mergeIntoV = mergeInto[k];
@@ -10523,6 +10527,7 @@ function mergeDicts(mergeInto, mergeFrom) {
       mergeInto[k] = mergeFromV;
     }
   }
+  return mergeInto;
 }
 function generateMergedRowKey(row) {
   return JSON.stringify(
@@ -10564,6 +10569,15 @@ function mergeRowBatch(rows) {
   out.push(...Object.values(rowGroups));
   return out;
 }
+var SpanTypeAttribute = /* @__PURE__ */ ((SpanTypeAttribute2) => {
+  SpanTypeAttribute2["LLM"] = "llm";
+  SpanTypeAttribute2["SCORE"] = "score";
+  SpanTypeAttribute2["FUNCTION"] = "function";
+  SpanTypeAttribute2["EVAL"] = "eval";
+  SpanTypeAttribute2["TASK"] = "task";
+  SpanTypeAttribute2["TOOL"] = "tool";
+  return SpanTypeAttribute2;
+})(SpanTypeAttribute || {});
 
 // src/isomorph.ts
 var DefaultAsyncLocalStorage = class {
@@ -10609,6 +10623,9 @@ function runFinally(f, finallyF) {
 function getCurrentUnixTimestamp() {
   return (/* @__PURE__ */ new Date()).getTime() / 1e3;
 }
+function isEmpty(a) {
+  return a === void 0 || a === null;
+}
 
 // src/logger.ts
 var NoopSpan = class {
@@ -10620,6 +10637,8 @@ var NoopSpan = class {
   }
   log(_) {
   }
+  logFeedback(event) {
+  }
   traced(callback, _1) {
     return callback(this);
   }
@@ -10794,6 +10813,70 @@ var HTTPConnection = class _HTTPConnection {
     return await resp.json();
   }
 };
+function logFeedbackImpl(bgLogger, parentIds, {
+  id,
+  expected,
+  scores,
+  metadata: inputMetadata,
+  comment,
+  source: inputSource
+}) {
+  const source = inputSource ?? "external";
+  if (!VALID_SOURCES.includes(source)) {
+    throw new Error(`source must be one of ${VALID_SOURCES}`);
+  }
+  if (isEmpty(scores) && isEmpty(expected) && isEmpty(comment)) {
+    throw new Error(
+      "At least one of scores, expected, or comment must be specified"
+    );
+  }
+  const validatedEvent = validateAndSanitizeExperimentLogPartialArgs({
+    scores,
+    metadata: inputMetadata,
+    expected
+  });
+  let { metadata, ...updateEvent } = validatedEvent;
+  updateEvent = Object.fromEntries(
+    Object.entries(updateEvent).filter(([_, v]) => !isEmpty(v))
+  );
+  const trueParentIds = (async () => {
+    const { kind, ...ids } = await parentIds;
+    return ids;
+  })();
+  if (Object.keys(updateEvent).length > 0) {
+    const record = (async () => {
+      return {
+        id,
+        ...updateEvent,
+        ...await trueParentIds,
+        [AUDIT_SOURCE_FIELD]: source,
+        [AUDIT_METADATA_FIELD]: metadata,
+        [IS_MERGE_FIELD]: true
+      };
+    })();
+    bgLogger.log([record]);
+  }
+  if (!isEmpty(comment)) {
+    const record = (async () => {
+      return {
+        id: v4_default(),
+        created: (/* @__PURE__ */ new Date()).toISOString(),
+        origin: {
+          // NOTE: We do not know (or care?) what the transaction id of the row that
+          // we're commenting on is here, so we omit it.
+          id
+        },
+        comment: {
+          text: comment
+        },
+        ...await trueParentIds,
+        [AUDIT_SOURCE_FIELD]: source,
+        [AUDIT_METADATA_FIELD]: metadata
+      };
+    })();
+    bgLogger.log([record]);
+  }
+}
 var MaxRequestSize = 6 * 1024 * 1024;
 function constructJsonArray(items) {
   return `[${items.join(",")}]`;
@@ -11177,6 +11260,13 @@ var Experiment = class {
       () => span.end()
     );
   }
+  async lazyParentIds() {
+    return {
+      kind: "experiment",
+      project_id: (await this.project).id,
+      experiment_id: await this.id
+    };
+  }
   /**
    * Lower-level alternative to `traced`, which does not automatically end the span or mark it as current.
    *
@@ -11184,13 +11274,8 @@ var Experiment = class {
    */
   startSpan(args) {
     const { name, ...argsRest } = args ?? {};
-    const parentIds = (async () => ({
-      kind: "experiment",
-      project_id: (await this.project).id,
-      experiment_id: await this.id
-    }))();
     return new SpanImpl({
-      parentIds,
+      parentIds: this.lazyParentIds(),
       bgLogger: this.bgLogger,
       name: name ?? "root",
       ...argsRest
@@ -11252,6 +11337,20 @@ var Experiment = class {
       metrics
     };
   }
+  /**
+   * Log feedback to an event in the experiment. Feedback is used to save feedback scores, set an expected value, or add a comment.
+   *
+   * @param event
+   * @param event.id The id of the event to log feedback for. This is the `id` returned by `log` or accessible as the `id` field of a span.
+   * @param event.scores (Optional) a dictionary of numeric values (between 0 and 1) to log. These scores will be merged into the existing scores for the event.
+   * @param event.expected (Optional) the ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not.
+   * @param event.comment (Optional) an optional comment string to log about the event.
+   * @param event.metadata (Optional) a dictionary with additional data about the feedback. If you have a `user_id`, you can log it here and access it in the Braintrust UI.
+   * @param event.source (Optional) the source of the feedback. Must be one of "external" (default), "app", or "api".
+   */
+  logFeedback(event) {
+    logFeedbackImpl(this.bgLogger, this.lazyParentIds(), event);
+  }
   /**
    * Flush any pending rows to the server.
    */
@@ -11302,10 +11401,12 @@ var SpanImpl = class _SpanImpl {
     this.rowIds = {
       id,
       span_id,
-      root_span_id: args.parentSpanInfo?.root_span_id ?? span_id
+      root_span_id: "parentSpanInfo" in args && args.parentSpanInfo?.root_span_id ? args.parentSpanInfo.root_span_id : span_id
     };
-    if (args.parentSpanInfo) {
+    if ("parentSpanInfo" in args && args.parentSpanInfo?.span_id) {
       this.internalData.span_parents = [args.parentSpanInfo.span_id];
+    } else if ("parentId" in args && !isEmpty(args.parentId)) {
+      this.rowIds[PARENT_ID_FIELD] = args.parentId;
     }
     this.isMerge = false;
     const { id: _id, ...eventRest } = args.event ?? {};
@@ -11329,16 +11430,26 @@ var SpanImpl = class _SpanImpl {
     if (sanitizedAndInternalData.metrics?.end) {
       this.loggedEndTime = sanitizedAndInternalData.metrics?.end;
     }
+    const parentIds = (async () => {
+      const { kind, ...ids } = await this.parentIds;
+      return ids;
+    })();
     const record = (async () => {
       return {
         ...sanitizedAndInternalData,
         ...this.rowIds,
-        ...await this.parentIds,
+        ...await parentIds,
         [IS_MERGE_FIELD]: this.isMerge
       };
     })();
     this.bgLogger.log([record]);
   }
+  logFeedback(event) {
+    logFeedbackImpl(this.bgLogger, this.parentIds, {
+      ...event,
+      id: this.id
+    });
+  }
   traced(callback, args) {
     const { setCurrent, ...argsRest } = args ?? {};
     const span = this.startSpan(argsRest);
@@ -11771,7 +11882,7 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
             }
             span.log({ input: datum.input, output });
           },
-          { name: "task" }
+          { name: "task", spanAttributes: { type: SpanTypeAttribute.TASK } }
         );
         rootSpan.log({ output });
         const scoringArgs = { ...datum, metadata, output };
@@ -11794,6 +11905,9 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
               },
               {
                 name: score.name || `scorer_${score_idx}`,
+                spanAttributes: {
+                  type: SpanTypeAttribute.SCORE
+                },
                 event: { input: scoringArgs }
               }
             );
@@ -11833,6 +11947,9 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
     } else {
       return await experiment.traced(callback, {
         name: "eval",
+        spanAttributes: {
+          type: SpanTypeAttribute.EVAL
+        },
         event: {
           input: datum.input,
           expected: datum.expected
@@ -15985,7 +16102,9 @@ async function getRepoStatus() {
     async () => (await git.raw(["rev-parse", "--abbrev-ref", "HEAD"])).trim()
   );
   if (dirty) {
-    git_diff = await attempt(async () => truncateToByteLimit(await git.raw(["diff", "HEAD"])));
+    git_diff = await attempt(
+      async () => truncateToByteLimit(await git.raw(["diff", "HEAD"]))
+    );
   }
   return {
     commit,