braintrust 0.0.124 → 0.0.126

package/dist/browser.js

@@ -6449,7 +6449,7 @@ var projectLogsEventSchema = z.strictObject({
   project_id: projectSchema.shape.id,
   log_id: z.literal("g").describe("A literal 'g' which identifies the log as a project log"),
   input: projectLogsEventBaseSchema.shape.input.describe(
-    "The arguments that uniquely define a user input(an arbitrary, JSON serializable object)."
+    "The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."
   ),
   output: projectLogsEventBaseSchema.shape.output.describe(
     "The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."
@@ -6729,7 +6729,7 @@ var summarizeExperimentResponseSchema = z.strictObject({
     z.strictObject({
       name: z.string().describe("Name of the score"),
       score: z.number().min(0).max(1).describe("Average score across all examples"),
-      diff: z.number().min(-1).max(1).describe(
+      diff: z.number().min(-1).max(1).optional().describe(
         "Difference in score between the current and comparison experiment"
       ),
       improvements: z.number().int().min(0).describe("Number of improvements in the score"),
@@ -6741,7 +6741,7 @@ var summarizeExperimentResponseSchema = z.strictObject({
       name: z.string().describe("Name of the metric"),
       metric: z.number().describe("Average metric across all examples"),
       unit: z.string().describe("Unit label for the metric"),
-      diff: z.number().describe(
+      diff: z.number().optional().describe(
         "Difference in metric between the current and comparison experiment"
       ),
       improvements: z.number().int().min(0).describe("Number of improvements in the metric"),
@@ -7477,7 +7477,7 @@ var HTTPConnection = class _HTTPConnection {
       } catch (e) {
         if (i < tries - 1) {
           console.log(
-            `Retrying API request ${object_type} ${args} ${e.status} ${e.text}`
+            `Retrying API request ${object_type} ${JSON.stringify(args)} ${e.status} ${e.text}`
           );
           continue;
         }
@@ -7770,7 +7770,8 @@ var BackgroundLogger = class {
   // 6 MB for the AWS lambda gateway (from our own testing).
   maxRequestSize = 6 * 1024 * 1024;
   defaultBatchSize = 100;
-  numRetries = 3;
+  numTries = 3;
+  failedPublishPayloadsDir = void 0;
   constructor(logConn) {
     this.logConn = logConn;
     const syncFlushEnv = Number(isomorph_default.getEnv("BRAINTRUST_SYNC_FLUSH"));
@@ -7787,9 +7788,15 @@ var BackgroundLogger = class {
     if (!isNaN(maxRequestSizeEnv)) {
       this.maxRequestSize = maxRequestSizeEnv;
     }
-    const numRetriesEnv = Number(isomorph_default.getEnv("BRAINTRUST_NUM_RETRIES"));
-    if (!isNaN(numRetriesEnv)) {
-      this.numRetries = numRetriesEnv;
+    const numTriesEnv = Number(isomorph_default.getEnv("BRAINTRUST_NUM_RETRIES"));
+    if (!isNaN(numTriesEnv)) {
+      this.numTries = numTriesEnv + 1;
+    }
+    const failedPublishPayloadsDirEnv = isomorph_default.getEnv(
+      "BRAINTRUST_FAILED_PUBLISH_PAYLOADS_DIR"
+    );
+    if (failedPublishPayloadsDirEnv) {
+      this.failedPublishPayloadsDir = failedPublishPayloadsDirEnv;
     }
     isomorph_default.processOn("beforeExit", async () => {
       await this.flush();
@@ -7853,13 +7860,13 @@ var BackgroundLogger = class {
     }
   }
   async unwrapLazyValues(wrappedItems) {
-    for (let i = 0; i < this.numRetries; ++i) {
+    for (let i = 0; i < this.numTries; ++i) {
       try {
         const itemPromises = wrappedItems.map((x) => x.get());
         return mergeRowBatch(await Promise.all(itemPromises));
       } catch (e) {
         let errmsg = "Encountered error when constructing records to flush";
-        const isRetrying = i + 1 < this.numRetries;
+        const isRetrying = i + 1 < this.numTries;
         if (isRetrying) {
           errmsg += ". Retrying";
         }
@@ -7873,14 +7880,14 @@ var BackgroundLogger = class {
       }
     }
     console.warn(
-      `Failed to construct log records to flush after ${this.numRetries} retries. Dropping batch`
+      `Failed to construct log records to flush after ${this.numTries} attempts. Dropping batch`
     );
     return [];
   }
   async submitLogsRequest(items) {
     const conn = await this.logConn.get();
     const dataStr = constructLogs3Data(items);
-    for (let i = 0; i < this.numRetries; i++) {
+    for (let i = 0; i < this.numTries; i++) {
       const startTime = now();
       let error = void 0;
       try {
@@ -7900,7 +7907,7 @@ var BackgroundLogger = class {
       if (error === void 0) {
         return;
       }
-      const isRetrying = i + 1 < this.numRetries;
+      const isRetrying = i + 1 < this.numTries;
       const retryingText = isRetrying ? "" : " Retrying";
       const errorText = (() => {
         if (error instanceof FailedHTTPResponse) {
@@ -7911,6 +7918,22 @@ var BackgroundLogger = class {
       })();
       const errMsg = `log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${dataStr.length}.${retryingText}
 Error: ${errorText}`;
+      if (!isRetrying && this.failedPublishPayloadsDir && isomorph_default.pathJoin && isomorph_default.mkdir && isomorph_default.writeFile) {
+        const payloadFile = isomorph_default.pathJoin(
+          this.failedPublishPayloadsDir,
+          `payload_${getCurrentUnixTimestamp()}_${v4_default().slice(0, 8)}.json`
+        );
+        try {
+          await isomorph_default.mkdir(this.failedPublishPayloadsDir, { recursive: true });
+          await isomorph_default.writeFile(payloadFile, dataStr);
+        } catch (e) {
+          console.error(
+            `Failed to write failed payload to output file ${payloadFile}:
+`,
+            e
+          );
+        }
+      }
       if (!isRetrying && this.syncFlush) {
         throw new Error(errMsg);
       } else {
@@ -7921,7 +7944,7 @@ Error: ${errorText}`;
       }
     }
     console.warn(
-      `log request failed after ${this.numRetries} retries. Dropping batch`
+      `log request failed after ${this.numTries} retries. Dropping batch`
     );
     return;
   }
@@ -8353,7 +8376,7 @@ async function login(options = {}) {
     conn.set_token(apiKey);
   } else {
     throw new Error(
-      "Please specify an api key. Token based login is not yet implemented in the JS client."
+      "Please specify an api key (e.g. by setting BRAINTRUST_API_KEY)."
     );
   }
   if (!conn) {
@@ -8787,18 +8810,16 @@ var Experiment = class extends ObjectFetcher {
           comparisonExperimentName = baseExperiment.name;
         }
       }
-      if (comparisonExperimentId !== void 0) {
-        const results = await state.logConn().get_json(
-          "/experiment-comparison2",
-          {
-            experiment_id: await this.id,
-            base_experiment_id: comparisonExperimentId
-          },
-          3
-        );
-        scores = results["scores"];
-        metrics = results["metrics"];
-      }
+      const results = await state.logConn().get_json(
+        "/experiment-comparison2",
+        {
+          experiment_id: await this.id,
+          base_experiment_id: comparisonExperimentId
+        },
+        3
+      );
+      scores = results["scores"];
+      metrics = results["metrics"];
     }
     return {
       projectName: (await this.project).name,