braintrust 0.0.206 → 0.0.207

package/dev/dist/index.d.mts

@@ -573,7 +573,7 @@ interface BackgroundLogger {
 }
 declare class HTTPBackgroundLogger implements BackgroundLogger {
     private apiConn;
-    private items;
+    private queue;
     private activeFlush;
     private activeFlushResolved;
     private activeFlushError;
@@ -582,7 +582,7 @@ declare class HTTPBackgroundLogger implements BackgroundLogger {
     maxRequestSize: number;
     defaultBatchSize: number;
     numTries: number;
-    queueDropExceedingMaxsize: number | undefined;
+    queueDropExceedingMaxsize: number;
     queueDropLoggingPeriod: number;
     failedPublishPayloadsDir: string | undefined;
     allPublishPayloadsDir: string | undefined;
@@ -938,6 +938,19 @@ declare class Prompt<HasId extends boolean = true, HasVersion extends boolean =
         messages?: Message[];
         strict?: boolean;
     }): CompiledPrompt<Flavor>;
+    /**
+     * This is a special build method that first resolves attachment references, and then
+     * calls the regular build method. You should use this if you are building prompts from
+     * dataset rows that contain attachments.
+     *
+     * @param buildArgs Args to forward along to the prompt template.
+     */
+    buildWithAttachments<Flavor extends "chat" | "completion" = "chat">(buildArgs: unknown, options?: {
+        flavor?: Flavor;
+        messages?: Message[];
+        strict?: boolean;
+        state?: BraintrustState;
+    }): Promise<CompiledPrompt<Flavor>>;
     private runBuild;
     static renderPrompt({ prompt, buildArgs, options, }: {
         prompt: PromptBlockData;
@@ -1223,13 +1236,13 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
             } | undefined;
         }>, z.ZodObject<{
             image_url: z.ZodObject<{
-                url: z.ZodString;
+                url: z.ZodEffects<z.ZodString, string, unknown>;
                 detail: z.ZodOptional<z.ZodUnion<[z.ZodLiteral<"auto">, z.ZodLiteral<"low">, z.ZodLiteral<"high">]>>;
             }, "strip", z.ZodTypeAny, {
                 url: string;
                 detail?: "auto" | "low" | "high" | undefined;
             }, {
-                url: string;
+                url?: unknown;
                 detail?: "auto" | "low" | "high" | undefined;
             }>;
             type: z.ZodLiteral<"image_url">;
@@ -1242,7 +1255,7 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
         }, {
             type: "image_url";
             image_url: {
-                url: string;
+                url?: unknown;
                 detail?: "auto" | "low" | "high" | undefined;
             };
         }>]>, "many">]>;
@@ -1287,7 +1300,7 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
         } | {
             type: "image_url";
             image_url: {
-                url: string;
+                url?: unknown;
                 detail?: "auto" | "low" | "high" | undefined;
             };
         })[] | undefined;
@@ -1635,7 +1648,7 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
         } | {
             type: "image_url";
             image_url: {
-                url: string;
+                url?: unknown;
                 detail?: "auto" | "low" | "high" | undefined;
             };
         })[] | undefined;
@@ -2332,13 +2345,13 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
                 } | undefined;
             }>, z.ZodObject<{
                 image_url: z.ZodObject<{
-                    url: z.ZodString;
+                    url: z.ZodEffects<z.ZodString, string, unknown>;
                     detail: z.ZodOptional<z.ZodUnion<[z.ZodLiteral<"auto">, z.ZodLiteral<"low">, z.ZodLiteral<"high">]>>;
                 }, "strip", z.ZodTypeAny, {
                     url: string;
                     detail?: "auto" | "low" | "high" | undefined;
                 }, {
-                    url: string;
+                    url?: unknown;
                     detail?: "auto" | "low" | "high" | undefined;
                 }>;
                 type: z.ZodLiteral<"image_url">;
@@ -2351,7 +2364,7 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
             }, {
                 type: "image_url";
                 image_url: {
-                    url: string;
+                    url?: unknown;
                     detail?: "auto" | "low" | "high" | undefined;
                 };
             }>]>, "many">]>;
@@ -2396,7 +2409,7 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
             } | {
                 type: "image_url";
                 image_url: {
-                    url: string;
+                    url?: unknown;
                     detail?: "auto" | "low" | "high" | undefined;
                 };
             })[] | undefined;
@@ -2744,7 +2757,7 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
             } | {
                 type: "image_url";
                 image_url: {
-                    url: string;
+                    url?: unknown;
                     detail?: "auto" | "low" | "high" | undefined;
                 };
             })[] | undefined;
@@ -3630,7 +3643,7 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
             } | {
                 type: "image_url";
                 image_url: {
-                    url: string;
+                    url?: unknown;
                     detail?: "auto" | "low" | "high" | undefined;
                 };
             })[] | undefined;

package/dev/dist/index.d.ts

@@ -573,7 +573,7 @@ interface BackgroundLogger {
 }
 declare class HTTPBackgroundLogger implements BackgroundLogger {
     private apiConn;
-    private items;
+    private queue;
     private activeFlush;
     private activeFlushResolved;
     private activeFlushError;
@@ -582,7 +582,7 @@ declare class HTTPBackgroundLogger implements BackgroundLogger {
     maxRequestSize: number;
     defaultBatchSize: number;
     numTries: number;
-    queueDropExceedingMaxsize: number | undefined;
+    queueDropExceedingMaxsize: number;
     queueDropLoggingPeriod: number;
     failedPublishPayloadsDir: string | undefined;
     allPublishPayloadsDir: string | undefined;
@@ -938,6 +938,19 @@ declare class Prompt<HasId extends boolean = true, HasVersion extends boolean =
         messages?: Message[];
         strict?: boolean;
     }): CompiledPrompt<Flavor>;
+    /**
+     * This is a special build method that first resolves attachment references, and then
+     * calls the regular build method. You should use this if you are building prompts from
+     * dataset rows that contain attachments.
+     *
+     * @param buildArgs Args to forward along to the prompt template.
+     */
+    buildWithAttachments<Flavor extends "chat" | "completion" = "chat">(buildArgs: unknown, options?: {
+        flavor?: Flavor;
+        messages?: Message[];
+        strict?: boolean;
+        state?: BraintrustState;
+    }): Promise<CompiledPrompt<Flavor>>;
     private runBuild;
     static renderPrompt({ prompt, buildArgs, options, }: {
         prompt: PromptBlockData;
@@ -1223,13 +1236,13 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
             } | undefined;
         }>, z.ZodObject<{
             image_url: z.ZodObject<{
-                url: z.ZodString;
+                url: z.ZodEffects<z.ZodString, string, unknown>;
                 detail: z.ZodOptional<z.ZodUnion<[z.ZodLiteral<"auto">, z.ZodLiteral<"low">, z.ZodLiteral<"high">]>>;
             }, "strip", z.ZodTypeAny, {
                 url: string;
                 detail?: "auto" | "low" | "high" | undefined;
             }, {
-                url: string;
+                url?: unknown;
                 detail?: "auto" | "low" | "high" | undefined;
             }>;
             type: z.ZodLiteral<"image_url">;
@@ -1242,7 +1255,7 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
         }, {
             type: "image_url";
             image_url: {
-                url: string;
+                url?: unknown;
                 detail?: "auto" | "low" | "high" | undefined;
             };
         }>]>, "many">]>;
@@ -1287,7 +1300,7 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
         } | {
             type: "image_url";
             image_url: {
-                url: string;
+                url?: unknown;
                 detail?: "auto" | "low" | "high" | undefined;
             };
         })[] | undefined;
@@ -1635,7 +1648,7 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
         } | {
             type: "image_url";
             image_url: {
-                url: string;
+                url?: unknown;
                 detail?: "auto" | "low" | "high" | undefined;
             };
         })[] | undefined;
@@ -2332,13 +2345,13 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
                 } | undefined;
             }>, z.ZodObject<{
                 image_url: z.ZodObject<{
-                    url: z.ZodString;
+                    url: z.ZodEffects<z.ZodString, string, unknown>;
                     detail: z.ZodOptional<z.ZodUnion<[z.ZodLiteral<"auto">, z.ZodLiteral<"low">, z.ZodLiteral<"high">]>>;
                 }, "strip", z.ZodTypeAny, {
                     url: string;
                     detail?: "auto" | "low" | "high" | undefined;
                 }, {
-                    url: string;
+                    url?: unknown;
                     detail?: "auto" | "low" | "high" | undefined;
                 }>;
                 type: z.ZodLiteral<"image_url">;
@@ -2351,7 +2364,7 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
             }, {
                 type: "image_url";
                 image_url: {
-                    url: string;
+                    url?: unknown;
                     detail?: "auto" | "low" | "high" | undefined;
                 };
             }>]>, "many">]>;
@@ -2396,7 +2409,7 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
             } | {
                 type: "image_url";
                 image_url: {
-                    url: string;
+                    url?: unknown;
                     detail?: "auto" | "low" | "high" | undefined;
                 };
             })[] | undefined;
@@ -2744,7 +2757,7 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
             } | {
                 type: "image_url";
                 image_url: {
-                    url: string;
+                    url?: unknown;
                     detail?: "auto" | "low" | "high" | undefined;
                 };
             })[] | undefined;
@@ -3630,7 +3643,7 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
             } | {
                 type: "image_url";
                 image_url: {
-                    url: string;
+                    url?: unknown;
                     detail?: "auto" | "low" | "high" | undefined;
                 };
             })[] | undefined;

package/dev/dist/index.js

@@ -1084,6 +1084,83 @@ var import_pluralize = __toESM(require("pluralize"));
 
 // src/logger.ts
 var import_uuid = require("uuid");
+
+// src/queue.ts
+var DEFAULT_QUEUE_SIZE = 5e3;
+var Queue = class {
+  buffer;
+  head = 0;
+  // the index of the first item in the queue
+  tail = 0;
+  // the index of the next item to be added
+  size = 0;
+  // the number of items in the queue
+  capacity;
+  // the maximum number of items the queue can hold
+  constructor(maxSize) {
+    if (maxSize < 1) {
+      console.warn(
+        `maxSize ${maxSize} is <1, using default ${DEFAULT_QUEUE_SIZE}`
+      );
+      maxSize = DEFAULT_QUEUE_SIZE;
+    }
+    this.capacity = maxSize;
+    this.buffer = new Array(this.capacity);
+  }
+  push(...items) {
+    const dropped = [];
+    for (const item of items) {
+      if (this.size === this.capacity) {
+        const droppedItem = this.buffer[this.head];
+        if (droppedItem !== void 0) {
+          dropped.push(droppedItem);
+        }
+        this.head = (this.head + 1) % this.capacity;
+      } else {
+        this.size++;
+      }
+      this.buffer[this.tail] = item;
+      this.tail = (this.tail + 1) % this.capacity;
+    }
+    return dropped;
+  }
+  peek() {
+    if (this.size === 0) {
+      return void 0;
+    }
+    return this.buffer[this.head];
+  }
+  drain() {
+    const items = [];
+    if (this.size === 0) {
+      return items;
+    }
+    if (this.head < this.tail) {
+      items.push(...this.buffer.slice(this.head, this.tail));
+      this.buffer.fill(void 0, this.head, this.tail);
+    } else {
+      items.push(...this.buffer.slice(this.head));
+      items.push(...this.buffer.slice(0, this.tail));
+      this.buffer.fill(void 0, this.head, this.capacity);
+      this.buffer.fill(void 0, 0, this.tail);
+    }
+    this.head = 0;
+    this.tail = 0;
+    this.size = 0;
+    return items;
+  }
+  clear() {
+    this.buffer.fill(void 0);
+    this.head = 0;
+    this.tail = 0;
+    this.size = 0;
+  }
+  length() {
+    return this.size;
+  }
+};
+
+// src/logger.ts
 var import_core2 = require("@braintrust/core");
 var import_typespecs2 = require("@braintrust/core/typespecs");
 var import_functions = require("@vercel/functions");
@@ -2174,6 +2251,17 @@ var ReadonlyAttachment = class {
   async data() {
     return this._data.get();
   }
+  /**
+   * Returns the attachment contents as a base64-encoded URL that is suitable
+   * for use in a prompt.
+   *
+   * @returns The attachment contents as a base64-encoded URL.
+   */
+  async asBase64Url() {
+    const buf = await (await this.data()).arrayBuffer();
+    const base64 = Buffer.from(buf).toString("base64");
+    return `data:${this.reference.content_type};base64,${base64}`;
+  }
   /**
    * Fetch the attachment metadata, which includes a downloadUrl and a status.
    * This will re-fetch the status each time in case it changes over time.
@@ -2654,7 +2742,7 @@ function now() {
 var BACKGROUND_LOGGER_BASE_SLEEP_TIME_S = 1;
 var HTTPBackgroundLogger = class _HTTPBackgroundLogger {
   apiConn;
-  items = [];
+  queue;
   activeFlush = Promise.resolve();
   activeFlushResolved = true;
   activeFlushError = void 0;
@@ -2664,7 +2752,7 @@ var HTTPBackgroundLogger = class _HTTPBackgroundLogger {
   maxRequestSize = 6 * 1024 * 1024;
   defaultBatchSize = 100;
   numTries = 3;
-  queueDropExceedingMaxsize = void 0;
+  queueDropExceedingMaxsize = DEFAULT_QUEUE_SIZE;
   queueDropLoggingPeriod = 60;
   failedPublishPayloadsDir = void 0;
   allPublishPayloadsDir = void 0;
@@ -2700,6 +2788,7 @@ var HTTPBackgroundLogger = class _HTTPBackgroundLogger {
     if (!isNaN(queueDropExceedingMaxsizeEnv)) {
       this.queueDropExceedingMaxsize = queueDropExceedingMaxsizeEnv;
     }
+    this.queue = new Queue(this.queueDropExceedingMaxsize);
     const queueDropLoggingPeriodEnv = Number(
       isomorph_default.getEnv("BRAINTRUST_QUEUE_DROP_LOGGING_PERIOD")
     );
@@ -2729,17 +2818,7 @@ var HTTPBackgroundLogger = class _HTTPBackgroundLogger {
     if (this._disabled) {
       return;
     }
-    const [addedItems, droppedItems] = (() => {
-      if (this.queueDropExceedingMaxsize === void 0) {
-        return [items, []];
-      }
-      const numElementsToAdd = Math.min(
-        Math.max(this.queueDropExceedingMaxsize - this.items.length, 0),
-        items.length
-      );
-      return [items.slice(0, numElementsToAdd), items.slice(numElementsToAdd)];
-    })();
-    this.items.push(...addedItems);
+    const droppedItems = this.queue.push(...items);
     if (!this.syncFlush) {
       this.triggerActiveFlush();
     }
@@ -2765,12 +2844,11 @@ var HTTPBackgroundLogger = class _HTTPBackgroundLogger {
   }
   async flushOnce(args) {
     if (this._disabled) {
-      this.items = [];
+      this.queue.clear();
       return;
     }
     const batchSize = args?.batchSize ?? this.defaultBatchSize;
-    const wrappedItems = this.items;
-    this.items = [];
+    const wrappedItems = this.queue.drain();
     const [allItems, attachments] = await this.unwrapLazyValues(wrappedItems);
     if (allItems.length === 0) {
       return;
@@ -2822,7 +2900,7 @@ var HTTPBackgroundLogger = class _HTTPBackgroundLogger {
         `Encountered the following errors while uploading attachments:`
       );
     }
-    if (this.items.length > 0) {
+    if (this.queue.length() > 0) {
       await this.flushOnce(args);
     }
   }
@@ -3646,6 +3724,19 @@ function enrichAttachments(event, state) {
   }
   return event;
 }
+async function resolveAttachmentsToBase64(event, state) {
+  for (const [key, value] of Object.entries(event)) {
+    if (value instanceof ReadonlyAttachment) {
+      event[key] = await value.asBase64Url();
+      continue;
+    }
+    if (!(value instanceof Object)) {
+      continue;
+    }
+    await resolveAttachmentsToBase64(value, state);
+  }
+  return event;
+}
 function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
   if ("input" in event && !isEmpty(event.input) && "inputs" in event && !isEmpty(event.inputs) || !("input" in event) && !("inputs" in event)) {
     throw new Error(
@@ -4623,6 +4714,11 @@ function renderMessage(render, message) {
           case "text":
             return { ...c, text: render(c.text) };
           case "image_url":
+            if ((0, import_core2.isObject)(c.image_url.url)) {
+              throw new Error(
+                "Attachments must be replaced with URLs before calling `build()`"
+              );
+            }
             return {
               ...c,
               image_url: {
@@ -4762,6 +4858,21 @@ var Prompt = class _Prompt {
       strict: options.strict
     });
   }
+  /**
+   * This is a special build method that first resolves attachment references, and then
+   * calls the regular build method. You should use this if you are building prompts from
+   * dataset rows that contain attachments.
+   *
+   * @param buildArgs Args to forward along to the prompt template.
+   */
+  async buildWithAttachments(buildArgs, options = {}) {
+    const hydrated = buildArgs instanceof Object ? await resolveAttachmentsToBase64(buildArgs, options.state) : buildArgs;
+    return this.runBuild(hydrated, {
+      flavor: options.flavor ?? "chat",
+      messages: options.messages,
+      strict: options.strict
+    });
+  }
   runBuild(buildArgs, options) {
     const { flavor } = options;
     const params = {
@@ -4797,15 +4908,6 @@ var Prompt = class _Prompt {
     if (!prompt) {
       throw new Error("Empty prompt");
     }
-    const escape = (v) => {
-      if (v === void 0) {
-        throw new Error("Missing!");
-      } else if (typeof v === "string") {
-        return v;
-      } else {
-        return JSON.stringify(v);
-      }
-    };
     const dictArgParsed = import_zod2.z.record(import_zod2.z.unknown()).safeParse(buildArgs);
     const variables = {
       input: buildArgs,
@@ -4853,6 +4955,10 @@ var Prompt = class _Prompt {
         throw new Error("Missing!");
       } else if (typeof v === "string") {
         return v;
+      } else if (v instanceof ReadonlyAttachment) {
+        throw new Error(
+          "Use buildWithAttachments() to build prompts with attachments"
+        );
       } else {
         return JSON.stringify(v);
       }
@@ -4905,7 +5011,7 @@ var Prompt = class _Prompt {
       };
     } else {
       const _ = prompt;
-      throw new Error("never!");
+      throw new Error(`Invalid prompt type: ${_}`);
     }
   }
   getParsedPromptData() {