braintrust 0.0.199 → 0.0.201

package/dist/browser.js

@@ -60,6 +60,8 @@ __export(browser_exports, {
   default: () => browser_default,
   deserializePlainStringAsJSON: () => deserializePlainStringAsJSON,
   devNullWritableStream: () => devNullWritableStream,
+  evaluatorDefinitionSchema: () => evaluatorDefinitionSchema,
+  evaluatorDefinitionsSchema: () => evaluatorDefinitionsSchema,
   flush: () => flush,
   getSpanParentObject: () => getSpanParentObject,
   init: () => init,
@@ -396,11 +398,11 @@ function createFinalValuePassThroughStream(onFinal, onError) {
             break;
           default:
             const _type = chunkType;
-            throw new Error(`Unknown chunk type ${_type}`);
+            onError(`Unknown chunk type: ${_type}`);
         }
         controller.enqueue(chunk);
       } else {
-        throw new Error(`Unknown chunk type ${chunk}`);
+        onError(`Unknown chunk type ${JSON.stringify(chunk)}`);
       }
     },
     flush(controller) {
@@ -431,11 +433,12 @@ function devNullWritableStream() {
 
 // src/prompt-cache/disk-cache.ts
 function canUseDiskCache() {
-  return !!(isomorph_default.gunzip && isomorph_default.gzip && isomorph_default.stat && isomorph_default.readFile && isomorph_default.writeFile && isomorph_default.utimes && isomorph_default.readdir && isomorph_default.mkdir && isomorph_default.unlink && isomorph_default.homedir);
+  return !!(isomorph_default.hash && isomorph_default.gunzip && isomorph_default.gzip && isomorph_default.stat && isomorph_default.readFile && isomorph_default.writeFile && isomorph_default.utimes && isomorph_default.readdir && isomorph_default.mkdir && isomorph_default.unlink && isomorph_default.homedir);
 }
 var DiskCache = class {
   dir;
   max;
+  logWarnings;
   /**
    * Creates a new DiskCache instance.
    * @param options - Configuration options for the cache.
@@ -446,14 +449,11 @@ var DiskCache = class {
     }
     this.dir = options.cacheDir;
     this.max = options.max;
+    this.logWarnings = options.logWarnings ?? true;
   }
-  /**
-   * Gets the file path for a cache entry.
-   * @param key - The cache key to get the path for.
-   * @returns The full filesystem path for the cache entry.
-   */
   getEntryPath(key) {
-    return isomorph_default.pathJoin(this.dir, key);
+    const hashed = isomorph_default.hash(key);
+    return isomorph_default.pathJoin(this.dir, hashed);
   }
   /**
    * Retrieves a value from the cache.
@@ -461,7 +461,6 @@ var DiskCache = class {
    *
    * @param key - The key to look up in the cache.
    * @returns The cached value if found, undefined otherwise.
-   * @throws If there is an error reading from the disk cache (except for file not found).
    */
   async get(key) {
     try {
@@ -473,7 +472,10 @@ var DiskCache = class {
       if (e.code === "ENOENT") {
         return void 0;
       }
-      throw e;
+      if (this.logWarnings) {
+        console.warn("Failed to read from disk cache", e);
+      }
+      return void 0;
     }
   }
   /**
@@ -487,33 +489,35 @@ var DiskCache = class {
     await isomorph_default.mkdir(this.dir, { recursive: true });
     const filePath = this.getEntryPath(key);
     const data = await isomorph_default.gzip(JSON.stringify(value));
-    await isomorph_default.writeFile(filePath, data);
-    if (this.max) {
-      const entries = await isomorph_default.readdir(this.dir);
-      if (entries.length > this.max) {
-        await this.evictOldest(entries);
-      }
+    try {
+      await isomorph_default.writeFile(filePath, data);
+      await this.evictOldestIfFull();
+    } catch (e) {
+      console.warn("Failed to write to disk cache", e);
+      return;
     }
   }
-  /**
-   * Evicts the oldest entries from the cache until it is under the maximum size.
-   * @param entries - List of all cache entry filenames.
-   */
-  async evictOldest(entries) {
+  async evictOldestIfFull() {
+    if (!this.max) {
+      return;
+    }
+    const files = await isomorph_default.readdir(this.dir);
+    const paths = files.map((file) => isomorph_default.pathJoin(this.dir, file));
+    if (paths.length <= this.max) {
+      return;
+    }
     const stats = await Promise.all(
-      entries.map(async (entry) => {
-        const stat = await isomorph_default.stat(this.getEntryPath(entry));
+      paths.map(async (path) => {
+        const stat = await isomorph_default.stat(path);
         return {
-          name: entry,
+          path,
           mtime: stat.mtime.getTime()
         };
       })
     );
     stats.sort((a, b) => a.mtime - b.mtime);
     const toRemove = stats.slice(0, stats.length - this.max);
-    await Promise.all(
-      toRemove.map((stat) => isomorph_default.unlink(this.getEntryPath(stat.name)))
-    );
+    await Promise.all(toRemove.map((stat) => isomorph_default.unlink(stat.path)));
   }
 };
 
@@ -903,7 +907,7 @@ var BraintrustState = class _BraintrustState {
     const serializedParsed = loginSchema.safeParse(serialized);
     if (!serializedParsed.success) {
       throw new Error(
-        `Cannot deserialize BraintrustState: ${serializedParsed.error.errors}`
+        `Cannot deserialize BraintrustState: ${serializedParsed.error.message}`
       );
     }
     const state = new _BraintrustState({ ...opts });
@@ -3106,17 +3110,17 @@ function extractAttachments(event, attachments) {
     extractAttachments(value, attachments);
   }
 }
-function enrichAttachments(event) {
+function enrichAttachments(event, state) {
   for (const [key, value] of Object.entries(event)) {
     const parsedValue = import_typespecs2.attachmentReferenceSchema.safeParse(value);
     if (parsedValue.success) {
-      event[key] = new ReadonlyAttachment(parsedValue.data);
+      event[key] = new ReadonlyAttachment(parsedValue.data, state);
       continue;
     }
     if (!(value instanceof Object)) {
       continue;
     }
-    enrichAttachments(value);
+    enrichAttachments(value, state);
   }
   return event;
 }
@@ -3248,7 +3252,7 @@ var Experiment = class extends ObjectFetcher {
   // For type identification.
   kind = "experiment";
   constructor(state, lazyMetadata, dataset) {
-    super("experiment", void 0, enrichAttachments);
+    super("experiment", void 0, (r) => enrichAttachments(r, state));
     this.lazyMetadata = lazyMetadata;
     this.dataset = dataset;
     this.lastStartTime = getCurrentUnixTimestamp();
@@ -3491,7 +3495,7 @@ var Experiment = class extends ObjectFetcher {
 };
 var ReadonlyExperiment = class extends ObjectFetcher {
   constructor(state, lazyMetadata) {
-    super("experiment", void 0, enrichAttachments);
+    super("experiment", void 0, (r) => enrichAttachments(r, state));
     this.state = state;
     this.lazyMetadata = lazyMetadata;
   }
@@ -3804,7 +3808,7 @@ var Dataset = class extends ObjectFetcher {
       (r) => (
         // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
         (0, import_core2.ensureDatasetRecord)(
-          enrichAttachments(r),
+          enrichAttachments(r, this.state),
           isLegacyDataset
         )
       ),
@@ -4140,6 +4144,7 @@ var Prompt = class _Prompt {
   }
   parsedPromptData;
   hasParsedPromptData = false;
+  __braintrust_prompt_marker = true;
   get id() {
     return this.metadata.id;
   }
@@ -4161,6 +4166,9 @@ var Prompt = class _Prompt {
   get options() {
     return this.getParsedPromptData()?.options || {};
   }
+  get promptData() {
+    return this.getParsedPromptData();
+  }
   /**
    * Build the prompt with the given formatting options. The args you pass in will
    * be forwarded to the mustache template that defines the prompt and rendered with
@@ -4328,6 +4336,20 @@ var Prompt = class _Prompt {
     }
     return this.parsedPromptData;
   }
+  static isPrompt(data) {
+    return typeof data === "object" && data !== null && "__braintrust_prompt_marker" in data;
+  }
+  static fromPromptData(name, promptData) {
+    return new _Prompt(
+      {
+        name,
+        slug: name,
+        prompt_data: promptData
+      },
+      {},
+      false
+    );
+  }
 };
 var TEST_API_KEY = "___TEST_API_KEY__THIS_IS_NOT_REAL___";
 function simulateLoginForTests() {
@@ -4359,6 +4381,16 @@ function configureBrowser() {
     }
     return process.env[name];
   };
+  isomorph_default.hash = (data) => {
+    let hash = 0;
+    for (let i = 0; i < data.length; i++) {
+      const char = data.charCodeAt(i);
+      hash = (hash << 5) - hash + char;
+      hash = hash & hash;
+    }
+    const hashHex = (hash >>> 0).toString(16).padStart(8, "0");
+    return hashHex.repeat(8).substring(0, 64);
+  };
   _internalSetInitialState();
   browserConfigured = true;
 }
@@ -4395,6 +4427,8 @@ __export(exports_browser_exports, {
   currentSpan: () => currentSpan,
   deserializePlainStringAsJSON: () => deserializePlainStringAsJSON,
   devNullWritableStream: () => devNullWritableStream,
+  evaluatorDefinitionSchema: () => evaluatorDefinitionSchema,
+  evaluatorDefinitionsSchema: () => evaluatorDefinitionsSchema,
   flush: () => flush,
   getSpanParentObject: () => getSpanParentObject,
   init: () => init,
@@ -5138,6 +5172,47 @@ var WrapperStream = class {
   }
 };
 
+// dev/types.ts
+var import_typespecs4 = require("@braintrust/core/typespecs");
+var import_zod3 = require("zod");
+var evalBodySchema = import_zod3.z.object({
+  name: import_zod3.z.string(),
+  parameters: import_zod3.z.record(import_zod3.z.string(), import_zod3.z.unknown()).nullish(),
+  data: import_typespecs4.runEvalSchema.shape.data,
+  scores: import_zod3.z.array(
+    import_zod3.z.object({
+      function_id: import_typespecs4.functionIdSchema,
+      name: import_zod3.z.string()
+    })
+  ).nullish(),
+  parent: import_typespecs4.invokeParent.optional(),
+  stream: import_zod3.z.boolean().optional()
+});
+var evalParametersSerializedSchema = import_zod3.z.record(
+  import_zod3.z.string(),
+  import_zod3.z.union([
+    import_zod3.z.object({
+      type: import_zod3.z.literal("prompt"),
+      default: import_typespecs4.promptDataSchema.optional(),
+      description: import_zod3.z.string().optional()
+    }),
+    import_zod3.z.object({
+      type: import_zod3.z.literal("data"),
+      schema: import_zod3.z.record(import_zod3.z.unknown()),
+      // JSON Schema
+      default: import_zod3.z.unknown().optional(),
+      description: import_zod3.z.string().optional()
+    })
+  ])
+);
+var evaluatorDefinitionSchema = import_zod3.z.object({
+  parameters: evalParametersSerializedSchema.optional()
+});
+var evaluatorDefinitionsSchema = import_zod3.z.record(
+  import_zod3.z.string(),
+  evaluatorDefinitionSchema
+);
+
 // src/browser.ts
 configureBrowser();
 var browser_default = exports_browser_exports;
@@ -5172,6 +5247,8 @@ var browser_default = exports_browser_exports;
   currentSpan,
   deserializePlainStringAsJSON,
   devNullWritableStream,
+  evaluatorDefinitionSchema,
+  evaluatorDefinitionsSchema,
   flush,
   getSpanParentObject,
   init,

package/dist/browser.mjs

@@ -341,11 +341,11 @@ function createFinalValuePassThroughStream(onFinal, onError) {
             break;
           default:
             const _type = chunkType;
-            throw new Error(`Unknown chunk type ${_type}`);
+            onError(`Unknown chunk type: ${_type}`);
         }
         controller.enqueue(chunk);
       } else {
-        throw new Error(`Unknown chunk type ${chunk}`);
+        onError(`Unknown chunk type ${JSON.stringify(chunk)}`);
       }
     },
     flush(controller) {
@@ -376,11 +376,12 @@ function devNullWritableStream() {
 
 // src/prompt-cache/disk-cache.ts
 function canUseDiskCache() {
-  return !!(isomorph_default.gunzip && isomorph_default.gzip && isomorph_default.stat && isomorph_default.readFile && isomorph_default.writeFile && isomorph_default.utimes && isomorph_default.readdir && isomorph_default.mkdir && isomorph_default.unlink && isomorph_default.homedir);
+  return !!(isomorph_default.hash && isomorph_default.gunzip && isomorph_default.gzip && isomorph_default.stat && isomorph_default.readFile && isomorph_default.writeFile && isomorph_default.utimes && isomorph_default.readdir && isomorph_default.mkdir && isomorph_default.unlink && isomorph_default.homedir);
 }
 var DiskCache = class {
   dir;
   max;
+  logWarnings;
   /**
    * Creates a new DiskCache instance.
    * @param options - Configuration options for the cache.
@@ -391,14 +392,11 @@ var DiskCache = class {
     }
     this.dir = options.cacheDir;
     this.max = options.max;
+    this.logWarnings = options.logWarnings ?? true;
   }
-  /**
-   * Gets the file path for a cache entry.
-   * @param key - The cache key to get the path for.
-   * @returns The full filesystem path for the cache entry.
-   */
   getEntryPath(key) {
-    return isomorph_default.pathJoin(this.dir, key);
+    const hashed = isomorph_default.hash(key);
+    return isomorph_default.pathJoin(this.dir, hashed);
   }
   /**
    * Retrieves a value from the cache.
@@ -406,7 +404,6 @@ var DiskCache = class {
    *
    * @param key - The key to look up in the cache.
    * @returns The cached value if found, undefined otherwise.
-   * @throws If there is an error reading from the disk cache (except for file not found).
    */
   async get(key) {
     try {
@@ -418,7 +415,10 @@ var DiskCache = class {
       if (e.code === "ENOENT") {
         return void 0;
       }
-      throw e;
+      if (this.logWarnings) {
+        console.warn("Failed to read from disk cache", e);
+      }
+      return void 0;
     }
   }
   /**
@@ -432,33 +432,35 @@ var DiskCache = class {
     await isomorph_default.mkdir(this.dir, { recursive: true });
     const filePath = this.getEntryPath(key);
     const data = await isomorph_default.gzip(JSON.stringify(value));
-    await isomorph_default.writeFile(filePath, data);
-    if (this.max) {
-      const entries = await isomorph_default.readdir(this.dir);
-      if (entries.length > this.max) {
-        await this.evictOldest(entries);
-      }
+    try {
+      await isomorph_default.writeFile(filePath, data);
+      await this.evictOldestIfFull();
+    } catch (e) {
+      console.warn("Failed to write to disk cache", e);
+      return;
     }
   }
-  /**
-   * Evicts the oldest entries from the cache until it is under the maximum size.
-   * @param entries - List of all cache entry filenames.
-   */
-  async evictOldest(entries) {
+  async evictOldestIfFull() {
+    if (!this.max) {
+      return;
+    }
+    const files = await isomorph_default.readdir(this.dir);
+    const paths = files.map((file) => isomorph_default.pathJoin(this.dir, file));
+    if (paths.length <= this.max) {
+      return;
+    }
     const stats = await Promise.all(
-      entries.map(async (entry) => {
-        const stat = await isomorph_default.stat(this.getEntryPath(entry));
+      paths.map(async (path) => {
+        const stat = await isomorph_default.stat(path);
         return {
-          name: entry,
+          path,
           mtime: stat.mtime.getTime()
         };
       })
     );
     stats.sort((a, b) => a.mtime - b.mtime);
     const toRemove = stats.slice(0, stats.length - this.max);
-    await Promise.all(
-      toRemove.map((stat) => isomorph_default.unlink(this.getEntryPath(stat.name)))
-    );
+    await Promise.all(toRemove.map((stat) => isomorph_default.unlink(stat.path)));
   }
 };
 
@@ -848,7 +850,7 @@ var BraintrustState = class _BraintrustState {
     const serializedParsed = loginSchema.safeParse(serialized);
     if (!serializedParsed.success) {
       throw new Error(
-        `Cannot deserialize BraintrustState: ${serializedParsed.error.errors}`
+        `Cannot deserialize BraintrustState: ${serializedParsed.error.message}`
       );
     }
     const state = new _BraintrustState({ ...opts });
@@ -3051,17 +3053,17 @@ function extractAttachments(event, attachments) {
     extractAttachments(value, attachments);
   }
 }
-function enrichAttachments(event) {
+function enrichAttachments(event, state) {
   for (const [key, value] of Object.entries(event)) {
     const parsedValue = attachmentReferenceSchema.safeParse(value);
     if (parsedValue.success) {
-      event[key] = new ReadonlyAttachment(parsedValue.data);
+      event[key] = new ReadonlyAttachment(parsedValue.data, state);
       continue;
     }
     if (!(value instanceof Object)) {
       continue;
     }
-    enrichAttachments(value);
+    enrichAttachments(value, state);
   }
   return event;
 }
@@ -3193,7 +3195,7 @@ var Experiment = class extends ObjectFetcher {
   // For type identification.
   kind = "experiment";
   constructor(state, lazyMetadata, dataset) {
-    super("experiment", void 0, enrichAttachments);
+    super("experiment", void 0, (r) => enrichAttachments(r, state));
     this.lazyMetadata = lazyMetadata;
     this.dataset = dataset;
     this.lastStartTime = getCurrentUnixTimestamp();
@@ -3436,7 +3438,7 @@ var Experiment = class extends ObjectFetcher {
 };
 var ReadonlyExperiment = class extends ObjectFetcher {
   constructor(state, lazyMetadata) {
-    super("experiment", void 0, enrichAttachments);
+    super("experiment", void 0, (r) => enrichAttachments(r, state));
     this.state = state;
     this.lazyMetadata = lazyMetadata;
   }
@@ -3749,7 +3751,7 @@ var Dataset = class extends ObjectFetcher {
       (r) => (
         // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
         ensureDatasetRecord(
-          enrichAttachments(r),
+          enrichAttachments(r, this.state),
           isLegacyDataset
         )
       ),
@@ -4085,6 +4087,7 @@ var Prompt = class _Prompt {
   }
   parsedPromptData;
   hasParsedPromptData = false;
+  __braintrust_prompt_marker = true;
   get id() {
     return this.metadata.id;
   }
@@ -4106,6 +4109,9 @@ var Prompt = class _Prompt {
   get options() {
     return this.getParsedPromptData()?.options || {};
   }
+  get promptData() {
+    return this.getParsedPromptData();
+  }
   /**
    * Build the prompt with the given formatting options. The args you pass in will
    * be forwarded to the mustache template that defines the prompt and rendered with
@@ -4273,6 +4279,20 @@ var Prompt = class _Prompt {
     }
     return this.parsedPromptData;
   }
+  static isPrompt(data) {
+    return typeof data === "object" && data !== null && "__braintrust_prompt_marker" in data;
+  }
+  static fromPromptData(name, promptData) {
+    return new _Prompt(
+      {
+        name,
+        slug: name,
+        prompt_data: promptData
+      },
+      {},
+      false
+    );
+  }
 };
 var TEST_API_KEY = "___TEST_API_KEY__THIS_IS_NOT_REAL___";
 function simulateLoginForTests() {
@@ -4304,6 +4324,16 @@ function configureBrowser() {
     }
     return process.env[name];
   };
+  isomorph_default.hash = (data) => {
+    let hash = 0;
+    for (let i = 0; i < data.length; i++) {
+      const char = data.charCodeAt(i);
+      hash = (hash << 5) - hash + char;
+      hash = hash & hash;
+    }
+    const hashHex = (hash >>> 0).toString(16).padStart(8, "0");
+    return hashHex.repeat(8).substring(0, 64);
+  };
   _internalSetInitialState();
   browserConfigured = true;
 }
@@ -4340,6 +4370,8 @@ __export(exports_browser_exports, {
   currentSpan: () => currentSpan,
   deserializePlainStringAsJSON: () => deserializePlainStringAsJSON,
   devNullWritableStream: () => devNullWritableStream,
+  evaluatorDefinitionSchema: () => evaluatorDefinitionSchema,
+  evaluatorDefinitionsSchema: () => evaluatorDefinitionsSchema,
   flush: () => flush,
   getSpanParentObject: () => getSpanParentObject,
   init: () => init,
@@ -5085,6 +5117,52 @@ var WrapperStream = class {
   }
 };
 
+// dev/types.ts
+import {
+  functionIdSchema as functionIdSchema2,
+  invokeParent,
+  runEvalSchema,
+  promptDataSchema as promptDataSchema2
+} from "@braintrust/core/typespecs";
+import { z as z3 } from "zod";
+var evalBodySchema = z3.object({
+  name: z3.string(),
+  parameters: z3.record(z3.string(), z3.unknown()).nullish(),
+  data: runEvalSchema.shape.data,
+  scores: z3.array(
+    z3.object({
+      function_id: functionIdSchema2,
+      name: z3.string()
+    })
+  ).nullish(),
+  parent: invokeParent.optional(),
+  stream: z3.boolean().optional()
+});
+var evalParametersSerializedSchema = z3.record(
+  z3.string(),
+  z3.union([
+    z3.object({
+      type: z3.literal("prompt"),
+      default: promptDataSchema2.optional(),
+      description: z3.string().optional()
+    }),
+    z3.object({
+      type: z3.literal("data"),
+      schema: z3.record(z3.unknown()),
+      // JSON Schema
+      default: z3.unknown().optional(),
+      description: z3.string().optional()
+    })
+  ])
+);
+var evaluatorDefinitionSchema = z3.object({
+  parameters: evalParametersSerializedSchema.optional()
+});
+var evaluatorDefinitionsSchema = z3.record(
+  z3.string(),
+  evaluatorDefinitionSchema
+);
+
 // src/browser.ts
 configureBrowser();
 var browser_default = exports_browser_exports;
@@ -5119,6 +5197,8 @@ export {
   browser_default as default,
   deserializePlainStringAsJSON,
   devNullWritableStream,
+  evaluatorDefinitionSchema,
+  evaluatorDefinitionsSchema,
   flush,
   getSpanParentObject,
   init,