braintrust 2.2.0 → 2.2.1

package/dev/dist/index.mjs

@@ -326,11 +326,19 @@ function getIdGenerator() {
 
 // util/db_fields.ts
 var TRANSACTION_ID_FIELD = "_xact_id";
+var OBJECT_DELETE_FIELD = "_object_delete";
 var IS_MERGE_FIELD = "_is_merge";
 var AUDIT_SOURCE_FIELD = "_audit_source";
 var AUDIT_METADATA_FIELD = "_audit_metadata";
 var VALID_SOURCES = ["app", "api", "external"];
-var PARENT_ID_FIELD = "_parent_id";
+var OBJECT_ID_KEYS = [
+  "experiment_id",
+  "dataset_id",
+  "prompt_session_id",
+  "project_id",
+  "log_id",
+  "function_data"
+];
 
 // util/span_identifier_v3.ts
 import * as uuid3 from "uuid";
@@ -1005,13 +1013,6 @@ function mergeDictsWithPathsHelper({
 function mergeDicts(mergeInto, mergeFrom) {
   return mergeDictsWithPaths({ mergeInto, mergeFrom, mergePaths: [] });
 }
-function mapAt(m, k) {
-  const ret = m.get(k);
-  if (ret === void 0) {
-    throw new Error(`Map does not contain key ${k}`);
-  }
-  return ret;
-}
 function recordFind(m, k) {
   return m[k];
 }
@@ -1026,72 +1027,8 @@ function getObjValueByPath(row, path2) {
   return curr;
 }
 
-// util/graph_util.ts
-function depthFirstSearch(args) {
-  const { graph, firstVisitF, lastVisitF } = args;
-  for (const vs of graph.values()) {
-    for (const v of vs.values()) {
-      if (!graph.has(v)) {
-        throw new Error(`Outgoing vertex ${v} must be a key in the graph`);
-      }
-    }
-  }
-  const firstVisitedVertices = /* @__PURE__ */ new Set();
-  const visitationOrder = args.visitationOrder ?? [...graph.keys()];
-  const events = visitationOrder.map((vertex) => ({ eventType: "first", vertex, extras: {} })).reverse();
-  while (events.length) {
-    const { eventType, vertex, extras } = events.pop();
-    if (eventType === "last") {
-      lastVisitF?.(vertex);
-      continue;
-    }
-    if (firstVisitedVertices.has(vertex)) {
-      continue;
-    }
-    firstVisitedVertices.add(vertex);
-    firstVisitF?.(vertex, { parentVertex: extras.parentVertex });
-    events.push({ eventType: "last", vertex, extras: {} });
-    mapAt(graph, vertex).forEach((child) => {
-      events.push({
-        eventType: "first",
-        vertex: child,
-        extras: { parentVertex: vertex }
-      });
-    });
-  }
-}
-function undirectedConnectedComponents(graph) {
-  const directedGraph = new Map(
-    [...graph.vertices].map((v) => [v, /* @__PURE__ */ new Set()])
-  );
-  for (const [i, j] of graph.edges) {
-    mapAt(directedGraph, i).add(j);
-    mapAt(directedGraph, j).add(i);
-  }
-  let labelCounter = 0;
-  const vertexLabels = /* @__PURE__ */ new Map();
-  const firstVisitF = (vertex, args) => {
-    const label = args?.parentVertex !== void 0 ? mapAt(vertexLabels, args?.parentVertex) : labelCounter++;
-    vertexLabels.set(vertex, label);
-  };
-  depthFirstSearch({ graph: directedGraph, firstVisitF });
-  const output = Array.from({ length: labelCounter }).map(() => []);
-  for (const [vertex, label] of vertexLabels.entries()) {
-    output[label].push(vertex);
-  }
-  return output;
-}
-function topologicalSort(graph, visitationOrder) {
-  const reverseOrdering = [];
-  const lastVisitF = (vertex) => {
-    reverseOrdering.push(vertex);
-  };
-  depthFirstSearch({ graph, lastVisitF, visitationOrder });
-  return reverseOrdering.reverse();
-}
-
 // util/merge_row_batch.ts
-function generateMergedRowKey(row, useParentIdForId) {
+function generateMergedRowKey(row) {
   return JSON.stringify(
     [
       "org_id",
@@ -1100,7 +1037,7 @@ function generateMergedRowKey(row, useParentIdForId) {
       "dataset_id",
       "prompt_session_id",
       "log_id",
-      useParentIdForId ?? false ? PARENT_ID_FIELD : "id"
+      "id"
     ].map((k) => row[k])
   );
 }
@@ -1154,96 +1091,34 @@ function mergeRowBatch(rows) {
       rowGroups.set(key, row);
     }
   }
-  const merged = [...rowGroups.values()];
-  const rowToLabel = new Map(
-    merged.map((r, i) => [generateMergedRowKey(r), i])
-  );
-  const graph = new Map(
-    Array.from({ length: merged.length }).map((_, i) => [i, /* @__PURE__ */ new Set()])
-  );
-  merged.forEach((r, i) => {
-    const parentId = r[PARENT_ID_FIELD];
-    if (!parentId) {
-      return;
-    }
-    const parentRowKey = generateMergedRowKey(
-      r,
-      true
-      /* useParentIdForId */
-    );
-    const parentLabel = rowToLabel.get(parentRowKey);
-    if (parentLabel !== void 0) {
-      mapAt(graph, parentLabel).add(i);
-    }
-  });
-  const connectedComponents = undirectedConnectedComponents({
-    vertices: new Set(graph.keys()),
-    edges: new Set(
-      [...graph.entries()].flatMap(
-        ([k, vs]) => [...vs].map((v) => {
-          const ret = [k, v];
-          return ret;
-        })
-      )
-    )
-  });
-  const buckets = connectedComponents.map(
-    (cc) => topologicalSort(
-      graph,
-      cc
-      /* visitationOrder */
-    )
-  );
-  return buckets.map((bucket) => bucket.map((i) => merged[i]));
+  return [...rowGroups.values()];
 }
 function batchItems(args) {
-  let { items } = args;
+  const { items } = args;
   const batchMaxNumItems = args.batchMaxNumItems ?? Number.POSITIVE_INFINITY;
   const batchMaxNumBytes = args.batchMaxNumBytes ?? Number.POSITIVE_INFINITY;
+  const getByteSize = args.getByteSize;
   const output = [];
-  let nextItems = [];
-  let batchSet = [];
   let batch = [];
   let batchLen = 0;
   function addToBatch(item) {
     batch.push(item);
-    batchLen += item.length;
+    batchLen += getByteSize(item);
   }
   function flushBatch() {
-    batchSet.push(batch);
+    output.push(batch);
     batch = [];
     batchLen = 0;
   }
-  while (items.length) {
-    for (const bucket of items) {
-      let i = 0;
-      for (const item of bucket) {
-        if (batch.length === 0 || item.length + batchLen < batchMaxNumBytes && batch.length < batchMaxNumItems) {
-          addToBatch(item);
-        } else if (i === 0) {
-          flushBatch();
-          addToBatch(item);
-        } else {
-          break;
-        }
-        ++i;
-      }
-      if (i < bucket.length) {
-        nextItems.push(bucket.slice(i));
-      }
-      if (batchLen >= batchMaxNumBytes || batch.length > batchMaxNumItems) {
-        flushBatch();
-      }
-    }
-    if (batch.length) {
+  for (const item of items) {
+    const itemSize = getByteSize(item);
+    if (batch.length > 0 && !(itemSize + batchLen < batchMaxNumBytes && batch.length < batchMaxNumItems)) {
       flushBatch();
     }
-    if (batchSet.length) {
-      output.push(batchSet);
-      batchSet = [];
-    }
-    items = nextItems;
-    nextItems = [];
+    addToBatch(item);
+  }
+  if (batch.length > 0) {
+    flushBatch();
   }
   return output;
 }
@@ -1841,10 +1716,15 @@ var FunctionTypeEnum = z6.enum([
   "preprocessor",
   "facet",
   "classifier",
-  "tag"
+  "tag",
+  "parameters"
 ]);
 var NullableSavedFunctionId = z6.union([
-  z6.object({ type: z6.literal("function"), id: z6.string() }),
+  z6.object({
+    type: z6.literal("function"),
+    id: z6.string(),
+    version: z6.string().optional()
+  }),
   z6.object({
     type: z6.literal("global"),
     name: z6.string(),
@@ -1852,6 +1732,67 @@ var NullableSavedFunctionId = z6.union([
   }),
   z6.null()
 ]);
+var TopicMapReport = z6.object({
+  version: z6.literal(1),
+  created_at: z6.string().optional(),
+  settings: z6.object({
+    algorithm: z6.enum(["hdbscan", "kmeans", "hierarchical"]),
+    dimension_reduction: z6.enum(["umap", "pca", "none"]),
+    vector_field: z6.string(),
+    embedding_model: z6.string(),
+    n_clusters: z6.union([z6.number(), z6.null()]).optional(),
+    umap_dimensions: z6.union([z6.number(), z6.null()]).optional(),
+    min_cluster_size: z6.union([z6.number(), z6.null()]).optional(),
+    min_samples: z6.union([z6.number(), z6.null()]).optional()
+  }),
+  query_settings: z6.object({
+    hierarchy_threshold: z6.union([z6.number(), z6.null()]),
+    auto_naming: z6.boolean(),
+    skip_cache: z6.boolean(),
+    viz_mode: z6.enum(["bar", "scatter"]),
+    naming_model: z6.string()
+  }).partial(),
+  clusters: z6.array(
+    z6.object({
+      cluster_id: z6.number(),
+      parent_cluster_id: z6.union([z6.number(), z6.null()]).optional(),
+      topic_id: z6.string(),
+      count: z6.number(),
+      sample_texts: z6.array(z6.string()),
+      samples: z6.array(
+        z6.object({
+          id: z6.string(),
+          text: z6.string(),
+          root_span_id: z6.string(),
+          span_id: z6.string()
+        })
+      ),
+      name: z6.string().optional(),
+      description: z6.string().optional(),
+      keywords: z6.array(z6.string()).optional(),
+      centroid: z6.array(z6.number()).optional(),
+      parent_id: z6.union([z6.number(), z6.null()]).optional(),
+      is_leaf: z6.boolean().optional(),
+      depth: z6.number().optional()
+    })
+  ),
+  embedding_points: z6.array(
+    z6.object({
+      x: z6.number(),
+      y: z6.number(),
+      cluster: z6.number(),
+      text: z6.string().optional()
+    })
+  ).optional()
+});
+var TopicMapData = z6.object({
+  type: z6.literal("topic_map"),
+  source_facet: z6.string(),
+  embedding_model: z6.string(),
+  bundle_key: z6.string(),
+  distance_threshold: z6.number().optional(),
+  report: TopicMapReport.optional()
+});
 var BatchedFacetData = z6.object({
   type: z6.literal("batched_facet"),
   preprocessor: NullableSavedFunctionId.and(z6.unknown()).optional(),
@@ -1860,9 +1801,17 @@ var BatchedFacetData = z6.object({
       name: z6.string(),
       prompt: z6.string(),
       model: z6.string().optional(),
+      embedding_model: z6.string().optional(),
       no_match_pattern: z6.string().optional()
     })
-  )
+  ),
+  topic_maps: z6.record(
+    z6.object({
+      function_name: z6.string(),
+      topic_map_id: z6.string().optional(),
+      topic_map_data: TopicMapData
+    })
+  ).optional()
 });
 var BraintrustModelParams = z6.object({
   use_cache: z6.boolean(),
@@ -2073,6 +2022,18 @@ var ObjectReferenceNullish = z6.union([
   }),
   z6.null()
 ]);
+var SavedFunctionId = z6.union([
+  z6.object({
+    type: z6.literal("function"),
+    id: z6.string(),
+    version: z6.string().optional()
+  }),
+  z6.object({
+    type: z6.literal("global"),
+    name: z6.string(),
+    function_type: FunctionTypeEnum.optional().default("scorer")
+  })
+]);
 var DatasetEvent = z6.object({
   id: z6.string(),
   _xact_id: z6.string(),
@@ -2092,7 +2053,36 @@ var DatasetEvent = z6.object({
   is_root: z6.union([z6.boolean(), z6.null()]).optional(),
   origin: ObjectReferenceNullish.optional(),
   comments: z6.union([z6.array(z6.unknown()), z6.null()]).optional(),
-  audit_data: z6.union([z6.array(z6.unknown()), z6.null()]).optional()
+  audit_data: z6.union([z6.array(z6.unknown()), z6.null()]).optional(),
+  facets: z6.union([z6.object({}).partial().passthrough(), z6.null()]).optional(),
+  classifications: z6.union([
+    z6.record(
+      z6.array(
+        z6.object({
+          id: z6.string(),
+          label: z6.string().optional(),
+          confidence: z6.union([z6.number(), z6.null()]).optional(),
+          metadata: z6.union([z6.object({}).partial().passthrough(), z6.null()]).optional(),
+          source: SavedFunctionId.and(
+            z6.union([
+              z6.object({
+                type: z6.literal("function"),
+                id: z6.string(),
+                version: z6.string().optional()
+              }),
+              z6.object({
+                type: z6.literal("global"),
+                name: z6.string(),
+                function_type: FunctionTypeEnum.optional().default("scorer")
+              }),
+              z6.null()
+            ])
+          ).optional()
+        })
+      )
+    ),
+    z6.null()
+  ]).optional()
 });
 var EnvVar = z6.object({
   id: z6.string().uuid(),
@@ -2169,7 +2159,8 @@ var SpanType = z6.union([
     "automation",
     "facet",
     "preprocessor",
-    "classifier"
+    "classifier",
+    "review"
   ]),
   z6.null()
 ]);
@@ -2210,10 +2201,43 @@ var ExperimentEvent = z6.object({
   is_root: z6.union([z6.boolean(), z6.null()]).optional(),
   origin: ObjectReferenceNullish.optional(),
   comments: z6.union([z6.array(z6.unknown()), z6.null()]).optional(),
-  audit_data: z6.union([z6.array(z6.unknown()), z6.null()]).optional()
+  audit_data: z6.union([z6.array(z6.unknown()), z6.null()]).optional(),
+  facets: z6.union([z6.object({}).partial().passthrough(), z6.null()]).optional(),
+  classifications: z6.union([
+    z6.record(
+      z6.array(
+        z6.object({
+          id: z6.string(),
+          label: z6.string().optional(),
+          confidence: z6.union([z6.number(), z6.null()]).optional(),
+          metadata: z6.union([z6.object({}).partial().passthrough(), z6.null()]).optional(),
+          source: SavedFunctionId.and(
+            z6.union([
+              z6.object({
+                type: z6.literal("function"),
+                id: z6.string(),
+                version: z6.string().optional()
+              }),
+              z6.object({
+                type: z6.literal("global"),
+                name: z6.string(),
+                function_type: FunctionTypeEnum.optional().default("scorer")
+              }),
+              z6.null()
+            ])
+          ).optional()
+        })
+      )
+    ),
+    z6.null()
+  ]).optional()
 });
 var ExtendedSavedFunctionId = z6.union([
-  z6.object({ type: z6.literal("function"), id: z6.string() }),
+  z6.object({
+    type: z6.literal("function"),
+    id: z6.string(),
+    version: z6.string().optional()
+  }),
   z6.object({
     type: z6.literal("global"),
     name: z6.string(),
@@ -2230,6 +2254,7 @@ var FacetData = z6.object({
   preprocessor: NullableSavedFunctionId.and(z6.unknown()).optional(),
   prompt: z6.string(),
   model: z6.string().optional(),
+  embedding_model: z6.string().optional(),
   no_match_pattern: z6.string().optional()
 });
 var PromptBlockDataNullish = z6.union([
@@ -2319,14 +2344,6 @@ var PromptParserNullish = z6.union([
   }),
   z6.null()
 ]);
-var SavedFunctionId = z6.union([
-  z6.object({ type: z6.literal("function"), id: z6.string() }),
-  z6.object({
-    type: z6.literal("global"),
-    name: z6.string(),
-    function_type: FunctionTypeEnum.optional().default("scorer")
-  })
-]);
 var PromptDataNullish = z6.union([
   z6.object({
     prompt: PromptBlockDataNullish,
@@ -2377,7 +2394,8 @@ var FunctionTypeEnumNullish = z6.union([
     "preprocessor",
     "facet",
     "classifier",
-    "tag"
+    "tag",
+    "parameters"
   ]),
   z6.null()
 ]);
@@ -2469,7 +2487,8 @@ var FunctionData = z6.union([
     type: z6.literal("remote_eval"),
     endpoint: z6.string(),
     eval_name: z6.string(),
-    parameters: z6.object({}).partial().passthrough()
+    parameters: z6.object({}).partial().passthrough(),
+    parameters_version: z6.union([z6.string(), z6.null()]).optional()
   }),
   z6.object({
     type: z6.literal("global"),
@@ -2478,7 +2497,18 @@ var FunctionData = z6.union([
     config: z6.union([z6.object({}).partial().passthrough(), z6.null()]).optional()
   }),
   FacetData,
-  BatchedFacetData
+  BatchedFacetData,
+  z6.object({
+    type: z6.literal("parameters"),
+    data: z6.object({}).partial().passthrough(),
+    __schema: z6.object({
+      type: z6.literal("object"),
+      properties: z6.record(z6.object({}).partial().passthrough()),
+      required: z6.array(z6.string()).optional(),
+      additionalProperties: z6.boolean().optional()
+    })
+  }),
+  TopicMapData.and(z6.unknown())
 ]);
 var Function = z6.object({
   id: z6.string().uuid(),
@@ -2508,7 +2538,13 @@ var Function = z6.object({
     z6.null()
   ]).optional()
 });
-var FunctionFormat = z6.enum(["llm", "code", "global", "graph"]);
+var FunctionFormat = z6.enum([
+  "llm",
+  "code",
+  "global",
+  "graph",
+  "topic_map"
+]);
 var PromptData = z6.object({
   prompt: PromptBlockDataNullish,
   options: PromptOptionsNullish,
@@ -2591,13 +2627,14 @@ var FunctionObjectType = z6.enum([
   "custom_view",
   "preprocessor",
   "facet",
-  "classifier"
+  "classifier",
+  "parameters"
 ]);
 var FunctionOutputType = z6.enum([
   "completion",
   "score",
   "facet",
-  "tag",
+  "classification",
   "any"
 ]);
 var GitMetadataSettings = z6.object({
@@ -2633,6 +2670,10 @@ var GroupScope = z6.object({
   idle_seconds: z6.number().optional()
 });
 var IfExists = z6.enum(["error", "ignore", "replace"]);
+var ImageRenderingMode = z6.union([
+  z6.enum(["auto", "click_to_load", "blocked"]),
+  z6.null()
+]);
 var InvokeParent = z6.union([
   z6.object({
     object_type: z6.enum(["project_logs", "experiment", "playground_logs"]),
@@ -2725,7 +2766,8 @@ var Organization = z6.object({
   is_universal_api: z6.union([z6.boolean(), z6.null()]).optional(),
   proxy_url: z6.union([z6.string(), z6.null()]).optional(),
   realtime_url: z6.union([z6.string(), z6.null()]).optional(),
-  created: z6.union([z6.string(), z6.null()]).optional()
+  created: z6.union([z6.string(), z6.null()]).optional(),
+  image_rendering_mode: ImageRenderingMode.optional()
 });
 var ProjectSettings = z6.union([
   z6.object({
@@ -2866,7 +2908,36 @@ var ProjectLogsEvent = z6.object({
   origin: ObjectReferenceNullish.optional(),
   comments: z6.union([z6.array(z6.unknown()), z6.null()]).optional(),
   audit_data: z6.union([z6.array(z6.unknown()), z6.null()]).optional(),
-  _async_scoring_state: z6.unknown().optional()
+  _async_scoring_state: z6.unknown().optional(),
+  facets: z6.union([z6.object({}).partial().passthrough(), z6.null()]).optional(),
+  classifications: z6.union([
+    z6.record(
+      z6.array(
+        z6.object({
+          id: z6.string(),
+          label: z6.string().optional(),
+          confidence: z6.union([z6.number(), z6.null()]).optional(),
+          metadata: z6.union([z6.object({}).partial().passthrough(), z6.null()]).optional(),
+          source: SavedFunctionId.and(
+            z6.union([
+              z6.object({
+                type: z6.literal("function"),
+                id: z6.string(),
+                version: z6.string().optional()
+              }),
+              z6.object({
+                type: z6.literal("global"),
+                name: z6.string(),
+                function_type: FunctionTypeEnum.optional().default("scorer")
+              }),
+              z6.null()
+            ])
+          ).optional()
+        })
+      )
+    ),
+    z6.null()
+  ]).optional()
 });
 var ProjectScoreType = z6.enum([
   "slider",
@@ -3168,12 +3239,15 @@ var View = z6.object({
     "datasets",
     "dataset",
     "prompts",
+    "parameters",
     "tools",
     "scorers",
     "classifiers",
     "logs",
     "monitor",
-    "for_review"
+    "for_review_project_log",
+    "for_review_experiments",
+    "for_review_datasets"
   ]),
   name: z6.string(),
   created: z6.union([z6.string(), z6.null()]).optional(),
@@ -3906,6 +3980,52 @@ var PromptCache = class {
   }
 };
 
+// src/prompt-cache/parameters-cache.ts
+function createCacheKey2(key) {
+  if (key.id) {
+    return `parameters:id:${key.id}`;
+  }
+  const prefix = key.projectId ?? key.projectName;
+  if (!prefix) {
+    throw new Error("Either projectId or projectName must be provided");
+  }
+  if (!key.slug) {
+    throw new Error("Slug must be provided when not using ID");
+  }
+  return `parameters:${prefix}:${key.slug}:${key.version ?? "latest"}`;
+}
+var ParametersCache = class {
+  memoryCache;
+  diskCache;
+  constructor(options) {
+    this.memoryCache = options.memoryCache;
+    this.diskCache = options.diskCache;
+  }
+  async get(key) {
+    const cacheKey = createCacheKey2(key);
+    const memoryParams = this.memoryCache.get(cacheKey);
+    if (memoryParams !== void 0) {
+      return memoryParams;
+    }
+    if (this.diskCache) {
+      const diskParams = await this.diskCache.get(cacheKey);
+      if (!diskParams) {
+        return void 0;
+      }
+      this.memoryCache.set(cacheKey, diskParams);
+      return diskParams;
+    }
+    return void 0;
+  }
+  async set(key, value) {
+    const cacheKey = createCacheKey2(key);
+    this.memoryCache.set(cacheKey, value);
+    if (this.diskCache) {
+      await this.diskCache.set(cacheKey, value);
+    }
+  }
+};
+
 // src/span-cache.ts
 var activeCaches = /* @__PURE__ */ new Set();
 var exitHandlersRegistered = false;
@@ -4196,7 +4316,24 @@ var SpanCache = class {
 // src/logger.ts
 var BRAINTRUST_ATTACHMENT = BraintrustAttachmentReference.shape.type.value;
 var EXTERNAL_ATTACHMENT = ExternalAttachmentReference.shape.type.value;
+var LOGS3_OVERFLOW_REFERENCE_TYPE = "logs3_overflow";
 var BRAINTRUST_PARAMS = Object.keys(BraintrustModelParams.shape);
+var DEFAULT_MAX_REQUEST_SIZE = 6 * 1024 * 1024;
+var parametersRowSchema = z8.object({
+  id: z8.string().uuid(),
+  _xact_id: z8.string(),
+  project_id: z8.string().uuid(),
+  name: z8.string(),
+  slug: z8.string(),
+  description: z8.union([z8.string(), z8.null()]).optional(),
+  function_type: z8.literal("parameters"),
+  function_data: z8.object({
+    type: z8.literal("parameters"),
+    data: z8.record(z8.unknown()).optional(),
+    __schema: z8.record(z8.unknown())
+  }),
+  metadata: z8.union([z8.object({}).partial().passthrough(), z8.null()]).optional()
+});
 var LoginInvalidOrgError = class extends Error {
   constructor(message) {
     super(message);
@@ -4373,6 +4510,17 @@ var BraintrustState = class _BraintrustState {
       max: Number(isomorph_default.getEnv("BRAINTRUST_PROMPT_CACHE_DISK_MAX")) ?? 1 << 20
     }) : void 0;
     this.promptCache = new PromptCache({ memoryCache, diskCache });
+    const parametersMemoryCache = new LRUCache({
+      max: Number(isomorph_default.getEnv("BRAINTRUST_PARAMETERS_CACHE_MEMORY_MAX")) ?? 1 << 10
+    });
+    const parametersDiskCache = canUseDiskCache() ? new DiskCache({
+      cacheDir: isomorph_default.getEnv("BRAINTRUST_PARAMETERS_CACHE_DIR") ?? `${isomorph_default.getEnv("HOME") ?? isomorph_default.homedir()}/.braintrust/parameters_cache`,
+      max: Number(isomorph_default.getEnv("BRAINTRUST_PARAMETERS_CACHE_DISK_MAX")) ?? 1 << 20
+    }) : void 0;
+    this.parametersCache = new ParametersCache({
+      memoryCache: parametersMemoryCache,
+      diskCache: parametersDiskCache
+    });
     this.spanCache = new SpanCache({ disabled: loginParams.disableSpanCache });
   }
   id;
@@ -4402,6 +4550,7 @@ var BraintrustState = class _BraintrustState {
   _apiConn = null;
   _proxyConn = null;
   promptCache;
+  parametersCache;
   spanCache;
   _idGenerator = null;
   _contextManager = null;
@@ -5503,8 +5652,100 @@ function castLogger(logger, asyncFlush) {
   }
   return logger;
 }
+var logs3OverflowUploadSchema = z8.object({
+  method: z8.enum(["PUT", "POST"]),
+  signedUrl: z8.string().url(),
+  headers: z8.record(z8.string()).optional(),
+  fields: z8.record(z8.string()).optional(),
+  key: z8.string().min(1)
+});
 function constructLogs3Data(items) {
-  return `{"rows": ${constructJsonArray(items)}, "api_version": 2}`;
+  return `{"rows": ${constructJsonArray(items.map((i) => i.str))}, "api_version": 2}`;
+}
+function constructLogs3OverflowRequest(key) {
+  return {
+    rows: {
+      type: LOGS3_OVERFLOW_REFERENCE_TYPE,
+      key
+    },
+    api_version: 2
+  };
+}
+function pickLogs3OverflowObjectIds(row) {
+  const objectIds = {};
+  for (const key of OBJECT_ID_KEYS) {
+    if (key in row) {
+      objectIds[key] = row[key];
+    }
+  }
+  return objectIds;
+}
+async function uploadLogs3OverflowPayload(upload, payload, fetchFn = fetch) {
+  if (upload.method === "POST") {
+    if (!upload.fields) {
+      throw new Error("Missing logs3 overflow upload fields");
+    }
+    if (typeof FormData === "undefined" || typeof Blob === "undefined") {
+      throw new Error("FormData is not available for logs3 overflow upload");
+    }
+    const form = new FormData();
+    for (const [key, value] of Object.entries(upload.fields)) {
+      form.append(key, value);
+    }
+    const contentType = upload.fields["Content-Type"] ?? "application/json";
+    form.append("file", new Blob([payload], { type: contentType }));
+    const headers2 = {};
+    for (const [key, value] of Object.entries(upload.headers ?? {})) {
+      if (key.toLowerCase() !== "content-type") {
+        headers2[key] = value;
+      }
+    }
+    const response2 = await fetchFn(upload.signedUrl, {
+      method: "POST",
+      headers: headers2,
+      body: form
+    });
+    if (!response2.ok) {
+      const responseText = await response2.text().catch(() => "");
+      throw new Error(
+        `Failed to upload logs3 overflow payload: ${response2.status} ${responseText}`
+      );
+    }
+    return;
+  }
+  const headers = { ...upload.headers ?? {} };
+  addAzureBlobHeaders(headers, upload.signedUrl);
+  const response = await fetchFn(upload.signedUrl, {
+    method: "PUT",
+    headers,
+    body: payload
+  });
+  if (!response.ok) {
+    const responseText = await response.text().catch(() => "");
+    throw new Error(
+      `Failed to upload logs3 overflow payload: ${response.status} ${responseText}`
+    );
+  }
+}
+function stringifyWithOverflowMeta(item) {
+  const str = JSON.stringify(item);
+  const record = item;
+  return {
+    str,
+    overflowMeta: {
+      object_ids: pickLogs3OverflowObjectIds(record),
+      is_delete: record[OBJECT_DELETE_FIELD] === true,
+      input_row: {
+        byte_size: utf8ByteLength(str)
+      }
+    }
+  };
+}
+function utf8ByteLength(value) {
+  if (typeof TextEncoder !== "undefined") {
+    return new TextEncoder().encode(value).length;
+  }
+  return value.length;
 }
 function now() {
   return (/* @__PURE__ */ new Date()).getTime();
@@ -5519,8 +5760,8 @@ var HTTPBackgroundLogger = class _HTTPBackgroundLogger {
   onFlushError;
   maskingFunction = null;
   syncFlush = false;
-  // 6 MB for the AWS lambda gateway (from our own testing).
-  maxRequestSize = 6 * 1024 * 1024;
+  maxRequestSizeOverride = null;
+  _maxRequestSizePromise = null;
   defaultBatchSize = 100;
   numTries = 3;
   queueDropExceedingMaxsize = DEFAULT_QUEUE_SIZE;
@@ -5548,7 +5789,7 @@ var HTTPBackgroundLogger = class _HTTPBackgroundLogger {
     }
     const maxRequestSizeEnv = Number(isomorph_default.getEnv("BRAINTRUST_MAX_REQUEST_SIZE"));
     if (!isNaN(maxRequestSizeEnv)) {
-      this.maxRequestSize = maxRequestSizeEnv;
+      this.maxRequestSizeOverride = maxRequestSizeEnv;
     }
     const numTriesEnv = Number(isomorph_default.getEnv("BRAINTRUST_NUM_RETRIES"));
     if (!isNaN(numTriesEnv)) {
@@ -5610,6 +5851,30 @@ var HTTPBackgroundLogger = class _HTTPBackgroundLogger {
       }
     }
   }
+  getMaxRequestSize() {
+    if (!this._maxRequestSizePromise) {
+      this._maxRequestSizePromise = (async () => {
+        let serverLimit = null;
+        try {
+          const conn = await this.apiConn.get();
+          const versionInfo = await conn.get_json("version");
+          serverLimit = z8.object({ logs3_payload_max_bytes: z8.number().nullish() }).parse(versionInfo).logs3_payload_max_bytes ?? null;
+        } catch (e) {
+          console.warn("Failed to fetch version info for payload limit:", e);
+        }
+        const validServerLimit = serverLimit !== null && serverLimit > 0 ? serverLimit : null;
+        const canUseOverflow = validServerLimit !== null;
+        let maxRequestSize = DEFAULT_MAX_REQUEST_SIZE;
+        if (this.maxRequestSizeOverride !== null) {
+          maxRequestSize = validServerLimit !== null ? Math.min(this.maxRequestSizeOverride, validServerLimit) : this.maxRequestSizeOverride;
+        } else if (validServerLimit !== null) {
+          maxRequestSize = validServerLimit;
+        }
+        return { maxRequestSize, canUseOverflow };
+      })();
+    }
+    return this._maxRequestSizePromise;
+  }
   async flush() {
     if (this.syncFlush) {
       this.triggerActiveFlush();
@@ -5653,33 +5918,33 @@ var HTTPBackgroundLogger = class _HTTPBackgroundLogger {
     if (allItems.length === 0) {
       return;
     }
-    const allItemsStr = allItems.map(
-      (bucket) => bucket.map((item) => JSON.stringify(item))
+    const allItemsWithMeta = allItems.map(
+      (item) => stringifyWithOverflowMeta(item)
     );
-    const batchSets = batchItems({
-      items: allItemsStr,
+    const maxRequestSizeResult = await this.getMaxRequestSize();
+    const batches = batchItems({
+      items: allItemsWithMeta,
       batchMaxNumItems: batchSize,
-      batchMaxNumBytes: this.maxRequestSize / 2
+      batchMaxNumBytes: maxRequestSizeResult.maxRequestSize / 2,
+      getByteSize: (item) => item.str.length
     });
-    for (const batchSet of batchSets) {
-      const postPromises = batchSet.map(
-        (batch) => (async () => {
-          try {
-            await this.submitLogsRequest(batch);
-            return { type: "success" };
-          } catch (e) {
-            return { type: "error", value: e };
-          }
-        })()
+    const postPromises = batches.map(
+      (batch) => (async () => {
+        try {
+          await this.submitLogsRequest(batch, maxRequestSizeResult);
+          return { type: "success" };
+        } catch (e) {
+          return { type: "error", value: e };
+        }
+      })()
+    );
+    const results = await Promise.all(postPromises);
+    const failingResultErrors = results.map((r) => r.type === "success" ? void 0 : r.value).filter((r) => r !== void 0);
+    if (failingResultErrors.length) {
+      throw new AggregateError(
+        failingResultErrors,
+        `Encountered the following errors while logging:`
       );
-      const results = await Promise.all(postPromises);
-      const failingResultErrors = results.map((r) => r.type === "success" ? void 0 : r.value).filter((r) => r !== void 0);
-      if (failingResultErrors.length) {
-        throw new AggregateError(
-          failingResultErrors,
-          `Encountered the following errors while logging:`
-        );
-      }
     }
     const attachmentErrors = [];
     for (const attachment of attachments) {
@@ -5709,32 +5974,30 @@ var HTTPBackgroundLogger = class _HTTPBackgroundLogger {
         items.forEach((item) => extractAttachments(item, attachments));
         let mergedItems = mergeRowBatch(items);
         if (this.maskingFunction) {
-          mergedItems = mergedItems.map(
-            (batch) => batch.map((item) => {
-              const maskedItem = { ...item };
-              for (const field of REDACTION_FIELDS) {
-                if (item[field] !== void 0) {
-                  const maskedValue = applyMaskingToField(
-                    this.maskingFunction,
-                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                    item[field],
-                    field
-                  );
-                  if (maskedValue instanceof MaskingError) {
-                    delete maskedItem[field];
-                    if (maskedItem.error) {
-                      maskedItem.error = `${maskedItem.error}; ${maskedValue.errorMsg}`;
-                    } else {
-                      maskedItem.error = maskedValue.errorMsg;
-                    }
+          mergedItems = mergedItems.map((item) => {
+            const maskedItem = { ...item };
+            for (const field of REDACTION_FIELDS) {
+              if (item[field] !== void 0) {
+                const maskedValue = applyMaskingToField(
+                  this.maskingFunction,
+                  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                  item[field],
+                  field
+                );
+                if (maskedValue instanceof MaskingError) {
+                  delete maskedItem[field];
+                  if (maskedItem.error) {
+                    maskedItem.error = `${maskedItem.error}; ${maskedValue.errorMsg}`;
                   } else {
-                    maskedItem[field] = maskedValue;
+                    maskedItem.error = maskedValue.errorMsg;
                   }
+                } else {
+                  maskedItem[field] = maskedValue;
                 }
               }
-              return maskedItem;
-            })
-          );
+            }
+            return maskedItem;
+          });
         }
         return [mergedItems, attachments];
       } catch (e) {
@@ -5761,20 +6024,73 @@ var HTTPBackgroundLogger = class _HTTPBackgroundLogger {
     }
     throw new Error("Impossible");
   }
-  async submitLogsRequest(items) {
+  async requestLogs3OverflowUpload(conn, args) {
+    let response;
+    try {
+      response = await conn.post_json("logs3/overflow", {
+        content_type: "application/json",
+        size_bytes: args.sizeBytes,
+        rows: args.rows
+      });
+    } catch (error) {
+      const errorStr = JSON.stringify(error);
+      throw new Error(
+        `Failed to request logs3 overflow upload URL: ${errorStr}`
+      );
+    }
+    try {
+      return logs3OverflowUploadSchema.parse(response);
+    } catch (error) {
+      if (error instanceof ZodError) {
+        const errorStr = JSON.stringify(error.flatten());
+        throw new Error(`Invalid response from API server: ${errorStr}`);
+      }
+      throw error;
+    }
+  }
+  async _uploadLogs3OverflowPayload(conn, upload, payload) {
+    await uploadLogs3OverflowPayload(upload, payload, conn.fetch.bind(conn));
+  }
+  async submitLogsRequest(items, {
+    maxRequestSize,
+    canUseOverflow
+  }) {
     const conn = await this.apiConn.get();
     const dataStr = constructLogs3Data(items);
+    const payloadBytes = utf8ByteLength(dataStr);
+    const useOverflow = canUseOverflow && payloadBytes > maxRequestSize;
     if (this.allPublishPayloadsDir) {
       await _HTTPBackgroundLogger.writePayloadToDir({
         payloadDir: this.allPublishPayloadsDir,
         payload: dataStr
       });
     }
+    let overflowUpload = null;
+    const overflowRows = useOverflow ? items.map((item) => item.overflowMeta) : null;
     for (let i = 0; i < this.numTries; i++) {
       const startTime = now();
       let error = void 0;
       try {
-        await conn.post_json("logs3", dataStr);
+        if (overflowRows) {
+          if (!overflowUpload) {
+            const currentUpload = await this.requestLogs3OverflowUpload(conn, {
+              rows: overflowRows,
+              sizeBytes: payloadBytes
+            });
+            await this._uploadLogs3OverflowPayload(
+              conn,
+              currentUpload,
+              dataStr
+            );
+            overflowUpload = currentUpload;
+          }
+          await conn.post_json(
+            "logs3",
+            constructLogs3OverflowRequest(overflowUpload.key)
+          );
+        } else {
+          await conn.post_json("logs3", dataStr);
+        }
       } catch (e) {
         error = e;
       }
@@ -5790,7 +6106,7 @@ var HTTPBackgroundLogger = class _HTTPBackgroundLogger {
           return `${error}`;
         }
       })();
-      const errMsg = `log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${dataStr.length}.${retryingText}
+      const errMsg = `log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${payloadBytes}.${retryingText}
 Error: ${errorText}`;
       if (!isRetrying && this.failedPublishPayloadsDir) {
         await _HTTPBackgroundLogger.writePayloadToDir({
@@ -5844,7 +6160,7 @@ Error: ${errorText}`;
     try {
       const [allItems, allAttachments] = await this.unwrapLazyValues(wrappedItems);
       const dataStr = constructLogs3Data(
-        allItems.map((x) => JSON.stringify(x))
+        allItems.map((x) => stringifyWithOverflowMeta(x))
       );
       const attachmentStr = JSON.stringify(
         allAttachments.map((a) => a.debugInfo())
@@ -8054,28 +8370,77 @@ var Prompt2 = class _Prompt {
     );
   }
 };
-var TEST_API_KEY = "___TEST_API_KEY__THIS_IS_NOT_REAL___";
-
-// src/node.ts
-import { promisify } from "util";
-import * as zlib from "zlib";
-function configureNode() {
-  isomorph_default.getRepoInfo = getRepoInfo;
-  isomorph_default.getPastNAncestors = getPastNAncestors;
-  isomorph_default.getEnv = (name) => process.env[name];
-  isomorph_default.getCallerLocation = getCallerLocation;
-  isomorph_default.newAsyncLocalStorage = () => new AsyncLocalStorage();
-  isomorph_default.processOn = (event, handler) => {
-    process.on(event, handler);
-  };
-  isomorph_default.basename = path.basename;
-  isomorph_default.writeln = (text) => process.stdout.write(text + "\n");
-  isomorph_default.pathJoin = path.join;
-  isomorph_default.pathDirname = path.dirname;
-  isomorph_default.mkdir = fs.mkdir;
-  isomorph_default.writeFile = fs.writeFile;
-  isomorph_default.readFile = fs.readFile;
-  isomorph_default.readdir = fs.readdir;
+var RemoteEvalParameters = class {
+  constructor(metadata) {
+    this.metadata = metadata;
+  }
+  __braintrust_parameters_marker = true;
+  get id() {
+    return this.metadata.id;
+  }
+  get projectId() {
+    return this.metadata.project_id;
+  }
+  get name() {
+    return this.metadata.name;
+  }
+  get slug() {
+    return this.metadata.slug;
+  }
+  get version() {
+    return this.metadata[TRANSACTION_ID_FIELD];
+  }
+  get schema() {
+    return this.metadata.function_data.__schema;
+  }
+  get data() {
+    return this.metadata.function_data.data ?? {};
+  }
+  validate(data) {
+    if (typeof data !== "object" || data === null) {
+      return false;
+    }
+    const schemaProps = this.schema.properties;
+    if (typeof schemaProps !== "object" || schemaProps === null) {
+      return true;
+    }
+    for (const key of Object.keys(schemaProps)) {
+      if (!(key in data)) {
+        const required = Array.isArray(this.schema.required) ? this.schema.required : [];
+        if (required.includes(key)) {
+          return false;
+        }
+      }
+    }
+    return true;
+  }
+  static isParameters(x) {
+    return typeof x === "object" && x !== null && "__braintrust_parameters_marker" in x && // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    x.__braintrust_parameters_marker === true;
+  }
+};
+var TEST_API_KEY = "___TEST_API_KEY__THIS_IS_NOT_REAL___";
+
+// src/node.ts
+import { promisify } from "util";
+import * as zlib from "zlib";
+function configureNode() {
+  isomorph_default.getRepoInfo = getRepoInfo;
+  isomorph_default.getPastNAncestors = getPastNAncestors;
+  isomorph_default.getEnv = (name) => process.env[name];
+  isomorph_default.getCallerLocation = getCallerLocation;
+  isomorph_default.newAsyncLocalStorage = () => new AsyncLocalStorage();
+  isomorph_default.processOn = (event, handler) => {
+    process.on(event, handler);
+  };
+  isomorph_default.basename = path.basename;
+  isomorph_default.writeln = (text) => process.stdout.write(text + "\n");
+  isomorph_default.pathJoin = path.join;
+  isomorph_default.pathDirname = path.dirname;
+  isomorph_default.mkdir = fs.mkdir;
+  isomorph_default.writeFile = fs.writeFile;
+  isomorph_default.readFile = fs.readFile;
+  isomorph_default.readdir = fs.readdir;
   isomorph_default.stat = fs.stat;
   isomorph_default.statSync = fsSync.statSync;
   isomorph_default.utimes = fs.utimes;
@@ -9133,6 +9498,85 @@ function waterfall(tasks, callback) {
 }
 var waterfall$1 = awaitify(waterfall);
 
+// src/functions/invoke.ts
+async function invoke(args) {
+  const {
+    orgName,
+    apiKey,
+    appUrl,
+    forceLogin,
+    fetch: fetch2,
+    input,
+    messages,
+    parent: parentArg,
+    metadata,
+    tags,
+    state: stateArg,
+    stream,
+    mode,
+    schema,
+    strict,
+    projectId,
+    ...functionIdArgs
+  } = args;
+  const state = stateArg ?? _internalGetGlobalState();
+  await state.login({
+    orgName,
+    apiKey,
+    appUrl,
+    forceLogin,
+    fetch: fetch2
+  });
+  const parent = parentArg ? typeof parentArg === "string" ? parentArg : await parentArg.export() : await getSpanParentObject().export();
+  const functionId = FunctionId.safeParse({
+    function_id: functionIdArgs.function_id,
+    project_name: functionIdArgs.projectName,
+    slug: functionIdArgs.slug,
+    global_function: functionIdArgs.globalFunction,
+    function_type: functionIdArgs.functionType,
+    prompt_session_id: functionIdArgs.promptSessionId,
+    prompt_session_function_id: functionIdArgs.promptSessionFunctionId,
+    version: functionIdArgs.version
+  });
+  if (!functionId.success) {
+    throw new Error(
+      `Invalid function ID arguments: ${functionId.error.message}`
+    );
+  }
+  const request = {
+    ...functionId.data,
+    input,
+    messages,
+    parent,
+    metadata,
+    tags,
+    stream,
+    mode,
+    strict
+  };
+  const headers = {
+    Accept: stream ? "text/event-stream" : "application/json"
+  };
+  if (projectId) {
+    headers["x-bt-project-id"] = projectId;
+  }
+  if (orgName) {
+    headers["x-bt-org-name"] = orgName;
+  }
+  const resp = await state.proxyConn().post(`function/invoke`, request, {
+    headers
+  });
+  if (stream) {
+    if (!resp.body) {
+      throw new Error("Received empty stream body");
+    }
+    return new BraintrustStream(resp.body);
+  } else {
+    const data = await resp.json();
+    return schema ? schema.parse(data) : data;
+  }
+}
+
 // src/trace.ts
 var SpanFetcher = class _SpanFetcher extends ObjectFetcher {
   constructor(objectType, _objectId, rootSpanId, _state, spanTypeFilter) {
@@ -9268,6 +9712,7 @@ var LocalTrace = class {
   spansFlushed = false;
   spansFlushPromise = null;
   cachedFetcher;
+  threadCache = /* @__PURE__ */ new Map();
   constructor({
     objectType,
     objectId,
@@ -9338,6 +9783,36 @@ var LocalTrace = class {
     }
     return this.cachedFetcher.getSpans({ spanType });
   }
+  /**
+   * Get the thread (preprocessed messages) for this trace.
+   * Calls the API with the project_default preprocessor (which falls back to "thread").
+   */
+  async getThread(options) {
+    const cacheKey = options?.preprocessor ?? "project_default";
+    if (!this.threadCache.has(cacheKey)) {
+      const promise = this.fetchThread(options);
+      this.threadCache.set(cacheKey, promise);
+    }
+    return this.threadCache.get(cacheKey);
+  }
+  async fetchThread(options) {
+    await this.ensureSpansReady();
+    await this.state.login({});
+    const result = await invoke({
+      globalFunction: options?.preprocessor ?? "project_default",
+      functionType: "preprocessor",
+      input: {
+        trace_ref: {
+          object_type: this.objectType,
+          object_id: this.objectId,
+          root_span_id: this.rootSpanId
+        }
+      },
+      mode: "json",
+      state: this.state
+    });
+    return Array.isArray(result) ? result : [];
+  }
   async ensureSpansReady() {
     if (this.spansFlushed || !this.ensureSpansFlushed) {
       return;
@@ -9372,634 +9847,357 @@ var SimpleProgressReporter = class {
 
 // src/eval-parameters.ts
 import { z as z10 } from "zod/v3";
+import Ajv from "ajv";
 
-// src/framework2.ts
+// src/prompt-schemas.ts
 import { z as z9 } from "zod/v3";
-var currentFilename = typeof __filename !== "undefined" ? __filename : "unknown";
-var ProjectBuilder = class {
-  create(opts) {
-    return new Project2(opts);
+var promptContentsSchema = z9.union([
+  z9.object({
+    prompt: z9.string()
+  }),
+  z9.object({
+    messages: z9.array(ChatCompletionMessageParam)
+  })
+]);
+var promptDefinitionSchema = promptContentsSchema.and(
+  z9.object({
+    model: z9.string(),
+    params: ModelParams.optional(),
+    templateFormat: z9.enum(["mustache", "nunjucks", "none"]).optional()
+  })
+);
+var promptDefinitionWithToolsSchema = promptDefinitionSchema.and(
+  z9.object({
+    tools: z9.array(ToolFunctionDefinition).optional()
+  })
+);
+function promptDefinitionToPromptData(promptDefinition, rawTools) {
+  const promptBlock = "messages" in promptDefinition ? {
+    type: "chat",
+    messages: promptDefinition.messages,
+    tools: rawTools && rawTools.length > 0 ? JSON.stringify(rawTools) : void 0
+  } : {
+    type: "completion",
+    content: promptDefinition.prompt
+  };
+  return {
+    prompt: promptBlock,
+    options: {
+      model: promptDefinition.model,
+      params: promptDefinition.params
+    },
+    ...promptDefinition.templateFormat ? { template_format: promptDefinition.templateFormat } : {}
+  };
+}
+
+// src/eval-parameters.ts
+var evalParametersSchema = z10.record(
+  z10.string(),
+  z10.union([
+    z10.object({
+      type: z10.literal("prompt"),
+      default: promptDefinitionWithToolsSchema.optional(),
+      description: z10.string().optional()
+    }),
+    z10.instanceof(z10.ZodType)
+    // For Zod schemas
+  ])
+);
+async function validateParameters(parameters, parameterSchema) {
+  let resolvedSchema = parameterSchema;
+  if (resolvedSchema instanceof Promise) {
+    resolvedSchema = await resolvedSchema;
+  }
+  if (resolvedSchema === void 0 || resolvedSchema === null) {
+    return parameters;
+  }
+  if (RemoteEvalParameters.isParameters(resolvedSchema)) {
+    const mergedParameters = parameters && Object.keys(parameters).length > 0 ? {
+      ...resolvedSchema.data,
+      ...parameters
+    } : resolvedSchema.data;
+    return validateParametersWithJsonSchema(
+      mergedParameters,
+      resolvedSchema.schema
+    );
   }
-};
-var projects = new ProjectBuilder();
-var Project2 = class {
-  name;
-  id;
-  tools;
-  prompts;
-  scorers;
-  _publishableCodeFunctions = [];
-  _publishablePrompts = [];
-  constructor(args) {
-    _initializeSpanContext();
-    this.name = "name" in args ? args.name : void 0;
-    this.id = "id" in args ? args.id : void 0;
-    this.tools = new ToolBuilder(this);
-    this.prompts = new PromptBuilder(this);
-    this.scorers = new ScorerBuilder(this);
+  return validateParametersWithZod(
+    parameters,
+    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    resolvedSchema
+  );
+}
+function validateParametersWithZod(parameters, parameterSchema) {
+  return Object.fromEntries(
+    Object.entries(parameterSchema).map(([name, schema]) => {
+      const value = parameters[name];
+      try {
+        if ("type" in schema && schema.type === "prompt") {
+          const promptData = value ? PromptData.parse(value) : schema.default ? promptDefinitionToPromptData(
+            schema.default,
+            schema.default.tools
+          ) : void 0;
+          if (!promptData) {
+            throw new Error(`Parameter '${name}' is required`);
+          }
+          return [name, Prompt2.fromPromptData(name, promptData)];
+        } else {
+          const schemaCasted = schema;
+          return [name, schemaCasted.parse(value)];
+        }
+      } catch (e) {
+        console.error("Error validating parameter", name, e);
+        throw Error(
+          `Invalid parameter '${name}': ${e instanceof Error ? e.message : String(e)}`
+        );
+      }
+    })
+  );
+}
+function validateParametersWithJsonSchema(parameters, schema) {
+  const ajv = new Ajv({ coerceTypes: true, useDefaults: true, strict: false });
+  const validate = ajv.compile(schema);
+  if (!validate(parameters)) {
+    const errorMessages = validate.errors?.map((err) => {
+      const path2 = err.instancePath || "root";
+      return `${path2}: ${err.message}`;
+    }).join(", ");
+    throw Error(`Invalid parameters: ${errorMessages}`);
   }
-  addPrompt(prompt) {
-    this._publishablePrompts.push(prompt);
-    if (globalThis._lazy_load) {
-      globalThis._evals.prompts.push(prompt);
-    }
+  return parameters;
+}
+
+// src/framework.ts
+var EvalResultWithSummary = class {
+  constructor(summary, results) {
+    this.summary = summary;
+    this.results = results;
   }
-  addCodeFunction(fn) {
-    this._publishableCodeFunctions.push(fn);
-    if (globalThis._lazy_load) {
-      globalThis._evals.functions.push(fn);
-    }
+  /**
+   * @deprecated Use `summary` instead.
+   */
+  toString() {
+    return JSON.stringify(this.summary);
   }
-  async publish() {
-    if (globalThis._lazy_load) {
-      console.warn("publish() is a no-op when running `braintrust push`.");
-      return;
-    }
-    await login();
-    const projectMap = new ProjectNameIdMap();
-    const functionDefinitions = [];
-    if (this._publishableCodeFunctions.length > 0) {
-      console.warn(
-        "Code functions cannot be published directly. Use `braintrust push` instead."
-      );
-    }
-    if (this._publishablePrompts.length > 0) {
-      for (const prompt of this._publishablePrompts) {
-        const functionDefinition = await prompt.toFunctionDefinition(projectMap);
-        functionDefinitions.push(functionDefinition);
-      }
-    }
-    await _internalGetGlobalState().apiConn().post_json("insert-functions", {
-      functions: functionDefinitions
-    });
+  [Symbol.for("nodejs.util.inspect.custom")]() {
+    return `EvalResultWithSummary(summary="...", results=[...])`;
+  }
+  toJSON() {
+    return {
+      summary: this.summary,
+      results: this.results
+    };
   }
 };
-var ToolBuilder = class {
-  constructor(project) {
-    this.project = project;
+function makeEvalName(projectName, experimentName) {
+  let out = projectName;
+  if (experimentName) {
+    out += ` [experimentName=${experimentName}]`;
   }
-  taskCounter = 0;
-  // This type definition is just a catch all so that the implementation can be
-  // less specific than the two more specific declarations above.
-  create(opts) {
-    this.taskCounter++;
-    opts = opts ?? {};
-    const { handler, name, slug, parameters, returns, ...rest } = opts;
-    let resolvedName = name ?? handler.name;
-    if (resolvedName.trim().length === 0) {
-      resolvedName = `Tool ${isomorph_default.basename(currentFilename)} ${this.taskCounter}`;
-    }
-    const tool = new CodeFunction(this.project, {
-      handler,
-      name: resolvedName,
-      slug: slug ?? slugify(resolvedName, { lower: true, strict: true }),
-      type: "tool",
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions
-      parameters,
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions
-      returns,
-      ...rest
-    });
-    this.project.addCodeFunction(tool);
-    return tool;
+  return out;
+}
+function initExperiment(state, options = {}) {
+  return init({
+    state,
+    ...options,
+    setCurrent: false
+  });
+}
+function callEvaluatorData(data) {
+  const dataResult = typeof data === "function" ? data() : data;
+  let baseExperiment = void 0;
+  if ("_type" in dataResult && dataResult._type === "BaseExperiment") {
+    baseExperiment = dataResult.name;
   }
+  return {
+    data: dataResult,
+    baseExperiment
+  };
+}
+function isAsyncIterable2(value) {
+  return typeof value === "object" && value !== null && typeof value[Symbol.asyncIterator] === "function";
+}
+function isIterable(value) {
+  return typeof value === "object" && value !== null && typeof value[Symbol.iterator] === "function";
+}
+globalThis._evals = {
+  functions: [],
+  prompts: [],
+  parameters: [],
+  evaluators: {},
+  reporters: {}
 };
-var ScorerBuilder = class {
-  constructor(project) {
-    this.project = project;
+function _initializeSpanContext() {
+  globalThis._spanContext = { currentSpan, withCurrent, startSpan, NOOP_SPAN };
+}
+async function Eval(name, evaluator, reporterOrOpts) {
+  const options = isEmpty2(reporterOrOpts) ? {} : typeof reporterOrOpts === "string" ? { reporter: reporterOrOpts } : "name" in reporterOrOpts ? { reporter: reporterOrOpts } : reporterOrOpts;
+  let evalName = makeEvalName(name, evaluator.experimentName);
+  if (globalThis._evals.evaluators[evalName]) {
+    evalName = `${evalName}_${Object.keys(_evals).length}`;
   }
-  taskCounter = 0;
-  create(opts) {
-    this.taskCounter++;
-    let resolvedName = opts.name;
-    if (!resolvedName && "handler" in opts) {
-      resolvedName = opts.handler.name;
+  if (globalThis._lazy_load) {
+    globalThis._evals.evaluators[evalName] = {
+      // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+      evaluator: {
+        evalName,
+        projectName: name,
+        ...evaluator
+      },
+      reporter: options.reporter
+    };
+    _initializeSpanContext();
+    return new EvalResultWithSummary(
+      {
+        scores: {},
+        metrics: {},
+        projectName: "",
+        experimentName: ""
+      },
+      []
+    );
+  }
+  const progressReporter = options.progress ?? new SimpleProgressReporter();
+  const shouldCollectResults = options.returnResults ?? true;
+  if (typeof options.reporter === "string") {
+    throw new Error(
+      "Must specify a reporter object, not a name. Can only specify reporter names when running 'braintrust eval'"
+    );
+  }
+  const resolvedReporter = options.reporter || defaultReporter;
+  try {
+    const { data, baseExperiment: defaultBaseExperiment } = callEvaluatorData(
+      evaluator.data
+    );
+    const experiment = options.parent || options.noSendLogs ? null : initExperiment(evaluator.state, {
+      ...evaluator.projectId ? { projectId: evaluator.projectId } : { project: name },
+      experiment: evaluator.experimentName,
+      description: evaluator.description,
+      metadata: evaluator.metadata,
+      isPublic: evaluator.isPublic,
+      update: evaluator.update,
+      baseExperiment: evaluator.baseExperimentName ?? defaultBaseExperiment,
+      baseExperimentId: evaluator.baseExperimentId,
+      gitMetadataSettings: evaluator.gitMetadataSettings,
+      repoInfo: evaluator.repoInfo,
+      dataset: Dataset2.isDataset(data) ? data : void 0
+    });
+    if (experiment && typeof process !== "undefined" && globalThis.BRAINTRUST_CONTEXT_MANAGER !== void 0) {
+      await experiment._waitForId();
     }
-    if (!resolvedName || resolvedName.trim().length === 0) {
-      resolvedName = `Scorer ${isomorph_default.basename(currentFilename)} ${this.taskCounter}`;
+    if (experiment && options.onStart) {
+      const summary = await experiment.summarize({ summarizeScores: false });
+      options.onStart(summary);
     }
-    const slug = opts.slug ?? slugify(resolvedName, { lower: true, strict: true });
-    if ("handler" in opts) {
-      const scorer = new CodeFunction(this.project, {
-        ...opts,
-        name: resolvedName,
-        slug,
-        type: "scorer"
-      });
-      this.project.addCodeFunction(scorer);
-    } else {
-      const promptBlock = "messages" in opts ? {
-        type: "chat",
-        messages: opts.messages
-      } : {
-        type: "completion",
-        content: opts.prompt
-      };
-      const promptData = {
-        prompt: promptBlock,
-        options: {
-          model: opts.model,
-          params: opts.params
-        },
-        parser: {
-          type: "llm_classifier",
-          use_cot: opts.useCot,
-          choice_scores: opts.choiceScores
-        }
+    try {
+      const evalDef = {
+        evalName,
+        projectName: name,
+        ...evaluator,
+        data
       };
-      const codePrompt = new CodePrompt(
-        this.project,
-        promptData,
-        [],
-        {
-          ...opts,
-          name: resolvedName,
-          slug
-        },
-        "scorer"
-      );
-      this.project.addPrompt(codePrompt);
-    }
-  }
-};
-var CodeFunction = class {
-  constructor(project, opts) {
-    this.project = project;
-    this.handler = opts.handler;
-    this.name = opts.name;
-    this.slug = opts.slug;
-    this.description = opts.description;
-    this.type = opts.type;
-    this.ifExists = opts.ifExists;
-    this.metadata = opts.metadata;
-    this.parameters = opts.parameters;
-    this.returns = opts.returns;
-    if (this.returns && !this.parameters) {
-      throw new Error("parameters are required if return type is defined");
+      const enableCache = options.enableCache ?? true;
+      let ret;
+      if (options.parent) {
+        ret = await withParent(
+          options.parent,
+          () => runEvaluator(
+            null,
+            evalDef,
+            progressReporter,
+            [],
+            options.stream,
+            options.parameters,
+            shouldCollectResults,
+            enableCache
+          ),
+          evaluator.state
+        );
+      } else {
+        ret = await runEvaluator(
+          experiment,
+          evalDef,
+          progressReporter,
+          [],
+          options.stream,
+          options.parameters,
+          shouldCollectResults,
+          enableCache
+        );
+      }
+      progressReporter.stop();
+      resolvedReporter.reportEval(evalDef, ret, {
+        verbose: true,
+        jsonl: false
+      });
+      return ret;
+    } finally {
+      if (experiment) {
+        await experiment.flush().catch(console.error);
+      } else if (options.parent) {
+        await flush().catch(console.error);
+      }
     }
+  } finally {
+    progressReporter.stop();
   }
-  handler;
-  name;
-  slug;
-  type;
-  description;
-  parameters;
-  returns;
-  ifExists;
-  metadata;
-  key() {
-    return JSON.stringify([
-      this.project.id ?? "",
-      this.project.name ?? "",
-      this.slug
-    ]);
+}
+function serializeJSONWithPlainString(v) {
+  if (typeof v === "string") {
+    return v;
+  } else {
+    return JSON.stringify(v);
   }
-};
-var CodePrompt = class {
-  project;
-  name;
-  slug;
-  prompt;
-  ifExists;
-  description;
-  id;
-  functionType;
-  toolFunctions;
-  metadata;
-  constructor(project, prompt, toolFunctions, opts, functionType) {
-    this.project = project;
-    this.name = opts.name;
-    this.slug = opts.slug;
-    this.prompt = prompt;
-    this.toolFunctions = toolFunctions;
-    this.ifExists = opts.ifExists;
-    this.description = opts.description;
-    this.id = opts.id;
-    this.functionType = functionType;
-    this.metadata = opts.metadata;
-  }
-  async toFunctionDefinition(projectNameToId) {
-    const prompt_data = {
-      ...this.prompt
-    };
-    if (this.toolFunctions.length > 0) {
-      const resolvableToolFunctions = await Promise.all(
-        this.toolFunctions.map(async (fn) => {
-          if ("slug" in fn) {
-            return {
-              type: "slug",
-              project_id: await projectNameToId.resolve(fn.project),
-              slug: fn.slug
-            };
-          } else {
-            return fn;
-          }
-        })
-      );
-      prompt_data.tool_functions = // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-      resolvableToolFunctions;
-    }
-    return {
-      project_id: await projectNameToId.resolve(this.project),
-      name: this.name,
-      slug: this.slug,
-      description: this.description ?? "",
-      function_data: {
-        type: "prompt"
-      },
-      function_type: this.functionType,
-      prompt_data,
-      if_exists: this.ifExists,
-      metadata: this.metadata
-    };
-  }
-};
-var promptContentsSchema = z9.union([
-  z9.object({
-    prompt: z9.string()
-  }),
-  z9.object({
-    messages: z9.array(ChatCompletionMessageParam)
-  })
-]);
-var promptDefinitionSchema = promptContentsSchema.and(
-  z9.object({
-    model: z9.string(),
-    params: ModelParams.optional(),
-    templateFormat: z9.enum(["mustache", "nunjucks", "none"]).optional()
-  })
-);
-var promptDefinitionWithToolsSchema = promptDefinitionSchema.and(
-  z9.object({
-    tools: z9.array(ToolFunctionDefinition).optional()
-  })
-);
-var PromptBuilder = class {
-  constructor(project) {
-    this.project = project;
-  }
-  create(opts) {
-    const toolFunctions = [];
-    const rawTools = [];
-    for (const tool of opts.tools ?? []) {
-      if (tool instanceof CodeFunction) {
-        toolFunctions.push(tool);
-      } else if ("type" in tool && !("function" in tool)) {
-        toolFunctions.push(tool);
-      } else {
-        rawTools.push(tool);
-      }
-    }
-    const slug = opts.slug ?? slugify(opts.name, { lower: true, strict: true });
-    const promptData = promptDefinitionToPromptData(opts, rawTools);
-    const promptRow = {
-      id: opts.id,
-      _xact_id: opts.version ? loadPrettyXact(opts.version) : void 0,
-      name: opts.name,
-      slug,
-      prompt_data: promptData,
-      ...this.project.id !== void 0 ? { project_id: this.project.id } : {}
-    };
-    const prompt = new Prompt2(
-      promptRow,
-      {},
-      // It doesn't make sense to specify defaults here.
-      opts.noTrace ?? false
-    );
-    const codePrompt = new CodePrompt(this.project, promptData, toolFunctions, {
-      ...opts,
-      slug
-    });
-    this.project.addPrompt(codePrompt);
-    return prompt;
-  }
-};
-function promptDefinitionToPromptData(promptDefinition, rawTools) {
-  const promptBlock = "messages" in promptDefinition ? {
-    type: "chat",
-    messages: promptDefinition.messages,
-    tools: rawTools && rawTools.length > 0 ? JSON.stringify(rawTools) : void 0
-  } : {
-    type: "completion",
-    content: promptDefinition.prompt
-  };
-  return {
-    prompt: promptBlock,
-    options: {
-      model: promptDefinition.model,
-      params: promptDefinition.params
-    },
-    ...promptDefinition.templateFormat ? { template_format: promptDefinition.templateFormat } : {}
-  };
 }
-var ProjectNameIdMap = class {
-  nameToId = {};
-  idToName = {};
-  async getId(projectName) {
-    if (!(projectName in this.nameToId)) {
-      const response = await _internalGetGlobalState().appConn().post_json("api/project/register", {
-        project_name: projectName
-      });
-      const result = z9.object({
-        project: Project
-      }).parse(response);
-      const projectId = result.project.id;
-      this.nameToId[projectName] = projectId;
-      this.idToName[projectId] = projectName;
-    }
-    return this.nameToId[projectName];
+function evaluateFilter(object, filter2) {
+  const { path: path2, pattern } = filter2;
+  const key = path2.reduce(
+    (acc, p) => typeof acc === "object" && acc !== null ? (
+      // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+      acc[p]
+    ) : void 0,
+    object
+  );
+  if (key === void 0) {
+    return false;
   }
-  async getName(projectId) {
-    if (!(projectId in this.idToName)) {
-      const response = await _internalGetGlobalState().appConn().post_json("api/project/get", {
-        id: projectId
-      });
-      const result = z9.array(Project).nonempty().parse(response);
-      const projectName = result[0].name;
-      this.idToName[projectId] = projectName;
-      this.nameToId[projectName] = projectId;
-    }
-    return this.idToName[projectId];
+  return pattern.test(serializeJSONWithPlainString(key));
+}
+function scorerName(scorer, scorer_idx) {
+  return scorer.name || `scorer_${scorer_idx}`;
+}
+async function runEvaluator(experiment, evaluator, progressReporter, filters, stream, parameters, collectResults = true, enableCache = true) {
+  return await runEvaluatorInternal(
+    experiment,
+    evaluator,
+    progressReporter,
+    filters,
+    stream,
+    parameters,
+    collectResults,
+    enableCache
+  );
+}
+async function runEvaluatorInternal(experiment, evaluator, progressReporter, filters, stream, parameters, collectResults, enableCache) {
+  if (enableCache) {
+    (evaluator.state ?? _internalGetGlobalState())?.spanCache?.start();
   }
-  async resolve(project) {
-    if (project.id) {
-      return project.id;
+  try {
+    if (typeof evaluator.data === "string") {
+      throw new Error("Unimplemented: string data paths");
     }
-    return this.getId(project.name);
-  }
-};
-
-// src/eval-parameters.ts
-var evalParametersSchema = z10.record(
-  z10.string(),
-  z10.union([
-    z10.object({
-      type: z10.literal("prompt"),
-      default: promptDefinitionWithToolsSchema.optional(),
-      description: z10.string().optional()
-    }),
-    z10.instanceof(z10.ZodType)
-    // For Zod schemas
-  ])
-);
-function validateParameters(parameters, parameterSchema) {
-  return Object.fromEntries(
-    Object.entries(parameterSchema).map(([name, schema]) => {
-      const value = parameters[name];
-      try {
-        if ("type" in schema && schema.type === "prompt") {
-          const promptData = value ? PromptData.parse(value) : schema.default ? promptDefinitionToPromptData(
-            schema.default,
-            schema.default.tools
-          ) : void 0;
-          if (!promptData) {
-            throw new Error(`Parameter '${name}' is required`);
-          }
-          return [name, Prompt2.fromPromptData(name, promptData)];
-        } else {
-          const schemaCasted = schema;
-          return [name, schemaCasted.parse(value)];
-        }
-      } catch (e) {
-        console.error("Error validating parameter", name, e);
-        throw Error(
-          `Invalid parameter '${name}': ${e instanceof Error ? e.message : String(e)}`
-        );
-      }
-    })
-  );
-}
-
-// src/framework.ts
-var EvalResultWithSummary = class {
-  constructor(summary, results) {
-    this.summary = summary;
-    this.results = results;
-  }
-  /**
-   * @deprecated Use `summary` instead.
-   */
-  toString() {
-    return JSON.stringify(this.summary);
-  }
-  [Symbol.for("nodejs.util.inspect.custom")]() {
-    return `EvalResultWithSummary(summary="...", results=[...])`;
-  }
-  toJSON() {
-    return {
-      summary: this.summary,
-      results: this.results
-    };
-  }
-};
-function makeEvalName(projectName, experimentName) {
-  let out = projectName;
-  if (experimentName) {
-    out += ` [experimentName=${experimentName}]`;
-  }
-  return out;
-}
-function initExperiment(state, options = {}) {
-  return init({
-    state,
-    ...options,
-    setCurrent: false
-  });
-}
-function callEvaluatorData(data) {
-  const dataResult = typeof data === "function" ? data() : data;
-  let baseExperiment = void 0;
-  if ("_type" in dataResult && dataResult._type === "BaseExperiment") {
-    baseExperiment = dataResult.name;
-  }
-  return {
-    data: dataResult,
-    baseExperiment
-  };
-}
-function isAsyncIterable2(value) {
-  return typeof value === "object" && value !== null && typeof value[Symbol.asyncIterator] === "function";
-}
-function isIterable(value) {
-  return typeof value === "object" && value !== null && typeof value[Symbol.iterator] === "function";
-}
-globalThis._evals = {
-  functions: [],
-  prompts: [],
-  evaluators: {},
-  reporters: {}
-};
-function _initializeSpanContext() {
-  globalThis._spanContext = { currentSpan, withCurrent, startSpan, NOOP_SPAN };
-}
-async function Eval(name, evaluator, reporterOrOpts) {
-  const options = isEmpty2(reporterOrOpts) ? {} : typeof reporterOrOpts === "string" ? { reporter: reporterOrOpts } : "name" in reporterOrOpts ? { reporter: reporterOrOpts } : reporterOrOpts;
-  let evalName = makeEvalName(name, evaluator.experimentName);
-  if (globalThis._evals.evaluators[evalName]) {
-    evalName = `${evalName}_${Object.keys(_evals).length}`;
-  }
-  if (globalThis._lazy_load) {
-    globalThis._evals.evaluators[evalName] = {
-      // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-      evaluator: {
-        evalName,
-        projectName: name,
-        ...evaluator
-      },
-      reporter: options.reporter
-    };
-    _initializeSpanContext();
-    return new EvalResultWithSummary(
-      {
-        scores: {},
-        metrics: {},
-        projectName: "",
-        experimentName: ""
-      },
-      []
-    );
-  }
-  const progressReporter = options.progress ?? new SimpleProgressReporter();
-  const shouldCollectResults = options.returnResults ?? true;
-  if (typeof options.reporter === "string") {
-    throw new Error(
-      "Must specify a reporter object, not a name. Can only specify reporter names when running 'braintrust eval'"
-    );
-  }
-  const resolvedReporter = options.reporter || defaultReporter;
-  try {
-    const { data, baseExperiment: defaultBaseExperiment } = callEvaluatorData(
-      evaluator.data
-    );
-    const experiment = options.parent || options.noSendLogs ? null : initExperiment(evaluator.state, {
-      ...evaluator.projectId ? { projectId: evaluator.projectId } : { project: name },
-      experiment: evaluator.experimentName,
-      description: evaluator.description,
-      metadata: evaluator.metadata,
-      isPublic: evaluator.isPublic,
-      update: evaluator.update,
-      baseExperiment: evaluator.baseExperimentName ?? defaultBaseExperiment,
-      baseExperimentId: evaluator.baseExperimentId,
-      gitMetadataSettings: evaluator.gitMetadataSettings,
-      repoInfo: evaluator.repoInfo,
-      dataset: Dataset2.isDataset(data) ? data : void 0
-    });
-    if (experiment && typeof process !== "undefined" && globalThis.BRAINTRUST_CONTEXT_MANAGER !== void 0) {
-      await experiment._waitForId();
-    }
-    if (experiment && options.onStart) {
-      const summary = await experiment.summarize({ summarizeScores: false });
-      options.onStart(summary);
-    }
-    try {
-      const evalDef = {
-        evalName,
-        projectName: name,
-        ...evaluator,
-        data
-      };
-      const enableCache = options.enableCache ?? true;
-      let ret;
-      if (options.parent) {
-        ret = await withParent(
-          options.parent,
-          () => runEvaluator(
-            null,
-            evalDef,
-            progressReporter,
-            [],
-            options.stream,
-            options.parameters,
-            shouldCollectResults,
-            enableCache
-          ),
-          evaluator.state
-        );
-      } else {
-        ret = await runEvaluator(
-          experiment,
-          evalDef,
-          progressReporter,
-          [],
-          options.stream,
-          options.parameters,
-          shouldCollectResults,
-          enableCache
-        );
-      }
-      progressReporter.stop();
-      resolvedReporter.reportEval(evalDef, ret, {
-        verbose: true,
-        jsonl: false
-      });
-      return ret;
-    } finally {
-      if (experiment) {
-        await experiment.flush().catch(console.error);
-      } else if (options.parent) {
-        await flush().catch(console.error);
-      }
-    }
-  } finally {
-    progressReporter.stop();
-  }
-}
-function serializeJSONWithPlainString(v) {
-  if (typeof v === "string") {
-    return v;
-  } else {
-    return JSON.stringify(v);
-  }
-}
-function evaluateFilter(object, filter2) {
-  const { path: path2, pattern } = filter2;
-  const key = path2.reduce(
-    (acc, p) => typeof acc === "object" && acc !== null ? (
-      // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-      acc[p]
-    ) : void 0,
-    object
-  );
-  if (key === void 0) {
-    return false;
-  }
-  return pattern.test(serializeJSONWithPlainString(key));
-}
-function scorerName(scorer, scorer_idx) {
-  return scorer.name || `scorer_${scorer_idx}`;
-}
-async function runEvaluator(experiment, evaluator, progressReporter, filters, stream, parameters, collectResults = true, enableCache = true) {
-  return await runEvaluatorInternal(
-    experiment,
-    evaluator,
-    progressReporter,
-    filters,
-    stream,
-    parameters,
-    collectResults,
-    enableCache
-  );
-}
-async function runEvaluatorInternal(experiment, evaluator, progressReporter, filters, stream, parameters, collectResults, enableCache) {
-  if (enableCache) {
-    (evaluator.state ?? _internalGetGlobalState())?.spanCache?.start();
-  }
-  try {
-    if (typeof evaluator.data === "string") {
-      throw new Error("Unimplemented: string data paths");
-    }
-    let dataResult = typeof evaluator.data === "function" ? evaluator.data() : evaluator.data;
-    parameters = validateParameters(
-      parameters ?? {},
-      evaluator.parameters ?? {}
-    );
-    if ("_type" in dataResult) {
-      if (dataResult._type !== "BaseExperiment") {
-        throw new Error("Invalid _type");
-      }
-      if (!experiment) {
-        throw new Error(
-          "Cannot use BaseExperiment() without connecting to Braintrust (you most likely set --no-send-logs)"
+    let dataResult = typeof evaluator.data === "function" ? evaluator.data() : evaluator.data;
+    parameters = await validateParameters(
+      parameters ?? {},
+      evaluator.parameters
+    );
+    if ("_type" in dataResult) {
+      if (dataResult._type !== "BaseExperiment") {
+        throw new Error("Invalid _type");
+      }
+      if (!experiment) {
+        throw new Error(
+          "Cannot use BaseExperiment() without connecting to Braintrust (you most likely set --no-send-logs)"
         );
       }
       let name = dataResult.name;
@@ -10150,6 +10348,9 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
             } else {
               rootSpan.log({ output, metadata, expected });
             }
+            if (evaluator.flushBeforeScoring) {
+              await rootSpan.flush();
+            }
             const scoringArgs = {
               input: datum.input,
               expected: "expected" in datum ? datum.expected : void 0,
@@ -10378,427 +10579,918 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
         collectedResults.length = 0;
       }
     }
-    const summary = experiment ? await experiment.summarize({
-      summarizeScores: evaluator.summarizeScores
-    }) : buildLocalSummary(
-      evaluator,
-      collectResults ? collectedResults : [],
-      localScoreAccumulator ?? void 0
-    );
-    return new EvalResultWithSummary(
-      summary,
-      collectResults ? collectedResults : []
-    );
-  } finally {
-    if (enableCache) {
-      const spanCache = (evaluator.state ?? _internalGetGlobalState())?.spanCache;
-      spanCache?.dispose();
-      spanCache?.stop();
+    const summary = experiment ? await experiment.summarize({
+      summarizeScores: evaluator.summarizeScores
+    }) : buildLocalSummary(
+      evaluator,
+      collectResults ? collectedResults : [],
+      localScoreAccumulator ?? void 0
+    );
+    return new EvalResultWithSummary(
+      summary,
+      collectResults ? collectedResults : []
+    );
+  } finally {
+    if (enableCache) {
+      const spanCache = (evaluator.state ?? _internalGetGlobalState())?.spanCache;
+      spanCache?.dispose();
+      spanCache?.stop();
+    }
+  }
+}
+var warning = (text) => `Warning: ${text}`;
+function logError2(e, verbose) {
+  if (!verbose) {
+    console.error(`${e}`);
+  } else {
+    console.error(e);
+  }
+}
+function accumulateScores(accumulator, scores) {
+  for (const [name, score] of Object.entries(scores)) {
+    if (score === null || score === void 0) {
+      continue;
+    }
+    const existing = accumulator[name] ?? { total: 0, count: 0 };
+    accumulator[name] = {
+      total: existing.total + score,
+      count: existing.count + 1
+    };
+  }
+}
+function ensureScoreAccumulator(results) {
+  const accumulator = {};
+  for (const result of results) {
+    accumulateScores(accumulator, result.scores);
+  }
+  return accumulator;
+}
+function buildLocalSummary(evaluator, results, precomputedScores) {
+  const scoresByName = precomputedScores ?? ensureScoreAccumulator(results);
+  return {
+    projectName: evaluator.projectName,
+    experimentName: evaluator.evalName,
+    scores: Object.fromEntries(
+      Object.entries(scoresByName).map(([name, { total, count }]) => [
+        name,
+        {
+          name,
+          score: count === 0 ? 0 : total / count,
+          improvements: 0,
+          regressions: 0
+        }
+      ])
+    )
+  };
+}
+function reportFailures(evaluator, failingResults, { verbose, jsonl }) {
+  if (failingResults.length > 0) {
+    console.error(
+      warning(
+        `Evaluator ${evaluator.evalName} failed with ${failingResults.length} error${failingResults.length === 1 ? "" : "s"}. This evaluation ("${evaluator.evalName}") will not be fully logged.`
+      )
+    );
+    if (jsonl) {
+      console.log(
+        JSON.stringify({
+          evaluatorName: evaluator.evalName,
+          errors: failingResults.map(
+            (r) => `${r.error instanceof Error ? r.error.stack : r.error}`
+          )
+        })
+      );
+    } else {
+      for (const result of failingResults) {
+        logError2(result.error, verbose);
+      }
+    }
+    if (!verbose && !jsonl) {
+      console.error(warning("Add --verbose to see full stack traces."));
+    }
+  }
+}
+var defaultReporter = {
+  name: "Braintrust default reporter",
+  async reportEval(evaluator, result, { verbose, jsonl }) {
+    const { results, summary } = result;
+    const failingResults = results.filter(
+      (r) => r.error !== void 0
+    );
+    if (failingResults.length > 0) {
+      reportFailures(evaluator, failingResults, { verbose, jsonl });
+    }
+    if (jsonl) {
+      isomorph_default.writeln(JSON.stringify(summary));
+    } else {
+      isomorph_default.writeln("Experiment summary");
+      isomorph_default.writeln("==================");
+      if (summary.comparisonExperimentName) {
+        isomorph_default.writeln(
+          `${summary.comparisonExperimentName} (baseline) <- ${summary.experimentName} (comparison)`
+        );
+        isomorph_default.writeln("");
+      }
+      const hasScores = Object.keys(summary.scores).length > 0;
+      const hasMetrics = Object.keys(summary.metrics ?? {}).length > 0;
+      const hasComparison = !!summary.comparisonExperimentName;
+      if (hasScores || hasMetrics) {
+        if (hasComparison) {
+          isomorph_default.writeln(
+            "Name                Value      Change     Improvements Regressions"
+          );
+          isomorph_default.writeln(
+            "----------------------------------------------------------------"
+          );
+        }
+        for (const score of Object.values(summary.scores)) {
+          const scorePercent = (score.score * 100).toFixed(2);
+          const scoreValue = `${scorePercent}%`;
+          if (hasComparison) {
+            let diffString = "-";
+            if (!isEmpty2(score.diff)) {
+              const diffPercent = (score.diff * 100).toFixed(2);
+              const diffSign = score.diff > 0 ? "+" : "";
+              diffString = `${diffSign}${diffPercent}%`;
+            }
+            const improvements = score.improvements > 0 ? score.improvements.toString() : "-";
+            const regressions = score.regressions > 0 ? score.regressions.toString() : "-";
+            isomorph_default.writeln(
+              `${score.name.padEnd(18)} ${scoreValue.padStart(10)} ${diffString.padStart(10)} ${improvements.padStart(12)} ${regressions.padStart(11)}`
+            );
+          } else {
+            isomorph_default.writeln(`${score.name.padEnd(20)} ${scoreValue.padStart(15)}`);
+          }
+        }
+        for (const metric of Object.values(summary.metrics ?? {})) {
+          const fractionDigits = Number.isInteger(metric.metric) ? 0 : 2;
+          const formattedValue = metric.metric.toFixed(fractionDigits);
+          const metricValue = metric.unit === "$" ? `${metric.unit}${formattedValue}` : `${formattedValue}${metric.unit}`;
+          if (hasComparison) {
+            let diffString = "-";
+            if (!isEmpty2(metric.diff)) {
+              const diffPercent = (metric.diff * 100).toFixed(2);
+              const diffSign = metric.diff > 0 ? "+" : "";
+              diffString = `${diffSign}${diffPercent}%`;
+            }
+            const improvements = metric.improvements > 0 ? metric.improvements.toString() : "-";
+            const regressions = metric.regressions > 0 ? metric.regressions.toString() : "-";
+            isomorph_default.writeln(
+              `${metric.name.padEnd(18)} ${metricValue.padStart(10)} ${diffString.padStart(10)} ${improvements.padStart(12)} ${regressions.padStart(11)}`
+            );
+          } else {
+            isomorph_default.writeln(
+              `${metric.name.padEnd(20)} ${metricValue.padStart(15)}`
+            );
+          }
+        }
+      }
+      if (summary.experimentUrl) {
+        isomorph_default.writeln("");
+        isomorph_default.writeln(`View results for ${summary.experimentName}`);
+        isomorph_default.writeln(`See results at ${summary.experimentUrl}`);
+      }
+    }
+    isomorph_default.writeln("");
+    return failingResults.length === 0;
+  },
+  async reportRun(evalReports) {
+    return evalReports.every((r) => r);
+  }
+};
+
+// dev/errorHandler.ts
+import { z as z11 } from "zod/v3";
+var errorHandler = (err, req, res, next) => {
+  if ("status" in err) {
+    res.status(err.status).json({
+      error: {
+        message: err.message,
+        status: err.status
+      }
+    });
+    return;
+  }
+  if (err instanceof z11.ZodError) {
+    res.status(400).json({
+      error: {
+        message: "Invalid request",
+        errors: err.errors
+      }
+    });
+    return;
+  }
+  console.error("Internal server error", err);
+  res.status(500).json({
+    error: {
+      message: "Internal server error",
+      status: 500
+    }
+  });
+};
+
+// dev/authorize.ts
+import createError from "http-errors";
+function authorizeRequest(req, res, next) {
+  try {
+    const ctx = {
+      appOrigin: extractAllowedOrigin(req.headers[ORIGIN_HEADER]),
+      token: void 0,
+      state: void 0,
+      projectId: parseHeader(req.headers, PROJECT_ID_HEADER)
+    };
+    if (req.headers.authorization || req.headers[BRAINTRUST_AUTH_TOKEN_HEADER]) {
+      const tokenText = parseBraintrustAuthHeader(req.headers);
+      if (!tokenText) {
+        return next(createError(400, "Invalid authorization token format"));
+      }
+      ctx.token = tokenText.toLowerCase() === "null" ? void 0 : tokenText;
+    }
+    req.ctx = ctx;
+    next();
+  } catch (e) {
+    next(e);
+  }
+}
+var loginCache = new LRUCache({
+  max: 32
+  // TODO: Make this configurable
+});
+async function cachedLogin(options) {
+  const key = JSON.stringify(options);
+  const cached = loginCache.get(key);
+  if (cached) {
+    return cached;
+  }
+  const state = await loginToState(options);
+  loginCache.set(key, state);
+  return state;
+}
+function makeCheckAuthorized(allowedOrgName) {
+  return async (req, _res, next) => {
+    if (!req.ctx?.token) {
+      return next(createError(401, "Unauthorized"));
+    }
+    try {
+      const orgName = parseHeader(req.headers, "x-bt-org-name");
+      if (!orgName) {
+        return next(createError(400, "Missing x-bt-org-name header"));
+      }
+      if (allowedOrgName && allowedOrgName !== orgName) {
+        const errorMessage = `Org '${orgName}' is not allowed. Only org '${allowedOrgName}' is allowed.`;
+        return next(createError(403, errorMessage));
+      }
+      const state = await cachedLogin({
+        apiKey: req.ctx?.token,
+        orgName
+      });
+      req.ctx.state = state;
+      next();
+    } catch (e) {
+      console.error("Authorization error:", e);
+      return next(createError(401, "Unauthorized"));
+    }
+  };
+}
+function parseBraintrustAuthHeader(headers) {
+  const tokenString = parseHeader(headers, BRAINTRUST_AUTH_TOKEN_HEADER);
+  return tokenString ?? parseAuthHeader(headers) ?? void 0;
+}
+function parseHeader(headers, headerName) {
+  const token = headers[headerName];
+  let tokenString;
+  if (typeof token === "string") {
+    tokenString = token;
+  } else if (Array.isArray(token) && token.length > 0) {
+    tokenString = token[0];
+  }
+  return tokenString;
+}
+function checkOrigin(requestOrigin, callback) {
+  if (!requestOrigin) {
+    return callback(null, true);
+  }
+  for (const origin of WHITELISTED_ORIGINS || []) {
+    if (origin instanceof RegExp && origin.test(requestOrigin) || origin === requestOrigin) {
+      return callback(null, requestOrigin);
+    }
+  }
+  return callback(null, false);
+}
+var BRAINTRUST_AUTH_TOKEN_HEADER = "x-bt-auth-token";
+var ORIGIN_HEADER = "origin";
+var PROJECT_ID_HEADER = "x-bt-project-id";
+function extractAllowedOrigin(originHeader) {
+  let allowedOrigin = MAIN_ORIGIN;
+  checkOrigin(originHeader, (err, origin) => {
+    if (!err && originHeader && origin) {
+      allowedOrigin = originHeader;
+    }
+  });
+  return allowedOrigin;
+}
+var MAIN_ORIGIN = "https://www.braintrust.dev";
+var WHITELISTED_ORIGINS = [
+  MAIN_ORIGIN,
+  "https://www.braintrustdata.com",
+  new RegExp("https://.*.preview.braintrust.dev")
+].concat(
+  process.env.WHITELISTED_ORIGIN ? [process.env.WHITELISTED_ORIGIN] : []
+).concat(
+  process.env.BRAINTRUST_APP_URL ? [process.env.BRAINTRUST_APP_URL] : []
+);
+function parseAuthHeader(headers) {
+  const authHeader = headers["authorization"];
+  let authValue = null;
+  if (Array.isArray(authHeader)) {
+    authValue = authHeader[authHeader.length - 1];
+  } else {
+    authValue = authHeader;
+  }
+  if (!authValue) {
+    return null;
+  }
+  const parts = authValue.split(" ");
+  if (parts.length !== 2) {
+    return null;
+  }
+  return parts[1];
+}
+var baseAllowedHeaders = [
+  "Content-Type",
+  "X-Amz-Date",
+  "Authorization",
+  "X-Api-Key",
+  "X-Amz-Security-Token",
+  "x-bt-auth-token",
+  "x-bt-parent",
+  // These are eval-specific
+  "x-bt-org-name",
+  "x-bt-project-id",
+  "x-bt-stream-fmt",
+  "x-bt-use-cache",
+  "x-stainless-os",
+  "x-stainless-lang",
+  "x-stainless-package-version",
+  "x-stainless-runtime",
+  "x-stainless-runtime-version",
+  "x-stainless-arch"
+];
+
+// dev/stream.ts
+function serializeSSEEvent(event) {
+  return Object.entries(event).filter(([_key, value]) => value !== void 0).map(([key, value]) => `${key}: ${value}`).join("\n") + "\n\n";
+}
+
+// dev/types.ts
+import { z as z12 } from "zod/v3";
+var evalBodySchema = z12.object({
+  name: z12.string(),
+  parameters: z12.record(z12.string(), z12.unknown()).nullish(),
+  data: RunEval.shape.data,
+  scores: z12.array(
+    z12.object({
+      function_id: FunctionId,
+      name: z12.string()
+    })
+  ).nullish(),
+  experiment_name: z12.string().nullish(),
+  project_id: z12.string().nullish(),
+  parent: InvokeParent.optional(),
+  stream: z12.boolean().optional()
+});
+var staticParametersSchema = z12.record(
+  z12.string(),
+  z12.union([
+    z12.object({
+      type: z12.literal("prompt"),
+      default: PromptData.optional(),
+      description: z12.string().optional()
+    }),
+    z12.object({
+      type: z12.literal("data"),
+      schema: z12.record(z12.unknown()),
+      default: z12.unknown().optional(),
+      description: z12.string().optional()
+    })
+  ])
+);
+var parametersSchema = z12.object({
+  type: z12.literal("object"),
+  properties: z12.record(z12.string(), z12.record(z12.unknown())),
+  required: z12.array(z12.string()).optional(),
+  additionalProperties: z12.boolean().optional()
+});
+var parametersSourceSchema = z12.object({
+  parametersId: z12.string().optional(),
+  slug: z12.string(),
+  name: z12.string(),
+  projectId: z12.string().optional(),
+  version: z12.string().optional()
+});
+var parametersContainerSchema = z12.object({
+  type: z12.literal("braintrust.parameters"),
+  schema: parametersSchema,
+  source: parametersSourceSchema
+});
+var staticParametersContainerSchema = z12.object({
+  type: z12.literal("braintrust.staticParameters"),
+  schema: staticParametersSchema,
+  source: z12.null()
+});
+var serializedParametersContainerSchema = z12.union([
+  parametersContainerSchema,
+  staticParametersContainerSchema,
+  // keeping this type here since old versions of the SDK will still pass the unwrapped schema and we need to handle this in the app
+  staticParametersSchema
+]);
+var evaluatorDefinitionSchema = z12.object({
+  parameters: serializedParametersContainerSchema.optional(),
+  scores: z12.array(z12.object({ name: z12.string() })).optional()
+});
+var evaluatorDefinitionsSchema = z12.record(
+  z12.string(),
+  evaluatorDefinitionSchema
+);
+
+// dev/server.ts
+import { z as z14 } from "zod/v3";
+import { ValidationError } from "ajv";
+
+// src/framework2.ts
+import { z as z13 } from "zod/v3";
+
+// src/zod/utils.ts
+import { zodToJsonSchema as zodToJsonSchemaV3 } from "zod-to-json-schema";
+import * as z42 from "zod/v4";
+function isZodV4(zodObject) {
+  return typeof zodObject === "object" && zodObject !== null && "_zod" in zodObject && zodObject._zod !== void 0;
+}
+function zodToJsonSchema(schema) {
+  if (isZodV4(schema)) {
+    return z42.toJSONSchema(schema, {
+      target: "draft-7"
+    });
+  }
+  return zodToJsonSchemaV3(schema);
+}
+
+// src/framework2.ts
+var currentFilename = typeof __filename !== "undefined" ? __filename : "unknown";
+var ProjectBuilder = class {
+  create(opts) {
+    return new Project2(opts);
+  }
+};
+var projects = new ProjectBuilder();
+var Project2 = class {
+  name;
+  id;
+  tools;
+  prompts;
+  parameters;
+  scorers;
+  _publishableCodeFunctions = [];
+  _publishablePrompts = [];
+  _publishableParameters = [];
+  constructor(args) {
+    _initializeSpanContext();
+    this.name = "name" in args ? args.name : void 0;
+    this.id = "id" in args ? args.id : void 0;
+    this.tools = new ToolBuilder(this);
+    this.prompts = new PromptBuilder(this);
+    this.parameters = new ParametersBuilder(this);
+    this.scorers = new ScorerBuilder(this);
+  }
+  addPrompt(prompt) {
+    this._publishablePrompts.push(prompt);
+    if (globalThis._lazy_load) {
+      globalThis._evals.prompts.push(prompt);
+    }
+  }
+  addParameters(parameters) {
+    this._publishableParameters.push(parameters);
+    if (globalThis._lazy_load) {
+      if (globalThis._evals.parameters == null)
+        globalThis._evals.parameters = [];
+      globalThis._evals.parameters.push(parameters);
     }
   }
-}
-var warning = (text) => `Warning: ${text}`;
-function logError2(e, verbose) {
-  if (!verbose) {
-    console.error(`${e}`);
-  } else {
-    console.error(e);
+  addCodeFunction(fn) {
+    this._publishableCodeFunctions.push(fn);
+    if (globalThis._lazy_load) {
+      globalThis._evals.functions.push(fn);
+    }
   }
-}
-function accumulateScores(accumulator, scores) {
-  for (const [name, score] of Object.entries(scores)) {
-    if (score === null || score === void 0) {
-      continue;
+  async publish() {
+    if (globalThis._lazy_load) {
+      console.warn("publish() is a no-op when running `braintrust push`.");
+      return;
     }
-    const existing = accumulator[name] ?? { total: 0, count: 0 };
-    accumulator[name] = {
-      total: existing.total + score,
-      count: existing.count + 1
-    };
+    await login();
+    const projectMap = new ProjectNameIdMap();
+    const functionDefinitions = [];
+    if (this._publishableCodeFunctions.length > 0) {
+      console.warn(
+        "Code functions cannot be published directly. Use `braintrust push` instead."
+      );
+    }
+    if (this._publishablePrompts.length > 0) {
+      for (const prompt of this._publishablePrompts) {
+        const functionDefinition = await prompt.toFunctionDefinition(projectMap);
+        functionDefinitions.push(functionDefinition);
+      }
+    }
+    await _internalGetGlobalState().apiConn().post_json("insert-functions", {
+      functions: functionDefinitions
+    });
   }
-}
-function ensureScoreAccumulator(results) {
-  const accumulator = {};
-  for (const result of results) {
-    accumulateScores(accumulator, result.scores);
+};
+var ToolBuilder = class {
+  constructor(project) {
+    this.project = project;
   }
-  return accumulator;
-}
-function buildLocalSummary(evaluator, results, precomputedScores) {
-  const scoresByName = precomputedScores ?? ensureScoreAccumulator(results);
-  return {
-    projectName: evaluator.projectName,
-    experimentName: evaluator.evalName,
-    scores: Object.fromEntries(
-      Object.entries(scoresByName).map(([name, { total, count }]) => [
-        name,
-        {
-          name,
-          score: count === 0 ? 0 : total / count,
-          improvements: 0,
-          regressions: 0
+  taskCounter = 0;
+  // This type definition is just a catch all so that the implementation can be
+  // less specific than the two more specific declarations above.
+  create(opts) {
+    this.taskCounter++;
+    opts = opts ?? {};
+    const { handler, name, slug, parameters, returns, ...rest } = opts;
+    let resolvedName = name ?? handler.name;
+    if (resolvedName.trim().length === 0) {
+      resolvedName = `Tool ${isomorph_default.basename(currentFilename)} ${this.taskCounter}`;
+    }
+    const tool = new CodeFunction(this.project, {
+      handler,
+      name: resolvedName,
+      slug: slug ?? slugify(resolvedName, { lower: true, strict: true }),
+      type: "tool",
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions
+      parameters,
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions
+      returns,
+      ...rest
+    });
+    this.project.addCodeFunction(tool);
+    return tool;
+  }
+};
+var ScorerBuilder = class {
+  constructor(project) {
+    this.project = project;
+  }
+  taskCounter = 0;
+  create(opts) {
+    this.taskCounter++;
+    let resolvedName = opts.name;
+    if (!resolvedName && "handler" in opts) {
+      resolvedName = opts.handler.name;
+    }
+    if (!resolvedName || resolvedName.trim().length === 0) {
+      resolvedName = `Scorer ${isomorph_default.basename(currentFilename)} ${this.taskCounter}`;
+    }
+    const slug = opts.slug ?? slugify(resolvedName, { lower: true, strict: true });
+    if ("handler" in opts) {
+      const scorer = new CodeFunction(this.project, {
+        ...opts,
+        name: resolvedName,
+        slug,
+        type: "scorer"
+      });
+      this.project.addCodeFunction(scorer);
+    } else {
+      const promptBlock = "messages" in opts ? {
+        type: "chat",
+        messages: opts.messages
+      } : {
+        type: "completion",
+        content: opts.prompt
+      };
+      const promptData = {
+        prompt: promptBlock,
+        options: {
+          model: opts.model,
+          params: opts.params
+        },
+        parser: {
+          type: "llm_classifier",
+          use_cot: opts.useCot,
+          choice_scores: opts.choiceScores
         }
-      ])
-    )
-  };
-}
-function reportFailures(evaluator, failingResults, { verbose, jsonl }) {
-  if (failingResults.length > 0) {
-    console.error(
-      warning(
-        `Evaluator ${evaluator.evalName} failed with ${failingResults.length} error${failingResults.length === 1 ? "" : "s"}. This evaluation ("${evaluator.evalName}") will not be fully logged.`
-      )
-    );
-    if (jsonl) {
-      console.log(
-        JSON.stringify({
-          evaluatorName: evaluator.evalName,
-          errors: failingResults.map(
-            (r) => `${r.error instanceof Error ? r.error.stack : r.error}`
-          )
-        })
+      };
+      const codePrompt = new CodePrompt(
+        this.project,
+        promptData,
+        [],
+        {
+          ...opts,
+          name: resolvedName,
+          slug
+        },
+        "scorer"
       );
-    } else {
-      for (const result of failingResults) {
-        logError2(result.error, verbose);
-      }
+      this.project.addPrompt(codePrompt);
     }
-    if (!verbose && !jsonl) {
-      console.error(warning("Add --verbose to see full stack traces."));
+  }
+};
+var CodeFunction = class {
+  constructor(project, opts) {
+    this.project = project;
+    this.handler = opts.handler;
+    this.name = opts.name;
+    this.slug = opts.slug;
+    this.description = opts.description;
+    this.type = opts.type;
+    this.ifExists = opts.ifExists;
+    this.metadata = opts.metadata;
+    this.parameters = opts.parameters;
+    this.returns = opts.returns;
+    if (this.returns && !this.parameters) {
+      throw new Error("parameters are required if return type is defined");
     }
   }
-}
-var defaultReporter = {
-  name: "Braintrust default reporter",
-  async reportEval(evaluator, result, { verbose, jsonl }) {
-    const { results, summary } = result;
-    const failingResults = results.filter(
-      (r) => r.error !== void 0
-    );
-    if (failingResults.length > 0) {
-      reportFailures(evaluator, failingResults, { verbose, jsonl });
-    }
-    if (jsonl) {
-      isomorph_default.writeln(JSON.stringify(summary));
-    } else {
-      isomorph_default.writeln("Experiment summary");
-      isomorph_default.writeln("==================");
-      if (summary.comparisonExperimentName) {
-        isomorph_default.writeln(
-          `${summary.comparisonExperimentName} (baseline) <- ${summary.experimentName} (comparison)`
-        );
-        isomorph_default.writeln("");
-      }
-      const hasScores = Object.keys(summary.scores).length > 0;
-      const hasMetrics = Object.keys(summary.metrics ?? {}).length > 0;
-      const hasComparison = !!summary.comparisonExperimentName;
-      if (hasScores || hasMetrics) {
-        if (hasComparison) {
-          isomorph_default.writeln(
-            "Name                Value      Change     Improvements Regressions"
-          );
-          isomorph_default.writeln(
-            "----------------------------------------------------------------"
-          );
-        }
-        for (const score of Object.values(summary.scores)) {
-          const scorePercent = (score.score * 100).toFixed(2);
-          const scoreValue = `${scorePercent}%`;
-          if (hasComparison) {
-            let diffString = "-";
-            if (!isEmpty2(score.diff)) {
-              const diffPercent = (score.diff * 100).toFixed(2);
-              const diffSign = score.diff > 0 ? "+" : "";
-              diffString = `${diffSign}${diffPercent}%`;
-            }
-            const improvements = score.improvements > 0 ? score.improvements.toString() : "-";
-            const regressions = score.regressions > 0 ? score.regressions.toString() : "-";
-            isomorph_default.writeln(
-              `${score.name.padEnd(18)} ${scoreValue.padStart(10)} ${diffString.padStart(10)} ${improvements.padStart(12)} ${regressions.padStart(11)}`
-            );
-          } else {
-            isomorph_default.writeln(`${score.name.padEnd(20)} ${scoreValue.padStart(15)}`);
-          }
-        }
-        for (const metric of Object.values(summary.metrics ?? {})) {
-          const fractionDigits = Number.isInteger(metric.metric) ? 0 : 2;
-          const formattedValue = metric.metric.toFixed(fractionDigits);
-          const metricValue = metric.unit === "$" ? `${metric.unit}${formattedValue}` : `${formattedValue}${metric.unit}`;
-          if (hasComparison) {
-            let diffString = "-";
-            if (!isEmpty2(metric.diff)) {
-              const diffPercent = (metric.diff * 100).toFixed(2);
-              const diffSign = metric.diff > 0 ? "+" : "";
-              diffString = `${diffSign}${diffPercent}%`;
-            }
-            const improvements = metric.improvements > 0 ? metric.improvements.toString() : "-";
-            const regressions = metric.regressions > 0 ? metric.regressions.toString() : "-";
-            isomorph_default.writeln(
-              `${metric.name.padEnd(18)} ${metricValue.padStart(10)} ${diffString.padStart(10)} ${improvements.padStart(12)} ${regressions.padStart(11)}`
-            );
+  handler;
+  name;
+  slug;
+  type;
+  description;
+  parameters;
+  returns;
+  ifExists;
+  metadata;
+  key() {
+    return JSON.stringify([
+      this.project.id ?? "",
+      this.project.name ?? "",
+      this.slug
+    ]);
+  }
+};
+var CodePrompt = class {
+  project;
+  name;
+  slug;
+  prompt;
+  ifExists;
+  description;
+  id;
+  functionType;
+  toolFunctions;
+  metadata;
+  constructor(project, prompt, toolFunctions, opts, functionType) {
+    this.project = project;
+    this.name = opts.name;
+    this.slug = opts.slug;
+    this.prompt = prompt;
+    this.toolFunctions = toolFunctions;
+    this.ifExists = opts.ifExists;
+    this.description = opts.description;
+    this.id = opts.id;
+    this.functionType = functionType;
+    this.metadata = opts.metadata;
+  }
+  async toFunctionDefinition(projectNameToId) {
+    const prompt_data = {
+      ...this.prompt
+    };
+    if (this.toolFunctions.length > 0) {
+      const resolvableToolFunctions = await Promise.all(
+        this.toolFunctions.map(async (fn) => {
+          if ("slug" in fn) {
+            return {
+              type: "slug",
+              project_id: await projectNameToId.resolve(fn.project),
+              slug: fn.slug
+            };
           } else {
-            isomorph_default.writeln(
-              `${metric.name.padEnd(20)} ${metricValue.padStart(15)}`
-            );
+            return fn;
           }
-        }
-      }
-      if (summary.experimentUrl) {
-        isomorph_default.writeln("");
-        isomorph_default.writeln(`View results for ${summary.experimentName}`);
-        isomorph_default.writeln(`See results at ${summary.experimentUrl}`);
-      }
+        })
+      );
+      prompt_data.tool_functions = // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+      resolvableToolFunctions;
     }
-    isomorph_default.writeln("");
-    return failingResults.length === 0;
-  },
-  async reportRun(evalReports) {
-    return evalReports.every((r) => r);
+    return {
+      project_id: await projectNameToId.resolve(this.project),
+      name: this.name,
+      slug: this.slug,
+      description: this.description ?? "",
+      function_data: {
+        type: "prompt"
+      },
+      function_type: this.functionType,
+      prompt_data,
+      if_exists: this.ifExists,
+      metadata: this.metadata
+    };
   }
 };
-
-// dev/errorHandler.ts
-import { z as z11 } from "zod/v3";
-var errorHandler = (err, req, res, next) => {
-  if ("status" in err) {
-    res.status(err.status).json({
-      error: {
-        message: err.message,
-        status: err.status
-      }
-    });
-    return;
+var PromptBuilder = class {
+  constructor(project) {
+    this.project = project;
   }
-  if (err instanceof z11.ZodError) {
-    res.status(400).json({
-      error: {
-        message: "Invalid request",
-        errors: err.errors
+  create(opts) {
+    const toolFunctions = [];
+    const rawTools = [];
+    for (const tool of opts.tools ?? []) {
+      if (tool instanceof CodeFunction) {
+        toolFunctions.push(tool);
+      } else if ("type" in tool && !("function" in tool)) {
+        toolFunctions.push(tool);
+      } else {
+        rawTools.push(tool);
       }
+    }
+    const slug = opts.slug ?? slugify(opts.name, { lower: true, strict: true });
+    const promptData = promptDefinitionToPromptData(opts, rawTools);
+    const promptRow = {
+      id: opts.id,
+      _xact_id: opts.version ? loadPrettyXact(opts.version) : void 0,
+      name: opts.name,
+      slug,
+      prompt_data: promptData,
+      ...this.project.id !== void 0 ? { project_id: this.project.id } : {}
+    };
+    const prompt = new Prompt2(
+      promptRow,
+      {},
+      // It doesn't make sense to specify defaults here.
+      opts.noTrace ?? false
+    );
+    const codePrompt = new CodePrompt(this.project, promptData, toolFunctions, {
+      ...opts,
+      slug
     });
-    return;
+    this.project.addPrompt(codePrompt);
+    return prompt;
   }
-  console.error("Internal server error", err);
-  res.status(500).json({
-    error: {
-      message: "Internal server error",
-      status: 500
-    }
-  });
 };
-
-// dev/authorize.ts
-import createError from "http-errors";
-function authorizeRequest(req, res, next) {
-  try {
-    const ctx = {
-      appOrigin: extractAllowedOrigin(req.headers[ORIGIN_HEADER]),
-      token: void 0,
-      state: void 0,
-      projectId: parseHeader(req.headers, PROJECT_ID_HEADER)
+var CodeParameters = class {
+  project;
+  name;
+  slug;
+  description;
+  schema;
+  ifExists;
+  metadata;
+  constructor(project, opts) {
+    this.project = project;
+    this.name = opts.name;
+    this.slug = opts.slug;
+    this.description = opts.description;
+    this.schema = opts.schema;
+    this.ifExists = opts.ifExists;
+    this.metadata = opts.metadata;
+  }
+  async toFunctionDefinition(projectNameToId) {
+    return {
+      project_id: await projectNameToId.resolve(this.project),
+      name: this.name,
+      slug: this.slug,
+      description: this.description ?? "",
+      function_type: "parameters",
+      function_data: {
+        type: "parameters",
+        data: {},
+        __schema: serializeEvalParameterstoParametersSchema(this.schema)
+      },
+      if_exists: this.ifExists,
+      metadata: this.metadata
     };
-    if (req.headers.authorization || req.headers[BRAINTRUST_AUTH_TOKEN_HEADER]) {
-      const tokenText = parseBraintrustAuthHeader(req.headers);
-      if (!tokenText) {
-        return next(createError(400, "Invalid authorization token format"));
-      }
-      ctx.token = tokenText.toLowerCase() === "null" ? void 0 : tokenText;
-    }
-    req.ctx = ctx;
-    next();
-  } catch (e) {
-    next(e);
   }
-}
-var loginCache = new LRUCache({
-  max: 32
-  // TODO: Make this configurable
-});
-async function cachedLogin(options) {
-  const key = JSON.stringify(options);
-  const cached = loginCache.get(key);
-  if (cached) {
-    return cached;
+};
+var ParametersBuilder = class {
+  constructor(project) {
+    this.project = project;
   }
-  const state = await loginToState(options);
-  loginCache.set(key, state);
-  return state;
+  create(opts) {
+    const slug = opts.slug ?? slugify(opts.name, { lower: true, strict: true });
+    const codeParameters = new CodeParameters(this.project, {
+      name: opts.name,
+      slug,
+      description: opts.description,
+      schema: opts.schema,
+      ifExists: opts.ifExists,
+      metadata: opts.metadata
+    });
+    this.project.addParameters(codeParameters);
+    return opts.schema;
+  }
+};
+function serializeEvalParametersToStaticParametersSchema(parameters) {
+  return Object.fromEntries(
+    Object.entries(parameters).map(([name, value]) => {
+      if ("type" in value && value.type === "prompt") {
+        return [
+          name,
+          {
+            type: "prompt",
+            default: value.default ? promptDefinitionToPromptData(value.default) : void 0,
+            description: value.description
+          }
+        ];
+      } else {
+        const schemaObj = zodToJsonSchema(value);
+        return [
+          name,
+          {
+            type: "data",
+            schema: schemaObj,
+            default: schemaObj.default,
+            description: schemaObj.description
+          }
+        ];
+      }
+    })
+  );
 }
-function makeCheckAuthorized(allowedOrgName) {
-  return async (req, _res, next) => {
-    if (!req.ctx?.token) {
-      return next(createError(401, "Unauthorized"));
-    }
-    try {
-      const orgName = parseHeader(req.headers, "x-bt-org-name");
-      if (!orgName) {
-        return next(createError(400, "Missing x-bt-org-name header"));
+function serializeEvalParameterstoParametersSchema(parameters) {
+  const properties = {};
+  const required = [];
+  for (const [name, value] of Object.entries(parameters)) {
+    if ("type" in value && value.type === "prompt") {
+      const defaultPromptData = value.default ? promptDefinitionToPromptData(value.default) : void 0;
+      properties[name] = {
+        type: "object",
+        "x-bt-type": "prompt",
+        ...value.description ? { description: value.description } : {},
+        ...defaultPromptData ? { default: defaultPromptData } : {}
+      };
+      if (!defaultPromptData) {
+        required.push(name);
       }
-      if (allowedOrgName && allowedOrgName !== orgName) {
-        const errorMessage = `Org '${orgName}' is not allowed. Only org '${allowedOrgName}' is allowed.`;
-        return next(createError(403, errorMessage));
+    } else {
+      const schemaObj = zodToJsonSchema(value);
+      properties[name] = schemaObj;
+      if (!("default" in schemaObj)) {
+        required.push(name);
       }
-      const state = await cachedLogin({
-        apiKey: req.ctx?.token,
-        orgName
-      });
-      req.ctx.state = state;
-      next();
-    } catch (e) {
-      console.error("Authorization error:", e);
-      return next(createError(401, "Unauthorized"));
     }
+  }
+  return {
+    type: "object",
+    properties,
+    ...required.length > 0 ? { required } : {},
+    additionalProperties: true
   };
 }
-function parseBraintrustAuthHeader(headers) {
-  const tokenString = parseHeader(headers, BRAINTRUST_AUTH_TOKEN_HEADER);
-  return tokenString ?? parseAuthHeader(headers) ?? void 0;
-}
-function parseHeader(headers, headerName) {
-  const token = headers[headerName];
-  let tokenString;
-  if (typeof token === "string") {
-    tokenString = token;
-  } else if (Array.isArray(token) && token.length > 0) {
-    tokenString = token[0];
+function serializeRemoteEvalParametersContainer(parameters) {
+  if (RemoteEvalParameters.isParameters(parameters)) {
+    return {
+      type: "braintrust.parameters",
+      // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+      schema: parameters.schema,
+      source: {
+        parametersId: parameters.id,
+        slug: parameters.slug,
+        name: parameters.name,
+        projectId: parameters.projectId,
+        version: parameters.version
+      }
+    };
   }
-  return tokenString;
+  return {
+    type: "braintrust.staticParameters",
+    schema: serializeEvalParametersToStaticParametersSchema(parameters),
+    source: null
+  };
 }
-function checkOrigin(requestOrigin, callback) {
-  if (!requestOrigin) {
-    return callback(null, true);
-  }
-  for (const origin of WHITELISTED_ORIGINS || []) {
-    if (origin instanceof RegExp && origin.test(requestOrigin) || origin === requestOrigin) {
-      return callback(null, requestOrigin);
+var ProjectNameIdMap = class {
+  nameToId = {};
+  idToName = {};
+  async getId(projectName) {
+    if (!(projectName in this.nameToId)) {
+      const response = await _internalGetGlobalState().appConn().post_json("api/project/register", {
+        project_name: projectName
+      });
+      const result = z13.object({
+        project: Project
+      }).parse(response);
+      const projectId = result.project.id;
+      this.nameToId[projectName] = projectId;
+      this.idToName[projectId] = projectName;
     }
+    return this.nameToId[projectName];
   }
-  return callback(null, false);
-}
-var BRAINTRUST_AUTH_TOKEN_HEADER = "x-bt-auth-token";
-var ORIGIN_HEADER = "origin";
-var PROJECT_ID_HEADER = "x-bt-project-id";
-function extractAllowedOrigin(originHeader) {
-  let allowedOrigin = MAIN_ORIGIN;
-  checkOrigin(originHeader, (err, origin) => {
-    if (!err && originHeader && origin) {
-      allowedOrigin = originHeader;
+  async getName(projectId) {
+    if (!(projectId in this.idToName)) {
+      const response = await _internalGetGlobalState().appConn().post_json("api/project/get", {
+        id: projectId
+      });
+      const result = z13.array(Project).nonempty().parse(response);
+      const projectName = result[0].name;
+      this.idToName[projectId] = projectName;
+      this.nameToId[projectName] = projectId;
     }
-  });
-  return allowedOrigin;
-}
-var MAIN_ORIGIN = "https://www.braintrust.dev";
-var WHITELISTED_ORIGINS = [
-  MAIN_ORIGIN,
-  "https://www.braintrustdata.com",
-  new RegExp("https://.*.preview.braintrust.dev")
-].concat(
-  process.env.WHITELISTED_ORIGIN ? [process.env.WHITELISTED_ORIGIN] : []
-).concat(
-  process.env.BRAINTRUST_APP_URL ? [process.env.BRAINTRUST_APP_URL] : []
-);
-function parseAuthHeader(headers) {
-  const authHeader = headers["authorization"];
-  let authValue = null;
-  if (Array.isArray(authHeader)) {
-    authValue = authHeader[authHeader.length - 1];
-  } else {
-    authValue = authHeader;
-  }
-  if (!authValue) {
-    return null;
-  }
-  const parts = authValue.split(" ");
-  if (parts.length !== 2) {
-    return null;
+    return this.idToName[projectId];
   }
-  return parts[1];
-}
-var baseAllowedHeaders = [
-  "Content-Type",
-  "X-Amz-Date",
-  "Authorization",
-  "X-Api-Key",
-  "X-Amz-Security-Token",
-  "x-bt-auth-token",
-  "x-bt-parent",
-  // These are eval-specific
-  "x-bt-org-name",
-  "x-bt-project-id",
-  "x-bt-stream-fmt",
-  "x-bt-use-cache",
-  "x-stainless-os",
-  "x-stainless-lang",
-  "x-stainless-package-version",
-  "x-stainless-runtime",
-  "x-stainless-runtime-version",
-  "x-stainless-arch"
-];
-
-// dev/stream.ts
-function serializeSSEEvent(event) {
-  return Object.entries(event).filter(([_key, value]) => value !== void 0).map(([key, value]) => `${key}: ${value}`).join("\n") + "\n\n";
-}
-
-// dev/types.ts
-import { z as z12 } from "zod/v3";
-var evalBodySchema = z12.object({
-  name: z12.string(),
-  parameters: z12.record(z12.string(), z12.unknown()).nullish(),
-  data: RunEval.shape.data,
-  scores: z12.array(
-    z12.object({
-      function_id: FunctionId,
-      name: z12.string()
-    })
-  ).nullish(),
-  experiment_name: z12.string().nullish(),
-  project_id: z12.string().nullish(),
-  parent: InvokeParent.optional(),
-  stream: z12.boolean().optional()
-});
-var evalParametersSerializedSchema = z12.record(
-  z12.string(),
-  z12.union([
-    z12.object({
-      type: z12.literal("prompt"),
-      default: PromptData.optional(),
-      description: z12.string().optional()
-    }),
-    z12.object({
-      type: z12.literal("data"),
-      schema: z12.record(z12.unknown()),
-      // JSON Schema
-      default: z12.unknown().optional(),
-      description: z12.string().optional()
-    })
-  ])
-);
-var evaluatorDefinitionSchema = z12.object({
-  parameters: evalParametersSerializedSchema.optional()
-});
-var evaluatorDefinitionsSchema = z12.record(
-  z12.string(),
-  evaluatorDefinitionSchema
-);
-
-// dev/server.ts
-import { z as z13 } from "zod/v3";
-
-// src/zod/utils.ts
-import { zodToJsonSchema as zodToJsonSchemaV3 } from "zod-to-json-schema";
-import * as z42 from "zod/v4";
-function isZodV4(zodObject) {
-  return typeof zodObject === "object" && zodObject !== null && "_zod" in zodObject && zodObject._zod !== void 0;
-}
-function zodToJsonSchema(schema) {
-  if (isZodV4(schema)) {
-    return z42.toJSONSchema(schema, {
-      target: "draft-7"
-    });
+  async resolve(project) {
+    if (project.id) {
+      return project.id;
+    }
+    return this.getId(project.name);
   }
-  return zodToJsonSchemaV3(schema);
-}
+};
 
 // dev/server.ts
 function runDevServer(evaluators, opts) {
@@ -10835,20 +11527,27 @@ function runDevServer(evaluators, opts) {
   app.get("/", (req, res) => {
     res.send("Hello, world!");
   });
-  app.get("/list", checkAuthorized, (req, res) => {
-    const evalDefs = Object.fromEntries(
-      Object.entries(allEvaluators).map(([name, evaluator]) => [
-        name,
-        {
-          parameters: evaluator.parameters ? makeEvalParametersSchema(evaluator.parameters) : void 0,
+  app.get(
+    "/list",
+    checkAuthorized,
+    asyncHandler(async (req, res) => {
+      const evalDefs = {};
+      for (const [name, evaluator] of Object.entries(allEvaluators)) {
+        let parameters;
+        if (evaluator.parameters) {
+          const resolvedParams = await Promise.resolve(evaluator.parameters);
+          parameters = serializeRemoteEvalParametersContainer(resolvedParams);
+        }
+        evalDefs[name] = {
+          parameters,
           scores: evaluator.scores.map((score, idx) => ({
             name: scorerName(score, idx)
           }))
-        }
-      ])
-    );
-    res.json(evalDefs);
-  });
+        };
+      }
+      res.json(evalDefs);
+    })
+  );
   app.post(
     "/eval",
     checkAuthorized,
@@ -10873,18 +11572,12 @@ function runDevServer(evaluators, opts) {
         res.status(404).json({ error: `Evaluator '${name}' not found` });
         return;
       }
-      if (evaluator.parameters && Object.keys(evaluator.parameters).length > 0) {
+      if (evaluator.parameters) {
         try {
-          if (!evaluator.parameters) {
-            res.status(400).json({
-              error: `Evaluator '${name}' does not accept parameters`
-            });
-            return;
-          }
-          validateParameters(parameters ?? {}, evaluator.parameters);
+          await validateParameters(parameters ?? {}, evaluator.parameters);
         } catch (e) {
           console.error("Error validating parameters", e);
-          if (e instanceof z13.ZodError || e instanceof Error) {
+          if (e instanceof z14.ZodError || e instanceof ValidationError || e instanceof Error) {
             res.status(400).json({
               error: e.message
             });
@@ -11033,9 +11726,9 @@ async function getDataset(state, data) {
     return data.data;
   }
 }
-var datasetFetchSchema = z13.object({
-  project_id: z13.string(),
-  name: z13.string()
+var datasetFetchSchema = z14.object({
+  project_id: z14.string(),
+  name: z14.string()
 });
 async function getDatasetById({
   state,
@@ -11044,7 +11737,7 @@ async function getDatasetById({
   const dataset = await state.appConn().post_json("api/dataset/get", {
     id: datasetId
   });
-  const parsed = z13.array(datasetFetchSchema).parse(dataset);
+  const parsed = z14.array(datasetFetchSchema).parse(dataset);
   if (parsed.length === 0) {
     throw new Error(`Dataset '${datasetId}' not found`);
   }
@@ -11077,33 +11770,6 @@ function makeScorer(state, name, score, projectId) {
   });
   return ret;
 }
-function makeEvalParametersSchema(parameters) {
-  return Object.fromEntries(
-    Object.entries(parameters).map(([name, value]) => {
-      if ("type" in value && value.type === "prompt") {
-        return [
-          name,
-          {
-            type: "prompt",
-            default: value.default ? promptDefinitionToPromptData(value.default) : void 0,
-            description: value.description
-          }
-        ];
-      } else {
-        const schemaObj = zodToJsonSchema(value);
-        return [
-          name,
-          {
-            type: "data",
-            schema: schemaObj,
-            default: schemaObj.default,
-            description: schemaObj.description
-          }
-        ];
-      }
-    })
-  );
-}
 
 // dev/index.ts
 configureNode();