braintrust 0.0.133 → 0.0.135

package/dist/browser.js

@@ -5841,13 +5841,15 @@ var objectTypes = z.enum([
   "acl",
   "user",
   "project_score",
-  "project_tag"
+  "project_tag",
+  "function"
 ]);
 var objectTypesWithEvent = z.enum([
   "project",
   "experiment",
   "dataset",
   "prompt",
+  "function",
   "prompt_session"
 ]);
 function getEventObjectType(objectType2) {
@@ -6120,6 +6122,9 @@ var apiKeySchema = z.strictObject({
   user_id: userSchema.shape.id.nullish(),
   org_id: organizationSchema.shape.id.nullish()
 }).openapi("ApiKey");
+var projectSettingsSchema = z.strictObject({
+  comparison_key: z.string().nullish().describe("The key used to join two experiments (defaults to `input`).")
+}).strip();
 var projectBaseSchema = generateBaseTableSchema("project");
 var projectSchema = z.strictObject({
   id: projectBaseSchema.shape.id,
@@ -6129,7 +6134,8 @@ var projectSchema = z.strictObject({
   name: projectBaseSchema.shape.name,
   created: projectBaseSchema.shape.created,
   deleted_at: projectBaseSchema.shape.deleted_at,
-  user_id: projectBaseSchema.shape.user_id
+  user_id: projectBaseSchema.shape.user_id,
+  settings: projectSettingsSchema.nullish()
 }).openapi("Project");
 var datasetBaseSchema = generateBaseTableSchema("dataset", {
   uniqueName: true
@@ -6143,17 +6149,20 @@ var datasetSchema = z.strictObject({
   deleted_at: datasetBaseSchema.shape.deleted_at,
   user_id: datasetBaseSchema.shape.user_id
 }).openapi("Dataset");
+var validRuntimesEnum = z.enum(["node"]);
+var runtimeContextSchema = z.strictObject({
+  runtime: validRuntimesEnum,
+  version: z.string()
+});
 var promptBaseSchema = generateBaseTableSchema("prompt");
-var promptSchema = z.strictObject({
+var promptSchemaObject = z.strictObject({
   id: promptBaseSchema.shape.id,
   // This has to be copy/pasted because zod blows up when there are circular dependencies
   _xact_id: z.string().describe(
     `The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the prompt (see the \`version\` parameter)`
   ),
   project_id: promptBaseSchema.shape.project_id,
-  log_id: z.literal("p").describe(
-    "A literal 'p' which identifies the object as a project prompt"
-  ),
+  log_id: z.literal("p").describe("A literal 'p' which identifies the object as a project prompt"),
   org_id: organizationSchema.shape.id,
   name: promptBaseSchema.shape.name,
   slug: z.string().describe("Unique identifier for the prompt"),
@@ -6162,7 +6171,42 @@ var promptSchema = z.strictObject({
   prompt_data: promptDataSchema.nullish().describe("The prompt, model, and its parameters"),
   tags: z.array(z.string()).nullish().describe("A list of tags for the prompt"),
   metadata: promptBaseSchema.shape.metadata
-}).openapi("Prompt");
+});
+var promptSchema = promptSchemaObject.openapi("Prompt");
+var codeBundleSchema = z.strictObject({
+  runtime_context: z.strictObject({
+    runtime: validRuntimesEnum,
+    version: z.string()
+  }),
+  // This should be a union, once we support code living in different places
+  // Other options should be:
+  //  - a "handler" function that has some signature [does AWS lambda assume it's always called "handler"?]
+  location: z.strictObject({
+    type: z.literal("experiment"),
+    eval_name: z.string(),
+    position: z.union([
+      z.literal("task"),
+      z.strictObject({ score: z.number() })
+    ])
+  }),
+  bundle_id: z.string()
+});
+var functionDataSchema = z.union([
+  z.strictObject({
+    type: z.literal("prompt")
+    // For backwards compatibility reasons, this is hoisted out and stored
+    // in the outer object
+  }),
+  z.strictObject({
+    type: z.literal("code"),
+    data: codeBundleSchema
+  })
+]);
+var functionSchema2 = promptSchemaObject.merge(
+  z.strictObject({
+    function_data: functionDataSchema
+  })
+).openapi("Function");
 var repoInfoSchema = z.strictObject({
   commit: z.string().nullish().describe("SHA of most recent commit"),
   branch: z.string().nullish().describe("Name of the branch the most recent commit belongs to"),
@@ -6415,7 +6459,10 @@ var createProjectSchema = z.strictObject({
   org_name: createProjectBaseSchema.shape.org_name
 }).openapi("CreateProject");
 var patchProjectSchema = z.strictObject({
-  name: projectSchema.shape.name.nullish()
+  name: projectSchema.shape.name.nullish(),
+  settings: projectSchema.shape.settings.describe(
+    "Project settings. Patch operations replace all settings, so make sure you include all settings you want to keep."
+  ).nullish()
 }).openapi("PatchProject");
 var createExperimentSchema = z.strictObject({
   project_id: experimentSchema.shape.project_id,
@@ -6449,12 +6496,27 @@ var createPromptSchema = promptSchema.omit({
   created: true,
   metadata: true
 }).openapi("CreatePrompt");
+var createFunctionSchema = functionSchema2.omit({
+  id: true,
+  _xact_id: true,
+  org_id: true,
+  log_id: true,
+  created: true,
+  metadata: true
+}).openapi("CreateFunction");
 var patchPromptSchema = z.strictObject({
   name: promptSchema.shape.name.nullish(),
   description: promptSchema.shape.description.nullish(),
   prompt_data: promptSchema.shape.prompt_data.nullish(),
   tags: promptSchema.shape.tags.nullish()
 }).openapi("PatchPrompt");
+var patchFunctionSchema = z.strictObject({
+  name: functionSchema2.shape.name.nullish(),
+  description: functionSchema2.shape.description.nullish(),
+  prompt_data: functionSchema2.shape.prompt_data.nullish(),
+  function_data: functionSchema2.shape.function_data.nullish(),
+  tags: functionSchema2.shape.tags.nullish()
+}).openapi("PatchFunction");
 var createRoleBaseSchema = generateBaseTableOpSchema("role");
 var createRoleSchema = z.strictObject({
   name: roleSchema.shape.name,
@@ -6786,6 +6848,7 @@ var promptSessionEventSchema = z.strictObject({
   completion: customTypes.any.describe("Data about the completion"),
   tags: promptSessionEventBaseSchema.shape.tags
 }).openapi("PromptSessionEvent");
+var functionEventBaseSchema = generateBaseEventOpSchema("function");
 var projectLogsEventBaseSchema = generateBaseEventOpSchema("project");
 var projectLogsEventSchema = z.strictObject({
   id: projectLogsEventBaseSchema.shape.id,
@@ -6968,6 +7031,17 @@ var feedbackPromptRequestSchema = makeFeedbackRequestSchema(
   "prompt",
   feedbackPromptItemSchema
 );
+var feedbackFunctionRequestBaseSchema = generateBaseEventFeedbackSchema("function");
+var feedbackFunctionItemSchema = z.strictObject({
+  id: feedbackFunctionRequestBaseSchema.shape.id,
+  comment: feedbackFunctionRequestBaseSchema.shape.comment,
+  metadata: feedbackFunctionRequestBaseSchema.shape.metadata,
+  source: feedbackFunctionRequestBaseSchema.shape.source
+}).openapi("FeedbackFunctionItem");
+var feedbackFunctionRequestSchema = makeFeedbackRequestSchema(
+  "function",
+  feedbackFunctionItemSchema
+);
 var feedbackPromptSessionRequestBaseSchema = generateBaseEventFeedbackSchema("prompt_session");
 var feedbackPromptSessionItemSchema = z.strictObject({
   id: feedbackPromptSessionRequestBaseSchema.shape.id,
@@ -7018,6 +7092,14 @@ var eventObjectSchemas = {
     feedbackItem: feedbackPromptItemSchema,
     feedbackRequest: feedbackPromptRequestSchema
   },
+  function: {
+    event: functionSchema2,
+    fetchResponse: void 0,
+    insertEvent: void 0,
+    insertRequest: void 0,
+    feedbackItem: feedbackFunctionItemSchema,
+    feedbackRequest: feedbackFunctionRequestSchema
+  },
   prompt_session: {
     event: promptSessionEventSchema,
     fetchResponse: void 0,
@@ -9425,6 +9507,9 @@ var ReadonlyExperiment = class extends ObjectFetcher {
   }
 };
 var executionCounter = 0;
+function newId() {
+  return v4_default();
+}
 var SpanImpl = class _SpanImpl {
   // `internalData` contains fields that are not part of the "user-sanitized"
   // set of fields which we want to log in just one of the span rows.
@@ -10230,6 +10315,7 @@ export {
   loadPrompt,
   log,
   login,
+  newId,
   parseCachedHeader,
   startSpan,
   summarize,

package/dist/cli.d.ts

@@ -1,2 +1,35 @@
 #!/usr/bin/env node
+import * as esbuild from "esbuild";
+import { EvaluatorDef, EvaluatorFile, ReporterDef } from "./framework";
+interface BuildSuccess {
+    type: "success";
+    result: esbuild.BuildResult;
+    evaluator: EvaluatorFile;
+    sourceFile: string;
+}
+interface BuildFailure {
+    type: "failure";
+    error: Error;
+    sourceFile: string;
+}
+type BuildResult = BuildSuccess | BuildFailure;
+export interface FileHandle {
+    inFile: string;
+    outFile: string;
+    bundleFile?: string;
+    rebuild: () => Promise<BuildResult>;
+    bundle: () => Promise<esbuild.BuildResult>;
+    watch: () => void;
+    destroy: () => Promise<void>;
+}
+export interface EvaluatorState {
+    evaluators: {
+        sourceFile: string;
+        evaluator: EvaluatorDef<any, any, any, any>;
+        reporter: string | ReporterDef<any> | undefined;
+    }[];
+    reporters: {
+        [reporter: string]: ReporterDef<any>;
+    };
+}
 export {};