braintrust 3.7.1 → 3.9.0

package/dev/dist/index.d.mts

@@ -131,6 +131,10 @@ type OtherExperimentLogFields = {
     error: unknown;
     tags: string[];
     scores: Record<string, number | null>;
+    classifications?: Record<string, {
+        id: string;
+        label?: string;
+    }[]>;
     metadata: Record<string, unknown>;
     metrics: Record<string, unknown>;
     datasetRecordId: string;
@@ -287,6 +291,42 @@ declare class SpanComponentsV3 {
     private static fromJsonObj;
 }
 
+/**
+ * The result returned by a classifier function. Unlike `Score`, `id` is
+ * required and the span will be recorded as a classifier span.
+ */
+interface Classification {
+    /**
+     * The name of this classification result. Used as the key in the
+     * `classifications` log record. If omitted, defaults to the classifier
+     * function's name.
+     */
+    name: string;
+    /**
+     * A machine-readable identifier for the classification outcome
+     * (e.g. `"positive"`, `"negative"`, `"neutral"`). This value is stored
+     * in the log and used for programmatic analysis.
+     */
+    id: string;
+    /**
+     * An optional human-readable display label for this outcome. If omitted,
+     * defaults to `id`. Use this when you want a friendlier label in the UI
+     * while keeping a stable `id` for programmatic use.
+     */
+    label?: string;
+    /**
+     * Optional arbitrary metadata to attach to this classification result.
+     */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * The serialized form of a classification stored in the `classifications` log record.
+ */
+interface ClassificationItem {
+    id: string;
+    label: string;
+    metadata?: Record<string, unknown>;
+}
 interface Score {
     name: string;
     score: number | null;
@@ -2114,6 +2154,15 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             }, {
                 type: "scorer";
                 index: number;
+            }>, z.ZodObject<{
+                type: z.ZodLiteral<"classifier">;
+                index: z.ZodNumber;
+            }, "strip", z.ZodTypeAny, {
+                type: "classifier";
+                index: number;
+            }, {
+                type: "classifier";
+                index: number;
             }>]>;
         }, "strip", z.ZodTypeAny, {
             type: "experiment";
@@ -2123,6 +2172,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         }, {
             type: "experiment";
@@ -2132,6 +2184,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         }>, z.ZodObject<{
             type: z.ZodLiteral<"function">;
@@ -2204,6 +2259,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         } | {
             type: "function";
@@ -2236,6 +2294,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         } | {
             type: "function";
@@ -2303,6 +2364,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         } | {
             type: "function";
@@ -2348,6 +2412,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         } | {
             type: "function";
@@ -3940,6 +4007,34 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             bundle_key: z.ZodOptional<z.ZodString>;
             report_key: z.ZodOptional<z.ZodString>;
             topic_names: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+            generation_settings: z.ZodOptional<z.ZodObject<{
+                algorithm: z.ZodEnum<["hdbscan", "kmeans"]>;
+                dimension_reduction: z.ZodEnum<["umap", "pca", "none"]>;
+                sample_size: z.ZodOptional<z.ZodNumber>;
+                n_clusters: z.ZodOptional<z.ZodNumber>;
+                min_cluster_size: z.ZodOptional<z.ZodNumber>;
+                min_samples: z.ZodOptional<z.ZodNumber>;
+                hierarchy_threshold: z.ZodOptional<z.ZodNumber>;
+                naming_model: z.ZodOptional<z.ZodString>;
+            }, "strip", z.ZodTypeAny, {
+                algorithm: "hdbscan" | "kmeans";
+                dimension_reduction: "none" | "umap" | "pca";
+                sample_size?: number | undefined;
+                n_clusters?: number | undefined;
+                min_cluster_size?: number | undefined;
+                min_samples?: number | undefined;
+                hierarchy_threshold?: number | undefined;
+                naming_model?: string | undefined;
+            }, {
+                algorithm: "hdbscan" | "kmeans";
+                dimension_reduction: "none" | "umap" | "pca";
+                sample_size?: number | undefined;
+                n_clusters?: number | undefined;
+                min_cluster_size?: number | undefined;
+                min_samples?: number | undefined;
+                hierarchy_threshold?: number | undefined;
+                naming_model?: string | undefined;
+            }>>;
             distance_threshold: z.ZodOptional<z.ZodNumber>;
         }, "strip", z.ZodTypeAny, {
             type: "topic_map";
@@ -3948,6 +4043,16 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             bundle_key?: string | undefined;
             report_key?: string | undefined;
             topic_names?: Record<string, string> | undefined;
+            generation_settings?: {
+                algorithm: "hdbscan" | "kmeans";
+                dimension_reduction: "none" | "umap" | "pca";
+                sample_size?: number | undefined;
+                n_clusters?: number | undefined;
+                min_cluster_size?: number | undefined;
+                min_samples?: number | undefined;
+                hierarchy_threshold?: number | undefined;
+                naming_model?: string | undefined;
+            } | undefined;
             distance_threshold?: number | undefined;
         }, {
             type: "topic_map";
@@ -3956,6 +4061,16 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             bundle_key?: string | undefined;
             report_key?: string | undefined;
             topic_names?: Record<string, string> | undefined;
+            generation_settings?: {
+                algorithm: "hdbscan" | "kmeans";
+                dimension_reduction: "none" | "umap" | "pca";
+                sample_size?: number | undefined;
+                n_clusters?: number | undefined;
+                min_cluster_size?: number | undefined;
+                min_samples?: number | undefined;
+                hierarchy_threshold?: number | undefined;
+                naming_model?: string | undefined;
+            } | undefined;
             distance_threshold?: number | undefined;
         }>;
     }, "strip", z.ZodTypeAny, {
@@ -3967,6 +4082,16 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             bundle_key?: string | undefined;
             report_key?: string | undefined;
             topic_names?: Record<string, string> | undefined;
+            generation_settings?: {
+                algorithm: "hdbscan" | "kmeans";
+                dimension_reduction: "none" | "umap" | "pca";
+                sample_size?: number | undefined;
+                n_clusters?: number | undefined;
+                min_cluster_size?: number | undefined;
+                min_samples?: number | undefined;
+                hierarchy_threshold?: number | undefined;
+                naming_model?: string | undefined;
+            } | undefined;
             distance_threshold?: number | undefined;
         };
         topic_map_id?: string | undefined;
@@ -3979,6 +4104,16 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             bundle_key?: string | undefined;
             report_key?: string | undefined;
             topic_names?: Record<string, string> | undefined;
+            generation_settings?: {
+                algorithm: "hdbscan" | "kmeans";
+                dimension_reduction: "none" | "umap" | "pca";
+                sample_size?: number | undefined;
+                n_clusters?: number | undefined;
+                min_cluster_size?: number | undefined;
+                min_samples?: number | undefined;
+                hierarchy_threshold?: number | undefined;
+                naming_model?: string | undefined;
+            } | undefined;
             distance_threshold?: number | undefined;
         };
         topic_map_id?: string | undefined;
@@ -4010,6 +4145,16 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             bundle_key?: string | undefined;
             report_key?: string | undefined;
             topic_names?: Record<string, string> | undefined;
+            generation_settings?: {
+                algorithm: "hdbscan" | "kmeans";
+                dimension_reduction: "none" | "umap" | "pca";
+                sample_size?: number | undefined;
+                n_clusters?: number | undefined;
+                min_cluster_size?: number | undefined;
+                min_samples?: number | undefined;
+                hierarchy_threshold?: number | undefined;
+                naming_model?: string | undefined;
+            } | undefined;
             distance_threshold?: number | undefined;
         };
         topic_map_id?: string | undefined;
@@ -4041,6 +4186,16 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             bundle_key?: string | undefined;
             report_key?: string | undefined;
             topic_names?: Record<string, string> | undefined;
+            generation_settings?: {
+                algorithm: "hdbscan" | "kmeans";
+                dimension_reduction: "none" | "umap" | "pca";
+                sample_size?: number | undefined;
+                n_clusters?: number | undefined;
+                min_cluster_size?: number | undefined;
+                min_samples?: number | undefined;
+                hierarchy_threshold?: number | undefined;
+                naming_model?: string | undefined;
+            } | undefined;
             distance_threshold?: number | undefined;
         };
         topic_map_id?: string | undefined;
@@ -4093,6 +4248,34 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
     bundle_key: z.ZodOptional<z.ZodString>;
     report_key: z.ZodOptional<z.ZodString>;
     topic_names: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    generation_settings: z.ZodOptional<z.ZodObject<{
+        algorithm: z.ZodEnum<["hdbscan", "kmeans"]>;
+        dimension_reduction: z.ZodEnum<["umap", "pca", "none"]>;
+        sample_size: z.ZodOptional<z.ZodNumber>;
+        n_clusters: z.ZodOptional<z.ZodNumber>;
+        min_cluster_size: z.ZodOptional<z.ZodNumber>;
+        min_samples: z.ZodOptional<z.ZodNumber>;
+        hierarchy_threshold: z.ZodOptional<z.ZodNumber>;
+        naming_model: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        algorithm: "hdbscan" | "kmeans";
+        dimension_reduction: "none" | "umap" | "pca";
+        sample_size?: number | undefined;
+        n_clusters?: number | undefined;
+        min_cluster_size?: number | undefined;
+        min_samples?: number | undefined;
+        hierarchy_threshold?: number | undefined;
+        naming_model?: string | undefined;
+    }, {
+        algorithm: "hdbscan" | "kmeans";
+        dimension_reduction: "none" | "umap" | "pca";
+        sample_size?: number | undefined;
+        n_clusters?: number | undefined;
+        min_cluster_size?: number | undefined;
+        min_samples?: number | undefined;
+        hierarchy_threshold?: number | undefined;
+        naming_model?: string | undefined;
+    }>>;
     distance_threshold: z.ZodOptional<z.ZodNumber>;
 }, "strip", z.ZodTypeAny, {
     type: "topic_map";
@@ -4101,6 +4284,16 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
     bundle_key?: string | undefined;
     report_key?: string | undefined;
     topic_names?: Record<string, string> | undefined;
+    generation_settings?: {
+        algorithm: "hdbscan" | "kmeans";
+        dimension_reduction: "none" | "umap" | "pca";
+        sample_size?: number | undefined;
+        n_clusters?: number | undefined;
+        min_cluster_size?: number | undefined;
+        min_samples?: number | undefined;
+        hierarchy_threshold?: number | undefined;
+        naming_model?: string | undefined;
+    } | undefined;
     distance_threshold?: number | undefined;
 }, {
     type: "topic_map";
@@ -4109,6 +4302,16 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
     bundle_key?: string | undefined;
     report_key?: string | undefined;
     topic_names?: Record<string, string> | undefined;
+    generation_settings?: {
+        algorithm: "hdbscan" | "kmeans";
+        dimension_reduction: "none" | "umap" | "pca";
+        sample_size?: number | undefined;
+        n_clusters?: number | undefined;
+        min_cluster_size?: number | undefined;
+        min_samples?: number | undefined;
+        hierarchy_threshold?: number | undefined;
+        naming_model?: string | undefined;
+    } | undefined;
     distance_threshold?: number | undefined;
 }>, z.ZodUnknown>]>;
 declare const PromptData: z.ZodObject<{
@@ -8279,8 +8482,8 @@ declare const PromptSessionEvent: z.ZodObject<{
     _xact_id: string;
     prompt_session_id: string;
     function_type?: "llm" | "scorer" | "task" | "tool" | "custom_view" | "preprocessor" | "facet" | "classifier" | "tag" | "parameters" | "sandbox" | null | undefined;
-    _pagination_key?: string | null | undefined;
     tags?: string[] | null | undefined;
+    _pagination_key?: string | null | undefined;
     completion?: unknown;
     prompt_data?: unknown;
     function_data?: unknown;
@@ -8293,8 +8496,8 @@ declare const PromptSessionEvent: z.ZodObject<{
     _xact_id: string;
     prompt_session_id: string;
     function_type?: "llm" | "scorer" | "task" | "tool" | "custom_view" | "preprocessor" | "facet" | "classifier" | "tag" | "parameters" | "sandbox" | null | undefined;
-    _pagination_key?: string | null | undefined;
     tags?: string[] | null | undefined;
+    _pagination_key?: string | null | undefined;
     completion?: unknown;
     prompt_data?: unknown;
     function_data?: unknown;
@@ -10697,6 +10900,12 @@ declare abstract class ContextManager {
     abstract getParentSpanIds(): ContextParentSpanIds | undefined;
     abstract runInContext<R>(span: Span, callback: () => R): R;
     abstract getCurrentSpan(): Span | undefined;
+    /**
+     * Returns the value to store in the ALS bound to a TracingChannel's start event.
+     * In default mode this is the Span itself; in OTEL mode it is the OTEL Context
+     * containing the span so that OTEL's own ALS stores a proper Context object.
+     */
+    wrapSpanForStore(span: Span): unknown;
 }
 declare global {
     var BRAINTRUST_CONTEXT_MANAGER: (new () => ContextManager) | undefined;
@@ -11164,6 +11373,7 @@ type EvalCase<Input, Expected, Metadata> = {
     _xact_id?: TransactionId;
     created?: string | null;
     upsert_id?: string;
+    trialCount?: number;
 } & (Expected extends void ? object : {
     expected: Expected;
 }) & (Metadata extends void ? object : {
@@ -13084,17 +13294,23 @@ type EvalScorerArgs<Input, Output, Expected, Metadata extends BaseMetadata = Def
 };
 type OneOrMoreScores = Score | number | null | Array<Score>;
 type EvalScorer<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = (args: EvalScorerArgs<Input, Output, Expected, Metadata>) => OneOrMoreScores | Promise<OneOrMoreScores>;
+type OneOrMoreClassifications = Classification | Classification[] | null;
+type EvalClassifier<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = (args: EvalScorerArgs<Input, Output, Expected, Metadata>) => OneOrMoreClassifications | Promise<OneOrMoreClassifications>;
 type EvalResult<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata> & {
     output: Output;
-    scores: Record<string, number | null>;
     error: unknown;
     origin?: ObjectReferenceType;
+    scores: Record<string, number | null>;
+    classifications?: Record<string, ClassificationItem[]>;
 };
 type ErrorScoreHandler = (args: {
     rootSpan: Span;
     data: EvalCase<any, any, any>;
     unhandledScores: string[];
 }) => Record<string, number> | undefined | void;
+/**
+ * Defines an evaluator. At least one of `scores` or `classifiers` must be provided.
+ */
 interface Evaluator<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType, Parameters extends EvalParameters = EvalParameters> {
     /**
      * A function that returns a list of inputs, expected outputs, and metadata.
@@ -13105,9 +13321,16 @@ interface Evaluator<Input, Output, Expected, Metadata extends BaseMetadata = Def
      */
     task: EvalTask<Input, Output, Expected, Metadata, Parameters>;
     /**
-     * A set of functions that take an input, output, and expected value and return a score.
+     * A set of functions that take an input, output, and expected value and return a {@link Score}.
+     * At least one of `scores` or `classifiers` must be provided.
+     */
+    scores?: EvalScorer<Input, Output, Expected, Metadata>[];
+    /**
+     * A set of functions that take an input, output, and expected value and return a
+     * {@link Classification}. Results are recorded under the `classifications` column.
+     * At least one of `scores` or `classifiers` must be provided.
      */
-    scores: EvalScorer<Input, Output, Expected, Metadata>[];
+    classifiers?: EvalClassifier<Input, Output, Expected, Metadata>[];
     /**
      * A set of parameters that will be passed to the evaluator.
      * Can be: