braintrust 3.8.0 → 3.9.0

package/dev/dist/index.d.mts

@@ -131,6 +131,10 @@ type OtherExperimentLogFields = {
     error: unknown;
     tags: string[];
     scores: Record<string, number | null>;
+    classifications?: Record<string, {
+        id: string;
+        label?: string;
+    }[]>;
     metadata: Record<string, unknown>;
     metrics: Record<string, unknown>;
     datasetRecordId: string;
@@ -287,6 +291,42 @@ declare class SpanComponentsV3 {
     private static fromJsonObj;
 }
 
+/**
+ * The result returned by a classifier function. Unlike `Score`, `id` is
+ * required and the span will be recorded as a classifier span.
+ */
+interface Classification {
+    /**
+     * The name of this classification result. Used as the key in the
+     * `classifications` log record. If omitted, defaults to the classifier
+     * function's name.
+     */
+    name: string;
+    /**
+     * A machine-readable identifier for the classification outcome
+     * (e.g. `"positive"`, `"negative"`, `"neutral"`). This value is stored
+     * in the log and used for programmatic analysis.
+     */
+    id: string;
+    /**
+     * An optional human-readable display label for this outcome. If omitted,
+     * defaults to `id`. Use this when you want a friendlier label in the UI
+     * while keeping a stable `id` for programmatic use.
+     */
+    label?: string;
+    /**
+     * Optional arbitrary metadata to attach to this classification result.
+     */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * The serialized form of a classification stored in the `classifications` log record.
+ */
+interface ClassificationItem {
+    id: string;
+    label: string;
+    metadata?: Record<string, unknown>;
+}
 interface Score {
     name: string;
     score: number | null;
@@ -2114,6 +2154,15 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             }, {
                 type: "scorer";
                 index: number;
+            }>, z.ZodObject<{
+                type: z.ZodLiteral<"classifier">;
+                index: z.ZodNumber;
+            }, "strip", z.ZodTypeAny, {
+                type: "classifier";
+                index: number;
+            }, {
+                type: "classifier";
+                index: number;
             }>]>;
         }, "strip", z.ZodTypeAny, {
             type: "experiment";
@@ -2123,6 +2172,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         }, {
             type: "experiment";
@@ -2132,6 +2184,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         }>, z.ZodObject<{
             type: z.ZodLiteral<"function">;
@@ -2204,6 +2259,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         } | {
             type: "function";
@@ -2236,6 +2294,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         } | {
             type: "function";
@@ -2303,6 +2364,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         } | {
             type: "function";
@@ -2348,6 +2412,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         } | {
             type: "function";
@@ -11306,6 +11373,7 @@ type EvalCase<Input, Expected, Metadata> = {
     _xact_id?: TransactionId;
     created?: string | null;
     upsert_id?: string;
+    trialCount?: number;
 } & (Expected extends void ? object : {
     expected: Expected;
 }) & (Metadata extends void ? object : {
@@ -13226,17 +13294,23 @@ type EvalScorerArgs<Input, Output, Expected, Metadata extends BaseMetadata = Def
 };
 type OneOrMoreScores = Score | number | null | Array<Score>;
 type EvalScorer<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = (args: EvalScorerArgs<Input, Output, Expected, Metadata>) => OneOrMoreScores | Promise<OneOrMoreScores>;
+type OneOrMoreClassifications = Classification | Classification[] | null;
+type EvalClassifier<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = (args: EvalScorerArgs<Input, Output, Expected, Metadata>) => OneOrMoreClassifications | Promise<OneOrMoreClassifications>;
 type EvalResult<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata> & {
     output: Output;
-    scores: Record<string, number | null>;
     error: unknown;
     origin?: ObjectReferenceType;
+    scores: Record<string, number | null>;
+    classifications?: Record<string, ClassificationItem[]>;
 };
 type ErrorScoreHandler = (args: {
     rootSpan: Span;
     data: EvalCase<any, any, any>;
     unhandledScores: string[];
 }) => Record<string, number> | undefined | void;
+/**
+ * Defines an evaluator. At least one of `scores` or `classifiers` must be provided.
+ */
 interface Evaluator<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType, Parameters extends EvalParameters = EvalParameters> {
     /**
      * A function that returns a list of inputs, expected outputs, and metadata.
@@ -13247,9 +13321,16 @@ interface Evaluator<Input, Output, Expected, Metadata extends BaseMetadata = Def
      */
     task: EvalTask<Input, Output, Expected, Metadata, Parameters>;
     /**
-     * A set of functions that take an input, output, and expected value and return a score.
+     * A set of functions that take an input, output, and expected value and return a {@link Score}.
+     * At least one of `scores` or `classifiers` must be provided.
+     */
+    scores?: EvalScorer<Input, Output, Expected, Metadata>[];
+    /**
+     * A set of functions that take an input, output, and expected value and return a
+     * {@link Classification}. Results are recorded under the `classifications` column.
+     * At least one of `scores` or `classifiers` must be provided.
      */
-    scores: EvalScorer<Input, Output, Expected, Metadata>[];
+    classifiers?: EvalClassifier<Input, Output, Expected, Metadata>[];
     /**
      * A set of parameters that will be passed to the evaluator.
      * Can be:

package/dev/dist/index.d.ts

@@ -131,6 +131,10 @@ type OtherExperimentLogFields = {
     error: unknown;
     tags: string[];
     scores: Record<string, number | null>;
+    classifications?: Record<string, {
+        id: string;
+        label?: string;
+    }[]>;
     metadata: Record<string, unknown>;
     metrics: Record<string, unknown>;
     datasetRecordId: string;
@@ -287,6 +291,42 @@ declare class SpanComponentsV3 {
     private static fromJsonObj;
 }
 
+/**
+ * The result returned by a classifier function. Unlike `Score`, `id` is
+ * required and the span will be recorded as a classifier span.
+ */
+interface Classification {
+    /**
+     * The name of this classification result. Used as the key in the
+     * `classifications` log record. If omitted, defaults to the classifier
+     * function's name.
+     */
+    name: string;
+    /**
+     * A machine-readable identifier for the classification outcome
+     * (e.g. `"positive"`, `"negative"`, `"neutral"`). This value is stored
+     * in the log and used for programmatic analysis.
+     */
+    id: string;
+    /**
+     * An optional human-readable display label for this outcome. If omitted,
+     * defaults to `id`. Use this when you want a friendlier label in the UI
+     * while keeping a stable `id` for programmatic use.
+     */
+    label?: string;
+    /**
+     * Optional arbitrary metadata to attach to this classification result.
+     */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * The serialized form of a classification stored in the `classifications` log record.
+ */
+interface ClassificationItem {
+    id: string;
+    label: string;
+    metadata?: Record<string, unknown>;
+}
 interface Score {
     name: string;
     score: number | null;
@@ -2114,6 +2154,15 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             }, {
                 type: "scorer";
                 index: number;
+            }>, z.ZodObject<{
+                type: z.ZodLiteral<"classifier">;
+                index: z.ZodNumber;
+            }, "strip", z.ZodTypeAny, {
+                type: "classifier";
+                index: number;
+            }, {
+                type: "classifier";
+                index: number;
             }>]>;
         }, "strip", z.ZodTypeAny, {
             type: "experiment";
@@ -2123,6 +2172,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         }, {
             type: "experiment";
@@ -2132,6 +2184,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         }>, z.ZodObject<{
             type: z.ZodLiteral<"function">;
@@ -2204,6 +2259,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         } | {
             type: "function";
@@ -2236,6 +2294,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         } | {
             type: "function";
@@ -2303,6 +2364,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         } | {
             type: "function";
@@ -2348,6 +2412,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         } | {
             type: "function";
@@ -11306,6 +11373,7 @@ type EvalCase<Input, Expected, Metadata> = {
     _xact_id?: TransactionId;
     created?: string | null;
     upsert_id?: string;
+    trialCount?: number;
 } & (Expected extends void ? object : {
     expected: Expected;
 }) & (Metadata extends void ? object : {
@@ -13226,17 +13294,23 @@ type EvalScorerArgs<Input, Output, Expected, Metadata extends BaseMetadata = Def
 };
 type OneOrMoreScores = Score | number | null | Array<Score>;
 type EvalScorer<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = (args: EvalScorerArgs<Input, Output, Expected, Metadata>) => OneOrMoreScores | Promise<OneOrMoreScores>;
+type OneOrMoreClassifications = Classification | Classification[] | null;
+type EvalClassifier<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = (args: EvalScorerArgs<Input, Output, Expected, Metadata>) => OneOrMoreClassifications | Promise<OneOrMoreClassifications>;
 type EvalResult<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata> & {
     output: Output;
-    scores: Record<string, number | null>;
     error: unknown;
     origin?: ObjectReferenceType;
+    scores: Record<string, number | null>;
+    classifications?: Record<string, ClassificationItem[]>;
 };
 type ErrorScoreHandler = (args: {
     rootSpan: Span;
     data: EvalCase<any, any, any>;
     unhandledScores: string[];
 }) => Record<string, number> | undefined | void;
+/**
+ * Defines an evaluator. At least one of `scores` or `classifiers` must be provided.
+ */
 interface Evaluator<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType, Parameters extends EvalParameters = EvalParameters> {
     /**
      * A function that returns a list of inputs, expected outputs, and metadata.
@@ -13247,9 +13321,16 @@ interface Evaluator<Input, Output, Expected, Metadata extends BaseMetadata = Def
      */
     task: EvalTask<Input, Output, Expected, Metadata, Parameters>;
     /**
-     * A set of functions that take an input, output, and expected value and return a score.
+     * A set of functions that take an input, output, and expected value and return a {@link Score}.
+     * At least one of `scores` or `classifiers` must be provided.
+     */
+    scores?: EvalScorer<Input, Output, Expected, Metadata>[];
+    /**
+     * A set of functions that take an input, output, and expected value and return a
+     * {@link Classification}. Results are recorded under the `classifications` column.
+     * At least one of `scores` or `classifiers` must be provided.
      */
-    scores: EvalScorer<Input, Output, Expected, Metadata>[];
+    classifiers?: EvalClassifier<Input, Output, Expected, Metadata>[];
     /**
      * A set of parameters that will be passed to the evaluator.
      * Can be: