braintrust 3.8.0 → 3.10.0

package/dev/dist/index.d.mts

@@ -131,6 +131,10 @@ type OtherExperimentLogFields = {
     error: unknown;
     tags: string[];
     scores: Record<string, number | null>;
+    classifications?: Record<string, {
+        id: string;
+        label?: string;
+    }[]>;
     metadata: Record<string, unknown>;
     metrics: Record<string, unknown>;
     datasetRecordId: string;
@@ -287,6 +291,42 @@ declare class SpanComponentsV3 {
     private static fromJsonObj;
 }
 
+/**
+ * The result returned by a classifier function. Unlike `Score`, `id` is
+ * required and the span will be recorded as a classifier span.
+ */
+interface Classification {
+    /**
+     * The name of this classification result. Used as the key in the
+     * `classifications` log record. If omitted, defaults to the classifier
+     * function's name.
+     */
+    name: string;
+    /**
+     * A machine-readable identifier for the classification outcome
+     * (e.g. `"positive"`, `"negative"`, `"neutral"`). This value is stored
+     * in the log and used for programmatic analysis.
+     */
+    id: string;
+    /**
+     * An optional human-readable display label for this outcome. If omitted,
+     * defaults to `id`. Use this when you want a friendlier label in the UI
+     * while keeping a stable `id` for programmatic use.
+     */
+    label?: string;
+    /**
+     * Optional arbitrary metadata to attach to this classification result.
+     */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * The serialized form of a classification stored in the `classifications` log record.
+ */
+interface ClassificationItem {
+    id: string;
+    label: string;
+    metadata?: Record<string, unknown>;
+}
 interface Score {
     name: string;
     score: number | null;
@@ -1434,6 +1474,29 @@ declare const SavedFunctionId: z.ZodUnion<[z.ZodObject<{
     function_type?: "llm" | "scorer" | "task" | "tool" | "custom_view" | "preprocessor" | "facet" | "classifier" | "tag" | "parameters" | "sandbox" | undefined;
 }>]>;
 type SavedFunctionIdType = z.infer<typeof SavedFunctionId>;
+declare const DatasetSnapshot: z.ZodObject<{
+    id: z.ZodString;
+    dataset_id: z.ZodString;
+    name: z.ZodString;
+    description: z.ZodUnion<[z.ZodString, z.ZodNull]>;
+    xact_id: z.ZodString;
+    created: z.ZodUnion<[z.ZodString, z.ZodNull]>;
+}, "strip", z.ZodTypeAny, {
+    id: string;
+    created: string | null;
+    name: string;
+    description: string | null;
+    dataset_id: string;
+    xact_id: string;
+}, {
+    id: string;
+    created: string | null;
+    name: string;
+    description: string | null;
+    dataset_id: string;
+    xact_id: string;
+}>;
+type DatasetSnapshotType = z.infer<typeof DatasetSnapshot>;
 declare const RepoInfo: z.ZodUnion<[z.ZodObject<{
     commit: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodNull]>>;
     branch: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodNull]>>;
@@ -2114,6 +2177,15 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             }, {
                 type: "scorer";
                 index: number;
+            }>, z.ZodObject<{
+                type: z.ZodLiteral<"classifier">;
+                index: z.ZodNumber;
+            }, "strip", z.ZodTypeAny, {
+                type: "classifier";
+                index: number;
+            }, {
+                type: "classifier";
+                index: number;
             }>]>;
         }, "strip", z.ZodTypeAny, {
             type: "experiment";
@@ -2123,6 +2195,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         }, {
             type: "experiment";
@@ -2132,6 +2207,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         }>, z.ZodObject<{
             type: z.ZodLiteral<"function">;
@@ -2204,6 +2282,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         } | {
             type: "function";
@@ -2236,6 +2317,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         } | {
             type: "function";
@@ -2303,6 +2387,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         } | {
             type: "function";
@@ -2348,6 +2435,9 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
             } | {
                 type: "scorer";
                 index: number;
+            } | {
+                type: "classifier";
+                index: number;
             };
         } | {
             type: "function";
@@ -3968,6 +4058,7 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
                 hierarchy_threshold?: number | undefined;
                 naming_model?: string | undefined;
             }>>;
+            disable_reconciliation: z.ZodOptional<z.ZodBoolean>;
             distance_threshold: z.ZodOptional<z.ZodNumber>;
         }, "strip", z.ZodTypeAny, {
             type: "topic_map";
@@ -3986,6 +4077,7 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
                 hierarchy_threshold?: number | undefined;
                 naming_model?: string | undefined;
             } | undefined;
+            disable_reconciliation?: boolean | undefined;
             distance_threshold?: number | undefined;
         }, {
             type: "topic_map";
@@ -4004,6 +4096,7 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
                 hierarchy_threshold?: number | undefined;
                 naming_model?: string | undefined;
             } | undefined;
+            disable_reconciliation?: boolean | undefined;
             distance_threshold?: number | undefined;
         }>;
     }, "strip", z.ZodTypeAny, {
@@ -4025,6 +4118,7 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
                 hierarchy_threshold?: number | undefined;
                 naming_model?: string | undefined;
             } | undefined;
+            disable_reconciliation?: boolean | undefined;
             distance_threshold?: number | undefined;
         };
         topic_map_id?: string | undefined;
@@ -4047,6 +4141,7 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
                 hierarchy_threshold?: number | undefined;
                 naming_model?: string | undefined;
             } | undefined;
+            disable_reconciliation?: boolean | undefined;
             distance_threshold?: number | undefined;
         };
         topic_map_id?: string | undefined;
@@ -4088,6 +4183,7 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
                 hierarchy_threshold?: number | undefined;
                 naming_model?: string | undefined;
             } | undefined;
+            disable_reconciliation?: boolean | undefined;
             distance_threshold?: number | undefined;
         };
         topic_map_id?: string | undefined;
@@ -4129,6 +4225,7 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
                 hierarchy_threshold?: number | undefined;
                 naming_model?: string | undefined;
             } | undefined;
+            disable_reconciliation?: boolean | undefined;
             distance_threshold?: number | undefined;
         };
         topic_map_id?: string | undefined;
@@ -4209,6 +4306,7 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
         hierarchy_threshold?: number | undefined;
         naming_model?: string | undefined;
     }>>;
+    disable_reconciliation: z.ZodOptional<z.ZodBoolean>;
     distance_threshold: z.ZodOptional<z.ZodNumber>;
 }, "strip", z.ZodTypeAny, {
     type: "topic_map";
@@ -4227,6 +4325,7 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
         hierarchy_threshold?: number | undefined;
         naming_model?: string | undefined;
     } | undefined;
+    disable_reconciliation?: boolean | undefined;
     distance_threshold?: number | undefined;
 }, {
     type: "topic_map";
@@ -4245,6 +4344,7 @@ declare const FunctionData: z.ZodUnion<[z.ZodObject<{
         hierarchy_threshold?: number | undefined;
         naming_model?: string | undefined;
     } | undefined;
+    disable_reconciliation?: boolean | undefined;
     distance_threshold?: number | undefined;
 }>, z.ZodUnknown>]>;
 declare const PromptData: z.ZodObject<{
@@ -10617,6 +10717,31 @@ type InferParameters<T extends EvalParameters> = {
 };
 
 declare const RESET_CONTEXT_MANAGER_STATE: unique symbol;
+declare const datasetRestorePreviewResultSchema: z.ZodObject<{
+    rows_to_restore: z.ZodNumber;
+    rows_to_delete: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    rows_to_restore: number;
+    rows_to_delete: number;
+}, {
+    rows_to_restore: number;
+    rows_to_delete: number;
+}>;
+type DatasetRestorePreviewResult = z.infer<typeof datasetRestorePreviewResultSchema>;
+declare const datasetRestoreResultSchema: z.ZodObject<{
+    xact_id: z.ZodNullable<z.ZodString>;
+    rows_restored: z.ZodNumber;
+    rows_deleted: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    xact_id: string | null;
+    rows_restored: number;
+    rows_deleted: number;
+}, {
+    xact_id: string | null;
+    rows_restored: number;
+    rows_deleted: number;
+}>;
+type DatasetRestoreResult = z.infer<typeof datasetRestoreResultSchema>;
 declare const parametersRowSchema: z.ZodObject<{
     id: z.ZodString;
     _xact_id: z.ZodString;
@@ -11190,6 +11315,20 @@ declare class HTTPBackgroundLogger implements BackgroundLogger {
     disable(): void;
     enforceQueueSizeLimit(enforce: boolean): void;
 }
+type DatasetSnapshotNameLookup = {
+    snapshotName: string;
+    xactId?: never;
+};
+type DatasetSnapshotXactLookup = {
+    snapshotName?: never;
+    xactId: string;
+};
+type DatasetSnapshotLookup = DatasetSnapshotNameLookup | DatasetSnapshotXactLookup;
+type DatasetPinState = {
+    lazyPinnedVersion?: LazyValue<string | undefined>;
+    pinnedEnvironment?: string;
+    pinnedSnapshotName?: string;
+};
 type AsyncFlushArg<IsAsyncFlush> = {
     asyncFlush?: IsAsyncFlush;
 };
@@ -11277,6 +11416,9 @@ declare class ObjectFetcher<RecordType> implements AsyncIterable<WithTransaction
     constructor(objectType: "dataset" | "experiment" | "project_logs" | "playground_logs", pinnedVersion: string | undefined, mutateRecord?: ((r: any) => WithTransactionId<RecordType>) | undefined, _internal_btql?: Record<string, unknown> | undefined);
     get id(): Promise<string>;
     protected getState(): Promise<BraintrustState>;
+    protected getPinnedVersion(): string | undefined;
+    protected setPinnedVersion(pinnedVersion: string | undefined): void;
+    protected getInternalBtql(): Record<string, unknown> | undefined;
     private fetchRecordsFromApi;
     /**
      * Fetch all records from the object.
@@ -11306,6 +11448,7 @@ type EvalCase<Input, Expected, Metadata> = {
     _xact_id?: TransactionId;
     created?: string | null;
     upsert_id?: string;
+    trialCount?: number;
 } & (Expected extends void ? object : {
     expected: Expected;
 }) & (Metadata extends void ? object : {
@@ -11441,12 +11584,25 @@ declare class Dataset<IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY
     private readonly lazyMetadata;
     private readonly __braintrust_dataset_marker;
     private newRecords;
-    constructor(state: BraintrustState, lazyMetadata: LazyValue<ProjectDatasetMetadata>, pinnedVersion?: string, legacy?: IsLegacyDataset, _internal_btql?: Record<string, unknown>);
+    private lazyPinnedVersion?;
+    private pinnedEnvironment?;
+    private pinnedSnapshotName?;
+    constructor(state: BraintrustState, lazyMetadata: LazyValue<ProjectDatasetMetadata>, pinnedVersion?: string, legacy?: IsLegacyDataset, _internal_btql?: Record<string, unknown>, pinState?: DatasetPinState);
     get id(): Promise<string>;
     get name(): Promise<string>;
     get project(): Promise<ObjectMetadata>;
     get loggingState(): BraintrustState;
+    toEvalData(): Promise<{
+        dataset_id: string;
+        dataset_version?: string;
+        dataset_environment?: string;
+        dataset_snapshot_name?: string;
+        _internal_btql?: Record<string, unknown>;
+    }>;
     protected getState(): Promise<BraintrustState>;
+    version(options?: {
+        batchSize?: number;
+    }): Promise<string | undefined>;
     private validateEvent;
     private createArgs;
     /**
@@ -11494,6 +11650,24 @@ declare class Dataset<IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY
         readonly metadata?: Record<string, unknown>;
     }): string;
     delete(id: string): string;
+    createSnapshot({ name, description, update, }: {
+        readonly name: string;
+        readonly description?: string;
+        readonly update?: boolean;
+    }): Promise<DatasetSnapshotType>;
+    listSnapshots(): Promise<DatasetSnapshotType[]>;
+    getSnapshot(lookup: DatasetSnapshotLookup): Promise<DatasetSnapshotType | undefined>;
+    updateSnapshot(snapshotId: string, { name, description, }: {
+        readonly name?: string;
+        readonly description?: string | null;
+    }): Promise<DatasetSnapshotType>;
+    deleteSnapshot(snapshotId: string): Promise<DatasetSnapshotType>;
+    restorePreview({ version, }: {
+        readonly version: string;
+    }): Promise<DatasetRestorePreviewResult>;
+    restore({ version, }: {
+        readonly version: string;
+    }): Promise<DatasetRestoreResult>;
     /**
      * Summarize the dataset, including high level metrics about its size and other metadata.
      * @param summarizeData Whether to summarize the data. If false, only the metadata will be returned.
@@ -13226,17 +13400,23 @@ type EvalScorerArgs<Input, Output, Expected, Metadata extends BaseMetadata = Def
 };
 type OneOrMoreScores = Score | number | null | Array<Score>;
 type EvalScorer<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = (args: EvalScorerArgs<Input, Output, Expected, Metadata>) => OneOrMoreScores | Promise<OneOrMoreScores>;
+type OneOrMoreClassifications = Classification | Classification[] | null;
+type EvalClassifier<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = (args: EvalScorerArgs<Input, Output, Expected, Metadata>) => OneOrMoreClassifications | Promise<OneOrMoreClassifications>;
 type EvalResult<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata> & {
     output: Output;
-    scores: Record<string, number | null>;
     error: unknown;
     origin?: ObjectReferenceType;
+    scores: Record<string, number | null>;
+    classifications?: Record<string, ClassificationItem[]>;
 };
 type ErrorScoreHandler = (args: {
     rootSpan: Span;
     data: EvalCase<any, any, any>;
     unhandledScores: string[];
 }) => Record<string, number> | undefined | void;
+/**
+ * Defines an evaluator. At least one of `scores` or `classifiers` must be provided.
+ */
 interface Evaluator<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType, Parameters extends EvalParameters = EvalParameters> {
     /**
      * A function that returns a list of inputs, expected outputs, and metadata.
@@ -13247,9 +13427,16 @@ interface Evaluator<Input, Output, Expected, Metadata extends BaseMetadata = Def
      */
     task: EvalTask<Input, Output, Expected, Metadata, Parameters>;
     /**
-     * A set of functions that take an input, output, and expected value and return a score.
+     * A set of functions that take an input, output, and expected value and return a {@link Score}.
+     * At least one of `scores` or `classifiers` must be provided.
+     */
+    scores?: EvalScorer<Input, Output, Expected, Metadata>[];
+    /**
+     * A set of functions that take an input, output, and expected value and return a
+     * {@link Classification}. Results are recorded under the `classifications` column.
+     * At least one of `scores` or `classifiers` must be provided.
      */
-    scores: EvalScorer<Input, Output, Expected, Metadata>[];
+    classifiers?: EvalClassifier<Input, Output, Expected, Metadata>[];
     /**
      * A set of parameters that will be passed to the evaluator.
      * Can be: