braintrust 0.0.97 → 0.0.99

package/dist/isomorph.d.ts

@@ -1,4 +1,4 @@
-import { GitMetadataSettings, RepoStatus } from "@braintrust/core";
+import { GitMetadataSettings, RepoInfo } from "@braintrust/core";
 export interface CallerLocation {
     caller_functionname: string;
     caller_filename: string;
@@ -10,7 +10,7 @@ export interface IsoAsyncLocalStorage<T> {
     getStore(): T | undefined;
 }
 export interface Common {
-    getRepoStatus: (settings?: GitMetadataSettings) => Promise<RepoStatus | undefined>;
+    getRepoInfo: (settings?: GitMetadataSettings) => Promise<RepoInfo | undefined>;
     getPastNAncestors: () => Promise<string[]>;
     getEnv: (name: string) => string | undefined;
     getCallerLocation: () => CallerLocation | undefined;

package/dist/logger.d.ts

@@ -1,8 +1,7 @@
 /// <reference lib="dom" />
-import { TRANSACTION_ID_FIELD, IS_MERGE_FIELD, PARENT_ID_FIELD, Source, AUDIT_SOURCE_FIELD, AUDIT_METADATA_FIELD, GitMetadataSettings, TransactionId } from "@braintrust/core";
+import { TRANSACTION_ID_FIELD, GitMetadataSettings, RepoInfo, TransactionId, ParentExperimentIds, ParentProjectLogIds, IdField, ExperimentLogPartialArgs, ExperimentLogFullArgs, LogFeedbackFullArgs, ExperimentEvent, BackgroundLogEvent, DEFAULT_IS_LEGACY_DATASET, DatasetRecord } from "@braintrust/core";
 import { IsoAsyncLocalStorage } from "./isomorph";
 import { LazyValue } from "./util";
-export type Metadata = Record<string, unknown>;
 export type SetCurrentArg = {
     setCurrent?: boolean;
 };
@@ -100,7 +99,7 @@ export declare class NoopSpan implements Span {
     constructor();
     log(_: ExperimentLogPartialArgs): void;
     logFeedback(event: Omit<LogFeedbackFullArgs, "id">): void;
-    traced<R>(callback: (span: Span) => R, _1: StartSpanArgs & SetCurrentArg): R;
+    traced<R>(callback: (span: Span) => R, _1?: StartSpanArgs & SetCurrentArg): R;
     startSpan(_1?: StartSpanArgs): this;
     end(args?: EndSpanArgs): number;
     close(args?: EndSpanArgs): number;
@@ -187,9 +186,13 @@ export declare class Logger<IsAsyncFlush extends boolean> {
      * @param event.metadata: (Optional) a dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings.
      * @param event.metrics: (Optional) a dictionary of metrics to log. The following keys are populated automatically: "start", "end".
      * @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
+     * @param options Additional logging options
+     * @param options.allowLogConcurrentWithActiveSpan in rare cases where you need to log at the top level separately from an active span on the logger, set this to true.
      * :returns: The `id` of the logged event.
      */
-    log(event: Readonly<StartSpanEventArgs>): PromiseUnless<IsAsyncFlush, string>;
+    log(event: Readonly<StartSpanEventArgs>, options?: {
+        allowLogConcurrentWithActiveSpan?: boolean;
+    }): PromiseUnless<IsAsyncFlush, string>;
     /**
      * Create a new toplevel span underneath the logger. The name defaults to "root".
      *
@@ -220,87 +223,6 @@ export declare class Logger<IsAsyncFlush extends boolean> {
     flush(): Promise<void>;
     get asyncFlush(): IsAsyncFlush | undefined;
 }
-export type IdField = {
-    id: string;
-};
-export type InputField = {
-    input: unknown;
-};
-export type InputsField = {
-    inputs: unknown;
-};
-export type OtherExperimentLogFields = {
-    output: unknown;
-    expected: unknown;
-    scores: Record<string, number | null>;
-    metadata: Record<string, unknown>;
-    metrics: Record<string, unknown>;
-    datasetRecordId: string;
-};
-export type ExperimentLogPartialArgs = Partial<OtherExperimentLogFields> & Partial<InputField | InputsField>;
-export type ExperimentLogFullArgs = Partial<Omit<OtherExperimentLogFields, "output" | "scores">> & Required<Pick<OtherExperimentLogFields, "output" | "scores">> & Partial<InputField | InputsField> & Partial<IdField>;
-export type LogFeedbackFullArgs = IdField & Partial<Omit<OtherExperimentLogFields, "output" | "metrics" | "datasetRecordId"> & {
-    comment: string;
-    source: Source;
-}>;
-export type LogCommentFullArgs = IdField & {
-    created: string;
-    origin: {
-        id: string;
-    };
-    comment: {
-        text: string;
-    };
-    [AUDIT_SOURCE_FIELD]: Source;
-    [AUDIT_METADATA_FIELD]?: Record<string, unknown>;
-} & Omit<ParentExperimentIds | ParentProjectLogIds, "kind">;
-type ExperimentEvent = Partial<InputField> & Partial<OtherExperimentLogFields> & {
-    id: string;
-    span_id?: string;
-    root_span_id?: string;
-    project_id: string;
-    experiment_id: string;
-    [IS_MERGE_FIELD]: boolean;
-} & Partial<{
-    created: string;
-    span_parents: string[];
-    span_attributes: Record<string, unknown>;
-    context: Record<string, unknown>;
-    [PARENT_ID_FIELD]: string;
-    [AUDIT_SOURCE_FIELD]: Source;
-    [AUDIT_METADATA_FIELD]?: Record<string, unknown>;
-}>;
-interface DatasetEvent {
-    inputs?: unknown;
-    output?: unknown;
-    metadata?: unknown;
-    id: string;
-    project_id: string;
-    dataset_id: string;
-    created: string;
-}
-type LoggingEvent = Omit<ExperimentEvent, "experiment_id"> & {
-    org_id: string;
-    log_id: "g";
-};
-export type CommentEvent = IdField & {
-    created: string;
-    origin: {
-        id: string;
-    };
-    comment: {
-        text: string;
-    };
-    [AUDIT_SOURCE_FIELD]: Source;
-    [AUDIT_METADATA_FIELD]?: Record<string, unknown>;
-} & Omit<ParentExperimentIds | ParentProjectLogIds, "kind">;
-type BackgroundLogEvent = ExperimentEvent | DatasetEvent | LoggingEvent | CommentEvent;
-export interface DatasetRecord {
-    id: string;
-    input: any;
-    output: any;
-    metadata: any;
-}
 declare class BackgroundLogger {
     private logConn;
     private items;
@@ -317,45 +239,61 @@ type InitOpenOption<IsOpen extends boolean> = {
 export type InitOptions<IsOpen extends boolean> = {
     experiment?: string;
     description?: string;
-    dataset?: Dataset;
+    dataset?: AnyDataset;
     update?: boolean;
     baseExperiment?: string;
     isPublic?: boolean;
     appUrl?: string;
     apiKey?: string;
     orgName?: string;
-    metadata?: Metadata;
+    metadata?: Record<string, unknown>;
     gitMetadataSettings?: GitMetadataSettings;
+    projectId?: string;
+    baseExperimentId?: string;
+    repoInfo?: RepoInfo;
     setCurrent?: boolean;
 } & InitOpenOption<IsOpen>;
+export type FullInitOptions<IsOpen extends boolean> = {
+    project?: string;
+} & InitOptions<IsOpen>;
 type InitializedExperiment<IsOpen extends boolean | undefined> = IsOpen extends true ? ReadonlyExperiment : Experiment;
 /**
  * Log in, and then initialize a new experiment in a specified project. If the project does not exist, it will be created.
  *
- * @param project The name of the project to create the experiment in.
- * @param options Additional options for configuring init().
+ * @param options Options for configuring init().
+ * @param options.project The name of the project to create the experiment in. Must specify at least one of `project` or `projectId`.
  * @param options.experiment The name of the experiment to create. If not specified, a name will be generated automatically.
  * @param options.description An optional description of the experiment.
- * @param options.dataset (Optional) A dataset to associate with the experiment. You can pass in the name of the dataset (in the same project) or a
- * dataset object (from any project).
+ * @param options.dataset (Optional) A dataset to associate with the experiment. You can pass in the name of the dataset (in the same project) or a dataset object (from any project).
  * @param options.update If the experiment already exists, continue logging to it.
- * @param options.baseExperiment An optional experiment name to use as a base. If specified, the new experiment will be summarized and compared to this
- * experiment. Otherwise, it will pick an experiment by finding the closest ancestor on the default (e.g. main) branch.
+ * @param options.baseExperiment An optional experiment name to use as a base. If specified, the new experiment will be summarized and compared to this experiment. Otherwise, it will pick an experiment by finding the closest ancestor on the default (e.g. main) branch.
  * @param options.isPublic An optional parameter to control whether the experiment is publicly visible to anybody with the link or privately visible to only members of the organization. Defaults to private.
  * @param options.appUrl The URL of the Braintrust App. Defaults to https://www.braintrustdata.com.
- * @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API
- * key is specified, will prompt the user to login.
+ * @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API key is specified, will prompt the user to login.
  * @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
- * @param options.metadata (Optional) A dictionary with additional data about the test example, model outputs, or just
- * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
- * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
- * JSON-serializable type, but its keys must be strings.
+ * @param options.metadata (Optional) A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings.
  * @param options.gitMetadataSettings (Optional) Settings for collecting git metadata. By default, will collect all git metadata fields allowed in org-level settings.
  * @param setCurrent If true (the default), set the global current-experiment to the newly-created one.
  * @param options.open If the experiment already exists, open it in read-only mode.
+ * @param options.projectId The id of the project to create the experiment in. This takes precedence over `project` if specified.
+ * @param options.baseExperimentId An optional experiment id to use as a base. If specified, the new experiment will be summarized and compared to this. This takes precedence over `baseExperiment` if specified.
+ * @param options.repoInfo (Optional) Explicitly specify the git metadata for this experiment. This takes precedence over `gitMetadataSettings` if specified.
  * @returns The newly created Experiment.
  */
+export declare function init<IsOpen extends boolean = false>(options: Readonly<FullInitOptions<IsOpen>>): InitializedExperiment<IsOpen>;
+/**
+ * Legacy form of `init` which accepts the project name as the first parameter,
+ * separately from the remaining options. See `init(options)` for full details.
+ */
 export declare function init<IsOpen extends boolean = false>(project: string, options?: Readonly<InitOptions<IsOpen>>): InitializedExperiment<IsOpen>;
+/**
+ * Alias for init(options).
+ */
+export declare function initExperiment<IsOpen extends boolean = false>(options: Readonly<InitOptions<IsOpen>>): InitializedExperiment<IsOpen>;
+/**
+ * Alias for init(project, options).
+ */
+export declare function initExperiment<IsOpen extends boolean = false>(project: string, options?: Readonly<InitOptions<IsOpen>>): InitializedExperiment<IsOpen>;
 /**
  * This function is deprecated. Use `init` instead.
  */
@@ -364,32 +302,46 @@ export declare function withExperiment<R>(project: string, callback: (experiment
  * This function is deprecated. Use `initLogger` instead.
  */
 export declare function withLogger<IsAsyncFlush extends boolean = false, R = void>(callback: (logger: Logger<IsAsyncFlush>) => R, options?: Readonly<InitLoggerOptions<IsAsyncFlush> & SetCurrentArg>): R;
-type InitDatasetOptions = {
+type UseOutputOption<IsLegacyDataset extends boolean> = {
+    useOutput?: IsLegacyDataset;
+};
+type InitDatasetOptions<IsLegacyDataset extends boolean> = {
     dataset?: string;
     description?: string;
     version?: string;
     appUrl?: string;
     apiKey?: string;
     orgName?: string;
-};
+    projectId?: string;
+} & UseOutputOption<IsLegacyDataset>;
+type FullInitDatasetOptions<IsLegacyDataset extends boolean> = {
+    project?: string;
+} & InitDatasetOptions<IsLegacyDataset>;
 /**
  * Create a new dataset in a specified project. If the project does not exist, it will be created.
  *
- * @param project The name of the project to create the dataset in.
- * @param options Additional options for configuring init().
+ * @param options Options for configuring initDataset().
+ * @param options.project The name of the project to create the dataset in. Must specify at least one of `project` or `projectId`.
  * @param options.dataset The name of the dataset to create. If not specified, a name will be generated automatically.
  * @param options.description An optional description of the dataset.
  * @param options.appUrl The URL of the Braintrust App. Defaults to https://www.braintrustdata.com.
- * @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API
- * key is specified, will prompt the user to login.
+ * @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API key is specified, will prompt the user to login.
  * @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
+ * @param options.projectId The id of the project to create the dataset in. This takes precedence over `project` if specified.
+ * @param options.useOutput If true (the default), records will be fetched from this dataset in the legacy format, with the "expected" field renamed to "output". This will default to false in a future version of Braintrust.
  * @returns The newly created Dataset.
  */
-export declare function initDataset(project: string, options?: Readonly<InitDatasetOptions>): Dataset;
+export declare function initDataset<IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY_DATASET>(options: Readonly<FullInitDatasetOptions<IsLegacyDataset>>): Dataset<IsLegacyDataset>;
+/**
+ * Legacy form of `initDataset` which accepts the project name as the first
+ * parameter, separately from the remaining options. See
+ * `initDataset(options)` for full details.
+ */
+export declare function initDataset<IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY_DATASET>(project: string, options?: Readonly<InitDatasetOptions<IsLegacyDataset>>): Dataset<IsLegacyDataset>;
 /**
  * This function is deprecated. Use `initDataset` instead.
  */
-export declare function withDataset<R>(project: string, callback: (dataset: Dataset) => R, options?: Readonly<InitDatasetOptions>): R;
+export declare function withDataset<R, IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY_DATASET>(project: string, callback: (dataset: Dataset<IsLegacyDataset>) => R, options?: Readonly<InitDatasetOptions<IsLegacyDataset>>): R;
 type AsyncFlushArg<IsAsyncFlush> = {
     asyncFlush?: IsAsyncFlush;
 };
@@ -494,11 +446,12 @@ export declare function startSpan<IsAsyncFlush extends boolean = false>(args?: S
 export type WithTransactionId<R> = R & {
     [TRANSACTION_ID_FIELD]: TransactionId;
 };
-declare class ObjectFetcher<RecordType> {
+declare class ObjectFetcher<RecordType> implements AsyncIterable<WithTransactionId<RecordType>> {
     private objectType;
     private pinnedVersion;
+    private mutateRecord?;
     private _fetchedData;
-    constructor(objectType: "dataset" | "experiment", pinnedVersion: string | undefined);
+    constructor(objectType: "dataset" | "experiment", pinnedVersion: string | undefined, mutateRecord?: ((r: any) => RecordType) | undefined);
     get id(): Promise<string>;
     protected getState(): Promise<BraintrustState>;
     fetch(): AsyncGenerator<WithTransactionId<RecordType>>;
@@ -507,11 +460,15 @@ declare class ObjectFetcher<RecordType> {
     clearCache(): void;
     version(): Promise<string | bigint | undefined>;
 }
-export interface EvalCase<Input, Expected> {
+export type BaseMetadata = Record<string, unknown> | void;
+export type DefaultMetadataType = void;
+export type EvalCase<Input, Expected, Metadata> = {
     input: Input;
-    expected?: Expected;
-    metadata?: Metadata;
-}
+} & (Expected extends void ? {} : {
+    expected: Expected;
+}) & (Metadata extends void ? {} : {
+    metadata: Metadata;
+});
 /**
  * An experiment is a collection of logged events, such as model inputs and outputs, which represent
  * a snapshot of your application at a particular point in time. An experiment is meant to capture more
@@ -526,11 +483,11 @@ export interface EvalCase<Input, Expected> {
  */
 export declare class Experiment extends ObjectFetcher<ExperimentEvent> {
     private readonly lazyMetadata;
-    readonly dataset?: Dataset;
+    readonly dataset?: AnyDataset;
     private bgLogger;
     private lastStartTime;
     kind: "experiment";
-    constructor(lazyMetadata: LazyValue<ProjectExperimentMetadata>, dataset?: Dataset);
+    constructor(lazyMetadata: LazyValue<ProjectExperimentMetadata>, dataset?: AnyDataset);
     get id(): Promise<string>;
     get name(): Promise<string>;
     get project(): Promise<ObjectMetadata>;
@@ -548,9 +505,13 @@ export declare class Experiment extends ObjectFetcher<ExperimentEvent> {
      * @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
      * @param event.dataset_record_id: (Optional) the id of the dataset record that this event is associated with. This field is required if and only if the experiment is associated with a dataset.
      * @param event.inputs: (Deprecated) the same as `input` (will be removed in a future version).
+     * @param options Additional logging options
+     * @param options.allowLogConcurrentWithActiveSpan in rare cases where you need to log at the top level separately from an active span on the experiment, set this to true.
      * :returns: The `id` of the logged event.
      */
-    log(event: Readonly<ExperimentLogFullArgs>): string;
+    log(event: Readonly<ExperimentLogFullArgs>, options?: {
+        allowLogConcurrentWithActiveSpan?: boolean;
+    }): string;
     /**
      * Create a new toplevel span underneath the experiment. The name defaults to "root".
      *
@@ -612,18 +573,7 @@ export declare class ReadonlyExperiment extends ObjectFetcher<ExperimentEvent> {
     get id(): Promise<string>;
     get name(): Promise<string>;
     protected getState(): Promise<BraintrustState>;
-    asDataset<Input = unknown, Expected = unknown>(): AsyncGenerator<EvalCase<Input, Expected>>;
-}
-interface ParentExperimentIds {
-    kind: "experiment";
-    project_id: string;
-    experiment_id: string;
-}
-interface ParentProjectLogIds {
-    kind: "project_log";
-    org_id: string;
-    project_id: string;
-    log_id: "g";
+    asDataset<Input, Expected>(): AsyncGenerator<EvalCase<Input, Expected, void>>;
 }
 /**
  * Primary implementation of the `Span` interface. See the `Span` interface for full details on each method.
@@ -635,10 +585,12 @@ export declare class SpanImpl implements Span {
     private internalData;
     private isMerge;
     private loggedEndTime;
+    parentObject: Experiment | Logger<any>;
     private parentIds;
     private readonly rowIds;
     kind: "span";
     constructor(args: {
+        parentObject: Experiment | Logger<any>;
         parentIds: LazyValue<ParentExperimentIds | ParentProjectLogIds>;
         bgLogger: BackgroundLogger;
     } & Omit<StartSpanArgs, "parentId"> & ({
@@ -660,16 +612,16 @@ export declare class SpanImpl implements Span {
     close(args?: EndSpanArgs): number;
 }
 /**
- * A dataset is a collection of records, such as model inputs and outputs, which represent
+ * A dataset is a collection of records, such as model inputs and expected outputs, which represent
  * data you can use to evaluate and fine-tune models. You can log production data to datasets,
  * curate them with interesting examples, edit/delete records, and run evaluations against them.
  *
  * You should not create `Dataset` objects directly. Instead, use the `braintrust.initDataset()` method.
  */
-export declare class Dataset extends ObjectFetcher<DatasetRecord> {
+declare class Dataset<IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY_DATASET> extends ObjectFetcher<DatasetRecord<IsLegacyDataset>> {
     private readonly lazyMetadata;
     private bgLogger;
-    constructor(lazyMetadata: LazyValue<ProjectDatasetMetadata>, pinnedVersion?: string);
+    constructor(lazyMetadata: LazyValue<ProjectDatasetMetadata>, pinnedVersion?: string, legacy?: IsLegacyDataset);
     get id(): Promise<string>;
     get name(): Promise<string>;
     get project(): Promise<ObjectMetadata>;
@@ -680,19 +632,21 @@ export declare class Dataset extends ObjectFetcher<DatasetRecord> {
      *
      * @param event The event to log.
      * @param event.input The argument that uniquely define an input case (an arbitrary, JSON serializable object).
-     * @param event.output The output of your application, including post-processing (an arbitrary, JSON serializable object).
+     * @param event.expected The output of your application, including post-processing (an arbitrary, JSON serializable object).
      * @param event.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
      * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
      * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
      * JSON-serializable type, but its keys must be strings.
      * @param event.id (Optional) a unique identifier for the event. If you don't provide one, Braintrust will generate one for you.
+     * @param event.output: (Deprecated) The output of your application. Use `expected` instead.
      * @returns The `id` of the logged record.
      */
-    insert({ input, output, metadata, id, }: {
+    insert({ input, expected, metadata, id, output, }: {
         readonly input?: unknown;
-        readonly output: unknown;
+        readonly expected?: unknown;
         readonly metadata?: Record<string, unknown>;
         readonly id?: string;
+        readonly output?: unknown;
     }): string;
     delete(id: string): string;
     /**
@@ -713,6 +667,7 @@ export declare class Dataset extends ObjectFetcher<DatasetRecord> {
      */
     close(): Promise<string>;
 }
+export type AnyDataset = Dataset<boolean>;
 /**
  * Summary of a score's performance.
  * @property name Name of the score.