braintrust 0.0.84 → 0.0.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +495 -478
- package/dist/cli.js +302 -381
- package/dist/index.js +536 -525
- package/dist/logger.d.ts +143 -138
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
package/dist/logger.d.ts
CHANGED
|
@@ -7,20 +7,20 @@ export type SetCurrentArg = {
|
|
|
7
7
|
};
|
|
8
8
|
type StartSpanEventArgs = ExperimentLogPartialArgs & Partial<IdField>;
|
|
9
9
|
export type StartSpanArgs = {
|
|
10
|
+
name?: string;
|
|
10
11
|
spanAttributes?: Record<any, any>;
|
|
11
12
|
startTime?: number;
|
|
12
13
|
event?: StartSpanEventArgs;
|
|
13
14
|
};
|
|
14
|
-
export type StartSpanOptionalNameArgs = StartSpanArgs & {
|
|
15
|
-
name?: string;
|
|
16
|
-
};
|
|
17
15
|
export type EndSpanArgs = {
|
|
18
16
|
endTime?: number;
|
|
19
17
|
};
|
|
20
18
|
/**
|
|
21
19
|
* A Span encapsulates logged data and metrics for a unit of work. This interface is shared by all span implementations.
|
|
22
20
|
*
|
|
23
|
-
* We suggest using one of the various `
|
|
21
|
+
* We suggest using one of the various `traced` methods, instead of creating Spans directly.
|
|
22
|
+
*
|
|
23
|
+
* See `Span.traced` for full details.
|
|
24
24
|
*/
|
|
25
25
|
export interface Span {
|
|
26
26
|
/**
|
|
@@ -42,27 +42,31 @@ export interface Span {
|
|
|
42
42
|
*/
|
|
43
43
|
log(event: ExperimentLogPartialArgs): void;
|
|
44
44
|
/**
|
|
45
|
-
* Create a new span. This is useful if you want to log more detailed trace information beyond the scope of a single log event. Data logged over several calls to `Span.log` will be merged into one logical row.
|
|
45
|
+
* Create a new span and run the provided callback. This is useful if you want to log more detailed trace information beyond the scope of a single log event. Data logged over several calls to `Span.log` will be merged into one logical row.
|
|
46
46
|
*
|
|
47
|
-
*
|
|
47
|
+
* Spans created within `traced` are ended automatically. By default, the span is marked as current, so they can be accessed using `braintrust.currentSpan`.
|
|
48
48
|
*
|
|
49
|
-
* @param
|
|
49
|
+
* @param callback The function to be run under the span context.
|
|
50
|
+
* @param args.name Optional name of the span. If not provided, a name will be inferred from the call stack.
|
|
50
51
|
* @param args.span_attributes Optional additional attributes to attach to the span, such as a type name.
|
|
51
52
|
* @param args.start_time Optional start time of the span, as a timestamp in seconds.
|
|
53
|
+
* @param args.setCurrent If true (the default), the span will be marked as the currently-active span for the duration of the callback.
|
|
52
54
|
* @param args.event Data to be logged. See `Experiment.log` for full details.
|
|
53
|
-
* @
|
|
55
|
+
* @Returns The result of running `callback`.
|
|
54
56
|
*/
|
|
55
|
-
|
|
57
|
+
traced<R>(callback: (span: Span) => R, args?: StartSpanArgs & SetCurrentArg): R;
|
|
56
58
|
/**
|
|
57
|
-
*
|
|
59
|
+
* Lower-level alternative to `traced`, which does not automatically end the span or mark it as current. Be sure to end the span with `span.end()` when it has finished.
|
|
60
|
+
*
|
|
61
|
+
* See `traced` for full details.
|
|
58
62
|
*
|
|
59
|
-
* @
|
|
63
|
+
* @returns The newly-created `Span`
|
|
60
64
|
*/
|
|
61
|
-
|
|
65
|
+
startSpan(args?: StartSpanArgs): Span;
|
|
62
66
|
/**
|
|
63
|
-
*
|
|
67
|
+
* Log an end time to the span (defaults to the current time). Returns the logged time.
|
|
64
68
|
*
|
|
65
|
-
* Will be invoked automatically if the span is constructed with traced
|
|
69
|
+
* Will be invoked automatically if the span is constructed with `traced`.
|
|
66
70
|
*
|
|
67
71
|
* @param args.endTime Optional end time of the span, as a timestamp in seconds.
|
|
68
72
|
* @returns The end time logged to the span metrics.
|
|
@@ -84,19 +88,19 @@ export declare class NoopSpan implements Span {
|
|
|
84
88
|
kind: "span";
|
|
85
89
|
constructor();
|
|
86
90
|
log(_: ExperimentLogPartialArgs): void;
|
|
87
|
-
|
|
88
|
-
|
|
91
|
+
traced<R>(callback: (span: Span) => R, _1: StartSpanArgs & SetCurrentArg): R;
|
|
92
|
+
startSpan(_1?: StartSpanArgs): this;
|
|
89
93
|
end(args?: EndSpanArgs): number;
|
|
90
94
|
close(args?: EndSpanArgs): number;
|
|
91
95
|
}
|
|
92
|
-
export declare const
|
|
96
|
+
export declare const NOOP_SPAN: NoopSpan;
|
|
93
97
|
declare global {
|
|
94
98
|
var __inherited_braintrust_state: BraintrustState;
|
|
95
99
|
}
|
|
96
100
|
declare class BraintrustState {
|
|
97
101
|
id: string;
|
|
98
|
-
currentExperiment:
|
|
99
|
-
currentLogger:
|
|
102
|
+
currentExperiment: Experiment | undefined;
|
|
103
|
+
currentLogger: Logger<false> | undefined;
|
|
100
104
|
currentSpan: IsoAsyncLocalStorage<Span>;
|
|
101
105
|
apiUrl: string | null;
|
|
102
106
|
loginToken: string | null;
|
|
@@ -107,6 +111,7 @@ declare class BraintrustState {
|
|
|
107
111
|
private _apiConn;
|
|
108
112
|
private _logConn;
|
|
109
113
|
constructor();
|
|
114
|
+
resetLoginInfo(): void;
|
|
110
115
|
apiConn(): HTTPConnection;
|
|
111
116
|
logConn(): HTTPConnection;
|
|
112
117
|
}
|
|
@@ -127,32 +132,37 @@ declare class HTTPConnection {
|
|
|
127
132
|
get_json(object_type: string, args?: Record<string, string> | undefined, retries?: number): Promise<any>;
|
|
128
133
|
post_json(object_type: string, args?: Record<string, unknown> | string | undefined): Promise<any>;
|
|
129
134
|
}
|
|
130
|
-
interface
|
|
135
|
+
export interface ObjectMetadata {
|
|
131
136
|
id: string;
|
|
132
137
|
name: string;
|
|
138
|
+
fullInfo: Record<string, unknown>;
|
|
133
139
|
}
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
140
|
+
interface ProjectExperimentMetadata {
|
|
141
|
+
project: ObjectMetadata;
|
|
142
|
+
experiment: ObjectMetadata;
|
|
143
|
+
}
|
|
144
|
+
interface ProjectDatasetMetadata {
|
|
145
|
+
project: ObjectMetadata;
|
|
146
|
+
dataset: ObjectMetadata;
|
|
147
|
+
}
|
|
148
|
+
interface OrgProjectMetadata {
|
|
149
|
+
org_id: string;
|
|
150
|
+
project: ObjectMetadata;
|
|
142
151
|
}
|
|
143
|
-
export interface LogOptions {
|
|
144
|
-
asyncFlush?:
|
|
152
|
+
export interface LogOptions<IsAsyncFlush> {
|
|
153
|
+
asyncFlush?: IsAsyncFlush;
|
|
145
154
|
}
|
|
146
|
-
export
|
|
147
|
-
|
|
148
|
-
private
|
|
149
|
-
private lazyProject;
|
|
155
|
+
export type PromiseUnless<B, R> = B extends true ? R : Promise<Awaited<R>>;
|
|
156
|
+
export declare class Logger<IsAsyncFlush extends boolean> {
|
|
157
|
+
private lazyMetadata;
|
|
150
158
|
private logOptions;
|
|
151
159
|
private bgLogger;
|
|
152
160
|
private lastStartTime;
|
|
153
161
|
kind: "logger";
|
|
154
|
-
constructor(
|
|
155
|
-
|
|
162
|
+
constructor(lazyMetadata: Promise<OrgProjectMetadata>, logOptions?: LogOptions<IsAsyncFlush>);
|
|
163
|
+
get org_id(): Promise<string>;
|
|
164
|
+
get project(): Promise<ObjectMetadata>;
|
|
165
|
+
private getState;
|
|
156
166
|
/**
|
|
157
167
|
* Log a single event. The event will be batched and uploaded behind the scenes if `logOptions.asyncFlush` is true.
|
|
158
168
|
*
|
|
@@ -166,18 +176,21 @@ export declare class Logger {
|
|
|
166
176
|
* @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
|
|
167
177
|
* :returns: The `id` of the logged event.
|
|
168
178
|
*/
|
|
169
|
-
log(event: Readonly<StartSpanEventArgs>):
|
|
179
|
+
log(event: Readonly<StartSpanEventArgs>): PromiseUnless<IsAsyncFlush, string>;
|
|
170
180
|
/**
|
|
171
|
-
* Create a new toplevel span. The name
|
|
181
|
+
* Create a new toplevel span underneath the logger. The name defaults to "root".
|
|
172
182
|
*
|
|
173
|
-
* See `Span.
|
|
183
|
+
* See `Span.traced` for full details.
|
|
174
184
|
*/
|
|
175
|
-
|
|
185
|
+
traced<R>(callback: (span: Span) => R, args?: StartSpanArgs & SetCurrentArg): PromiseUnless<IsAsyncFlush, R>;
|
|
176
186
|
/**
|
|
177
|
-
*
|
|
187
|
+
* Lower-level alternative to `traced`, which does not automatically end the span or mark it as current.
|
|
188
|
+
*
|
|
189
|
+
* See `traced` for full details.
|
|
178
190
|
*/
|
|
179
|
-
|
|
191
|
+
startSpan(args?: StartSpanArgs): Span;
|
|
180
192
|
flush(): Promise<void>;
|
|
193
|
+
get asyncFlush(): IsAsyncFlush | undefined;
|
|
181
194
|
}
|
|
182
195
|
export type IdField = {
|
|
183
196
|
id: string;
|
|
@@ -231,11 +244,12 @@ export interface DatasetRecord {
|
|
|
231
244
|
metadata: any;
|
|
232
245
|
}
|
|
233
246
|
declare class BackgroundLogger {
|
|
247
|
+
private logConn;
|
|
234
248
|
private items;
|
|
235
249
|
private active_flush;
|
|
236
250
|
private active_flush_resolved;
|
|
237
|
-
constructor();
|
|
238
|
-
log(items: BackgroundLogEvent[]): void;
|
|
251
|
+
constructor(logConn: Promise<HTTPConnection>);
|
|
252
|
+
log(items: Promise<BackgroundLogEvent>[]): void;
|
|
239
253
|
flush_once(batchSize?: number): Promise<string[]>;
|
|
240
254
|
flush(): Promise<void>;
|
|
241
255
|
}
|
|
@@ -249,14 +263,12 @@ export type InitOptions = {
|
|
|
249
263
|
apiUrl?: string;
|
|
250
264
|
apiKey?: string;
|
|
251
265
|
orgName?: string;
|
|
252
|
-
disableCache?: boolean;
|
|
253
266
|
metadata?: Metadata;
|
|
267
|
+
setCurrent?: boolean;
|
|
254
268
|
};
|
|
255
269
|
/**
|
|
256
270
|
* Log in, and then initialize a new experiment in a specified project. If the project does not exist, it will be created.
|
|
257
271
|
*
|
|
258
|
-
* Remember to close your experiment when it is finished by calling `Experiment.close`. We recommend initializing the experiment within a callback (using `braintrust.withExperiment`) to automatically mark it as current and ensure it is terminated.
|
|
259
|
-
*
|
|
260
272
|
* @param project The name of the project to create the experiment in.
|
|
261
273
|
* @param options Additional options for configuring init().
|
|
262
274
|
* @param options.experiment The name of the experiment to create. If not specified, a name will be generated automatically.
|
|
@@ -271,26 +283,22 @@ export type InitOptions = {
|
|
|
271
283
|
* @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API
|
|
272
284
|
* key is specified, will prompt the user to login.
|
|
273
285
|
* @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
|
|
274
|
-
* @param options.disableCache Do not use cached login information.
|
|
275
286
|
* @param options.metadata (Optional) A dictionary with additional data about the test example, model outputs, or just
|
|
276
287
|
* about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
|
|
277
288
|
* `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
|
|
278
289
|
* JSON-serializable type, but its keys must be strings.
|
|
290
|
+
* @param setCurrent If true (the default), set the global current-experiment to the newly-created one.
|
|
279
291
|
* @returns The newly created Experiment.
|
|
280
292
|
*/
|
|
281
|
-
export declare function init(project: string, options?: Readonly<InitOptions>):
|
|
293
|
+
export declare function init(project: string, options?: Readonly<InitOptions>): Experiment;
|
|
282
294
|
/**
|
|
283
|
-
*
|
|
284
|
-
*
|
|
285
|
-
* @param options.setCurrent If true (default), set the currently-active experiment to the newly-created one. Equivalent to calling `braintrust.withCurrent(experiment, callback)`.
|
|
295
|
+
* This function is deprecated. Use `init` instead.
|
|
286
296
|
*/
|
|
287
|
-
export declare function withExperiment<R>(project: string, callback: (experiment: Experiment) => R, options?: Readonly<InitOptions & SetCurrentArg>):
|
|
297
|
+
export declare function withExperiment<R>(project: string, callback: (experiment: Experiment) => R, options?: Readonly<InitOptions & SetCurrentArg>): R;
|
|
288
298
|
/**
|
|
289
|
-
*
|
|
290
|
-
*
|
|
291
|
-
* @param options.setCurrent If true (default), set the currently-active logger to the newly-created one. Equivalent to calling `braintrust.withCurrent(logger, callback)`.
|
|
299
|
+
* This function is deprecated. Use `initLogger` instead.
|
|
292
300
|
*/
|
|
293
|
-
export declare function withLogger<R>(callback: (logger: Logger) => R, options?: Readonly<InitLoggerOptions & SetCurrentArg>):
|
|
301
|
+
export declare function withLogger<IsAsyncFlush extends boolean = false, R = void>(callback: (logger: Logger<IsAsyncFlush>) => R, options?: Readonly<InitLoggerOptions<IsAsyncFlush> & SetCurrentArg>): R;
|
|
294
302
|
type InitDatasetOptions = {
|
|
295
303
|
dataset?: string;
|
|
296
304
|
description?: string;
|
|
@@ -298,13 +306,10 @@ type InitDatasetOptions = {
|
|
|
298
306
|
apiUrl?: string;
|
|
299
307
|
apiKey?: string;
|
|
300
308
|
orgName?: string;
|
|
301
|
-
disableCache?: boolean;
|
|
302
309
|
};
|
|
303
310
|
/**
|
|
304
311
|
* Create a new dataset in a specified project. If the project does not exist, it will be created.
|
|
305
312
|
*
|
|
306
|
-
* Remember to close your dataset when it is finished by calling `Dataset.close`. We recommend initializing the dataset within a callback (using `braintrust.withDataset`) to ensure it is terminated.
|
|
307
|
-
*
|
|
308
313
|
* @param project The name of the project to create the dataset in.
|
|
309
314
|
* @param options Additional options for configuring init().
|
|
310
315
|
* @param options.dataset The name of the dataset to create. If not specified, a name will be generated automatically.
|
|
@@ -313,23 +318,25 @@ type InitDatasetOptions = {
|
|
|
313
318
|
* @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API
|
|
314
319
|
* key is specified, will prompt the user to login.
|
|
315
320
|
* @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
|
|
316
|
-
* @param options.disableCache Do not use cached login information.
|
|
317
321
|
* @returns The newly created Dataset.
|
|
318
322
|
*/
|
|
319
|
-
export declare function initDataset(project: string, options?: Readonly<InitDatasetOptions>):
|
|
323
|
+
export declare function initDataset(project: string, options?: Readonly<InitDatasetOptions>): Dataset;
|
|
320
324
|
/**
|
|
321
|
-
*
|
|
325
|
+
* This function is deprecated. Use `initDataset` instead.
|
|
322
326
|
*/
|
|
323
|
-
export declare function withDataset<R>(project: string, callback: (dataset: Dataset) => R, options?: Readonly<InitDatasetOptions>):
|
|
324
|
-
type
|
|
327
|
+
export declare function withDataset<R>(project: string, callback: (dataset: Dataset) => R, options?: Readonly<InitDatasetOptions>): R;
|
|
328
|
+
type AsyncFlushArg<IsAsyncFlush> = {
|
|
329
|
+
asyncFlush?: IsAsyncFlush;
|
|
330
|
+
};
|
|
331
|
+
type InitLoggerOptions<IsAsyncFlush> = {
|
|
325
332
|
projectName?: string;
|
|
326
333
|
projectId?: string;
|
|
327
|
-
asyncFlush?: boolean;
|
|
328
334
|
apiUrl?: string;
|
|
329
335
|
apiKey?: string;
|
|
330
336
|
orgName?: string;
|
|
331
|
-
|
|
332
|
-
|
|
337
|
+
forceLogin?: boolean;
|
|
338
|
+
setCurrent?: boolean;
|
|
339
|
+
} & AsyncFlushArg<IsAsyncFlush>;
|
|
333
340
|
/**
|
|
334
341
|
* Create a new logger in a specified project. If the project does not exist, it will be created.
|
|
335
342
|
*
|
|
@@ -341,10 +348,11 @@ type InitLoggerOptions = {
|
|
|
341
348
|
* @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API
|
|
342
349
|
* key is specified, will prompt the user to login.
|
|
343
350
|
* @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
|
|
344
|
-
* @param options.
|
|
351
|
+
* @param options.forceLogin Login again, even if you have already logged in (by default, the logger will not login if you are already logged in)
|
|
352
|
+
* @param setCurrent If true (the default), set the global current-experiment to the newly-created one.
|
|
345
353
|
* @returns The newly created Logger.
|
|
346
354
|
*/
|
|
347
|
-
export declare function initLogger(options?: Readonly<InitLoggerOptions
|
|
355
|
+
export declare function initLogger<IsAsyncFlush extends boolean = false>(options?: Readonly<InitLoggerOptions<IsAsyncFlush>>): Logger<IsAsyncFlush>;
|
|
348
356
|
/**
|
|
349
357
|
* Log into Braintrust. This will prompt you for your API token, which you can find at
|
|
350
358
|
* https://www.braintrustdata.com/app/token. This method is called automatically by `init()`.
|
|
@@ -354,14 +362,12 @@ export declare function initLogger(options?: Readonly<InitLoggerOptions>): Logge
|
|
|
354
362
|
* @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API
|
|
355
363
|
* key is specified, will prompt the user to login.
|
|
356
364
|
* @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
|
|
357
|
-
* @param options.disableCache Do not use cached login information.
|
|
358
365
|
* @param options.forceLogin Login again, even if you have already logged in (by default, this function will exit quickly if you have already logged in)
|
|
359
366
|
*/
|
|
360
367
|
export declare function login(options?: {
|
|
361
368
|
apiUrl?: string;
|
|
362
369
|
apiKey?: string;
|
|
363
370
|
orgName?: string;
|
|
364
|
-
disableCache?: boolean;
|
|
365
371
|
forceLogin?: boolean;
|
|
366
372
|
}): Promise<void>;
|
|
367
373
|
/**
|
|
@@ -384,45 +390,38 @@ export declare function summarize(options?: {
|
|
|
384
390
|
readonly comparisonExperimentId?: string;
|
|
385
391
|
}): Promise<ExperimentSummary>;
|
|
386
392
|
/**
|
|
387
|
-
* Returns the currently-active experiment (set by `braintrust.
|
|
393
|
+
* Returns the currently-active experiment (set by `braintrust.init`). Returns undefined if no current experiment has been set.
|
|
388
394
|
*/
|
|
389
395
|
export declare function currentExperiment(): Experiment | undefined;
|
|
390
396
|
/**
|
|
391
|
-
* Returns the currently-active logger (set by `braintrust.
|
|
397
|
+
* Returns the currently-active logger (set by `braintrust.initLogger`). Returns undefined if no current logger has been set.
|
|
392
398
|
*/
|
|
393
|
-
export declare function currentLogger(): Logger | undefined;
|
|
399
|
+
export declare function currentLogger<IsAsyncFlush extends boolean>(options?: AsyncFlushArg<IsAsyncFlush>): Logger<IsAsyncFlush> | undefined;
|
|
394
400
|
/**
|
|
395
|
-
* Return the currently-active span for logging (set by `traced`
|
|
401
|
+
* Return the currently-active span for logging (set by one of the `traced` methods). If there is no active span, returns a no-op span object, which supports the same interface as spans but does no logging.
|
|
396
402
|
*
|
|
397
403
|
* See `Span` for full details.
|
|
398
404
|
*/
|
|
399
405
|
export declare function currentSpan(): Span;
|
|
406
|
+
/**
|
|
407
|
+
* Mainly for internal use. Return the parent object for starting a span in a global context.
|
|
408
|
+
*/
|
|
409
|
+
export declare function getSpanParentObject<IsAsyncFlush extends boolean>(options?: AsyncFlushArg<IsAsyncFlush>): Span | Experiment | Logger<IsAsyncFlush>;
|
|
400
410
|
/**
|
|
401
411
|
* Toplevel function for starting a span. It checks the following (in precedence order):
|
|
402
412
|
* * Currently-active span
|
|
403
413
|
* * Currently-active experiment
|
|
404
414
|
* * Currently-active logger
|
|
405
415
|
*
|
|
406
|
-
* and creates a span
|
|
407
|
-
*
|
|
408
|
-
* Unless a name is explicitly provided, the name of the span will be the name of the calling function, or "root" if no meaningful name can be determined.
|
|
416
|
+
* and creates a span under the first one that is active. If none of these are active, it returns a no-op span object.
|
|
409
417
|
*
|
|
410
|
-
*
|
|
411
|
-
*
|
|
412
|
-
* See `Span.startSpan` for full details.
|
|
413
|
-
*/
|
|
414
|
-
export declare function startSpan(args?: StartSpanOptionalNameArgs): Span;
|
|
415
|
-
/**
|
|
416
|
-
* Wrapper over `braintrust.startSpan`, which passes the initialized `Span` it to the given callback and ends it afterwards. See `Span.traced` for full details.
|
|
418
|
+
* See `Span.traced` for full details.
|
|
417
419
|
*/
|
|
418
|
-
export declare function traced<R>(callback: (span: Span) => R, args?:
|
|
420
|
+
export declare function traced<IsAsyncFlush extends boolean = false, R = void>(callback: (span: Span) => R, args?: StartSpanArgs & SetCurrentArg & AsyncFlushArg<IsAsyncFlush>): PromiseUnless<IsAsyncFlush, R>;
|
|
419
421
|
/**
|
|
420
|
-
*
|
|
421
|
-
*
|
|
422
|
-
* @param object: The experiment or span to be marked as current.
|
|
423
|
-
* @param callback: The callback to be run under the scope of the current object.
|
|
422
|
+
* Lower-level alternative to `traced`, which does not automatically end the span or mark it as current. See `traced` for full details.
|
|
424
423
|
*/
|
|
425
|
-
export declare function
|
|
424
|
+
export declare function startSpan<IsAsyncFlush extends boolean = false>(args?: StartSpanArgs & AsyncFlushArg<IsAsyncFlush>): Span;
|
|
426
425
|
/**
|
|
427
426
|
* An experiment is a collection of logged events, such as model inputs and outputs, which represent
|
|
428
427
|
* a snapshot of your application at a particular point in time. An experiment is meant to capture more
|
|
@@ -436,15 +435,16 @@ export declare function withCurrent<R>(object: Experiment | Logger | Span, callb
|
|
|
436
435
|
* You should not create `Experiment` objects directly. Instead, use the `braintrust.init()` method.
|
|
437
436
|
*/
|
|
438
437
|
export declare class Experiment {
|
|
439
|
-
readonly
|
|
440
|
-
readonly id: string;
|
|
441
|
-
readonly name: string;
|
|
438
|
+
private readonly lazyMetadata;
|
|
442
439
|
readonly dataset?: Dataset;
|
|
443
440
|
private bgLogger;
|
|
444
441
|
private lastStartTime;
|
|
445
|
-
private finished;
|
|
446
442
|
kind: "experiment";
|
|
447
|
-
constructor(
|
|
443
|
+
constructor(lazyMetadata: Promise<ProjectExperimentMetadata>, dataset?: Dataset);
|
|
444
|
+
get id(): Promise<string>;
|
|
445
|
+
get name(): Promise<string>;
|
|
446
|
+
get project(): Promise<ObjectMetadata>;
|
|
447
|
+
private getState;
|
|
448
448
|
/**
|
|
449
449
|
* Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
|
|
450
450
|
*
|
|
@@ -462,15 +462,17 @@ export declare class Experiment {
|
|
|
462
462
|
*/
|
|
463
463
|
log(event: Readonly<ExperimentLogFullArgs>): string;
|
|
464
464
|
/**
|
|
465
|
-
* Create a new toplevel span. The name
|
|
465
|
+
* Create a new toplevel span underneath the experiment. The name defaults to "root".
|
|
466
466
|
*
|
|
467
|
-
* See `Span.
|
|
467
|
+
* See `Span.traced` for full details.
|
|
468
468
|
*/
|
|
469
|
-
|
|
469
|
+
traced<R>(callback: (span: Span) => R, args?: StartSpanArgs & SetCurrentArg): R;
|
|
470
470
|
/**
|
|
471
|
-
*
|
|
471
|
+
* Lower-level alternative to `traced`, which does not automatically end the span or mark it as current.
|
|
472
|
+
*
|
|
473
|
+
* See `traced` for full details.
|
|
472
474
|
*/
|
|
473
|
-
|
|
475
|
+
startSpan(args?: StartSpanArgs): Span;
|
|
474
476
|
/**
|
|
475
477
|
* Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
|
|
476
478
|
*
|
|
@@ -484,51 +486,54 @@ export declare class Experiment {
|
|
|
484
486
|
readonly comparisonExperimentId?: string;
|
|
485
487
|
}): Promise<ExperimentSummary>;
|
|
486
488
|
/**
|
|
487
|
-
*
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
*
|
|
489
|
+
* Flush any pending rows to the server.
|
|
490
|
+
*/
|
|
491
|
+
flush(): Promise<void>;
|
|
492
|
+
/**
|
|
493
|
+
* This function is deprecated. You can simply remove it from your code.
|
|
492
494
|
*/
|
|
493
495
|
close(): Promise<string>;
|
|
494
|
-
|
|
496
|
+
}
|
|
497
|
+
interface ParentExperimentIds {
|
|
498
|
+
kind: "experiment";
|
|
499
|
+
project_id: string;
|
|
500
|
+
experiment_id: string;
|
|
501
|
+
}
|
|
502
|
+
interface ParentProjectLogIds {
|
|
503
|
+
kind: "project_log";
|
|
504
|
+
org_id: string;
|
|
505
|
+
project_id: string;
|
|
506
|
+
log_id: "g";
|
|
495
507
|
}
|
|
496
508
|
/**
|
|
497
509
|
* Primary implementation of the `Span` interface. See the `Span` interface for full details on each method.
|
|
498
510
|
*
|
|
499
|
-
* We suggest using one of the various `
|
|
511
|
+
* We suggest using one of the various `traced` methods, instead of creating Spans directly. See `Span.startSpan` for full details.
|
|
500
512
|
*/
|
|
501
513
|
export declare class SpanImpl implements Span {
|
|
502
|
-
private finished;
|
|
503
514
|
private bgLogger;
|
|
504
515
|
private internalData;
|
|
505
516
|
private isMerge;
|
|
506
517
|
private loggedEndTime;
|
|
507
|
-
private
|
|
518
|
+
private parentIds;
|
|
519
|
+
private readonly rowIds;
|
|
508
520
|
kind: "span";
|
|
509
521
|
constructor(args: {
|
|
522
|
+
parentIds: Promise<ParentExperimentIds | ParentProjectLogIds>;
|
|
510
523
|
bgLogger: BackgroundLogger;
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
} & ({
|
|
517
|
-
rootExperiment: Experiment;
|
|
518
|
-
} | {
|
|
519
|
-
rootProject: RegisteredProject;
|
|
520
|
-
} | {
|
|
521
|
-
parentSpan: SpanImpl;
|
|
522
|
-
}));
|
|
524
|
+
parentSpanInfo?: {
|
|
525
|
+
span_id: string;
|
|
526
|
+
root_span_id: string;
|
|
527
|
+
};
|
|
528
|
+
} & StartSpanArgs);
|
|
523
529
|
get id(): string;
|
|
524
530
|
get span_id(): string;
|
|
525
531
|
get root_span_id(): string;
|
|
526
532
|
log(event: ExperimentLogPartialArgs): void;
|
|
527
|
-
|
|
528
|
-
|
|
533
|
+
traced<R>(callback: (span: Span) => R, args?: StartSpanArgs & SetCurrentArg): R;
|
|
534
|
+
startSpan(args?: StartSpanArgs): Span;
|
|
529
535
|
end(args?: EndSpanArgs): number;
|
|
530
536
|
close(args?: EndSpanArgs): number;
|
|
531
|
-
private checkNotFinished;
|
|
532
537
|
}
|
|
533
538
|
/**
|
|
534
539
|
* A dataset is a collection of records, such as model inputs and outputs, which represent
|
|
@@ -538,14 +543,15 @@ export declare class SpanImpl implements Span {
|
|
|
538
543
|
* You should not create `Dataset` objects directly. Instead, use the `braintrust.initDataset()` method.
|
|
539
544
|
*/
|
|
540
545
|
export declare class Dataset {
|
|
541
|
-
readonly
|
|
542
|
-
readonly id: string;
|
|
543
|
-
readonly name: string;
|
|
546
|
+
private readonly lazyMetadata;
|
|
544
547
|
private pinnedVersion?;
|
|
545
548
|
private _fetchedData?;
|
|
546
|
-
private
|
|
547
|
-
|
|
548
|
-
|
|
549
|
+
private bgLogger;
|
|
550
|
+
constructor(lazyMetadata: Promise<ProjectDatasetMetadata>, pinnedVersion?: string);
|
|
551
|
+
get id(): Promise<string>;
|
|
552
|
+
get name(): Promise<string>;
|
|
553
|
+
get project(): Promise<ObjectMetadata>;
|
|
554
|
+
private getState;
|
|
549
555
|
/**
|
|
550
556
|
* Insert a single record to the dataset. The record will be batched and uploaded behind the scenes. If you pass in an `id`,
|
|
551
557
|
* and a record with that `id` already exists, it will be overwritten (upsert).
|
|
@@ -611,14 +617,13 @@ export declare class Dataset {
|
|
|
611
617
|
clearCache(): void;
|
|
612
618
|
version(): Promise<any>;
|
|
613
619
|
/**
|
|
614
|
-
*
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
*
|
|
620
|
+
* Flush any pending rows to the server.
|
|
621
|
+
*/
|
|
622
|
+
flush(): Promise<void>;
|
|
623
|
+
/**
|
|
624
|
+
* This function is deprecated. You can simply remove it from your code.
|
|
619
625
|
*/
|
|
620
626
|
close(): Promise<string>;
|
|
621
|
-
private checkNotFinished;
|
|
622
627
|
}
|
|
623
628
|
/**
|
|
624
629
|
* Summary of a score's performance.
|