braintrust 0.0.140 → 0.0.141-dev
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +42 -0
- package/.turbo/turbo-watch.log +1843 -0
- package/LICENSE +201 -0
- package/braintrust-0.0.141.tgz +0 -0
- package/dist/{logger.d.ts → browser.d.mts} +168 -92
- package/dist/browser.d.ts +1089 -2
- package/dist/browser.js +383 -9185
- package/dist/browser.mjs +254 -7
- package/dist/cli.js +3887 -27452
- package/dist/index.d.mts +1250 -0
- package/dist/index.d.ts +1236 -41
- package/dist/index.js +1706 -18925
- package/dist/index.mjs +255 -121
- package/package.json +40 -23
- package/tsup.config.ts +23 -0
- package/turbo.json +8 -0
- package/vitest.config.js +6 -0
- package/dist/browser-config.d.ts +0 -6
- package/dist/cli.d.ts +0 -35
- package/dist/framework.d.ts +0 -180
- package/dist/framework.test.d.ts +0 -1
- package/dist/functions.d.ts +0 -15
- package/dist/gitutil.d.ts +0 -17
- package/dist/isomorph.d.ts +0 -27
- package/dist/jest/nodeModulesPaths.d.ts +0 -15
- package/dist/jest/tryRealpath.d.ts +0 -7
- package/dist/node.d.ts +0 -1
- package/dist/progress.d.ts +0 -18
- package/dist/stackutil.d.ts +0 -8
- package/dist/tsconfig.tsbuildinfo +0 -1
- package/dist/util.d.ts +0 -11
- package/dist/wrappers/ai-sdk.d.ts +0 -8
- package/dist/wrappers/oai.d.ts +0 -34
- package/jest.config.js +0 -5
|
@@ -1,14 +1,28 @@
|
|
|
1
|
+
import { GitMetadataSettings, LogFeedbackFullArgs, BackgroundLogEvent, ExperimentEvent, ExperimentLogFullArgs, ExperimentLogPartialArgs, IdField, SpanType, RepoInfo, DEFAULT_IS_LEGACY_DATASET, TRANSACTION_ID_FIELD, TransactionId, SpanObjectTypeV2, DatasetRecord } from '@braintrust/core';
|
|
2
|
+
import { PromptData, OpenAIMessage, Tools, AnyModelParam, Prompt as Prompt$1, PromptSessionEvent, FunctionId } from '@braintrust/core/typespecs';
|
|
3
|
+
import { z } from 'zod';
|
|
4
|
+
|
|
5
|
+
interface IsoAsyncLocalStorage<T> {
|
|
6
|
+
enterWith(store: T): void;
|
|
7
|
+
run<R>(store: T | undefined, callback: () => R): R;
|
|
8
|
+
getStore(): T | undefined;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
declare class LazyValue<T> {
|
|
12
|
+
private callable;
|
|
13
|
+
private value;
|
|
14
|
+
constructor(callable: () => Promise<T>);
|
|
15
|
+
get(): Promise<T>;
|
|
16
|
+
get hasComputed(): boolean;
|
|
17
|
+
}
|
|
18
|
+
|
|
1
19
|
/// <reference lib="dom" />
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
import { IsoAsyncLocalStorage } from "./isomorph";
|
|
5
|
-
import { LazyValue } from "./util";
|
|
6
|
-
import { z } from "zod";
|
|
7
|
-
export type SetCurrentArg = {
|
|
20
|
+
|
|
21
|
+
type SetCurrentArg = {
|
|
8
22
|
setCurrent?: boolean;
|
|
9
23
|
};
|
|
10
24
|
type StartSpanEventArgs = ExperimentLogPartialArgs & Partial<IdField>;
|
|
11
|
-
|
|
25
|
+
type StartSpanArgs = {
|
|
12
26
|
name?: string;
|
|
13
27
|
type?: SpanType;
|
|
14
28
|
spanAttributes?: Record<any, any>;
|
|
@@ -16,9 +30,15 @@ export type StartSpanArgs = {
|
|
|
16
30
|
parent?: string;
|
|
17
31
|
event?: StartSpanEventArgs;
|
|
18
32
|
};
|
|
19
|
-
|
|
33
|
+
type EndSpanArgs = {
|
|
20
34
|
endTime?: number;
|
|
21
35
|
};
|
|
36
|
+
interface Exportable {
|
|
37
|
+
/**
|
|
38
|
+
* Return a serialized representation of the object that can be used to start subspans in other places. See `Span.traced` for more details.
|
|
39
|
+
*/
|
|
40
|
+
export(): Promise<string>;
|
|
41
|
+
}
|
|
22
42
|
/**
|
|
23
43
|
* A Span encapsulates logged data and metrics for a unit of work. This interface is shared by all span implementations.
|
|
24
44
|
*
|
|
@@ -26,7 +46,7 @@ export type EndSpanArgs = {
|
|
|
26
46
|
*
|
|
27
47
|
* See `Span.traced` for full details.
|
|
28
48
|
*/
|
|
29
|
-
|
|
49
|
+
interface Span extends Exportable {
|
|
30
50
|
/**
|
|
31
51
|
* Row ID of the span.
|
|
32
52
|
*/
|
|
@@ -78,10 +98,6 @@ export interface Span {
|
|
|
78
98
|
* @returns The end time logged to the span metrics.
|
|
79
99
|
*/
|
|
80
100
|
end(args?: EndSpanArgs): number;
|
|
81
|
-
/**
|
|
82
|
-
* Return a serialized representation of the span that can be used to start subspans in other places. See `Span.traced` for more details.
|
|
83
|
-
*/
|
|
84
|
-
export(): Promise<string>;
|
|
85
101
|
/**
|
|
86
102
|
* Flush any pending rows to the server.
|
|
87
103
|
*/
|
|
@@ -99,7 +115,7 @@ export interface Span {
|
|
|
99
115
|
/**
|
|
100
116
|
* A fake implementation of the Span API which does nothing. This can be used as the default span.
|
|
101
117
|
*/
|
|
102
|
-
|
|
118
|
+
declare class NoopSpan implements Span {
|
|
103
119
|
id: string;
|
|
104
120
|
kind: "span";
|
|
105
121
|
constructor();
|
|
@@ -113,7 +129,7 @@ export declare class NoopSpan implements Span {
|
|
|
113
129
|
close(args?: EndSpanArgs): number;
|
|
114
130
|
setAttributes(_args: Omit<StartSpanArgs, "event">): void;
|
|
115
131
|
}
|
|
116
|
-
|
|
132
|
+
declare const NOOP_SPAN: NoopSpan;
|
|
117
133
|
declare global {
|
|
118
134
|
var __inherited_braintrust_state: BraintrustState;
|
|
119
135
|
}
|
|
@@ -122,6 +138,7 @@ declare const loginSchema: z.ZodObject<{
|
|
|
122
138
|
appPublicUrl: z.ZodString;
|
|
123
139
|
orgName: z.ZodString;
|
|
124
140
|
logUrl: z.ZodString;
|
|
141
|
+
proxyUrl: z.ZodString;
|
|
125
142
|
loginToken: z.ZodString;
|
|
126
143
|
orgId: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
127
144
|
gitMetadataSettings: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
@@ -139,6 +156,7 @@ declare const loginSchema: z.ZodObject<{
|
|
|
139
156
|
appPublicUrl: string;
|
|
140
157
|
orgName: string;
|
|
141
158
|
logUrl: string;
|
|
159
|
+
proxyUrl: string;
|
|
142
160
|
loginToken: string;
|
|
143
161
|
orgId?: string | null | undefined;
|
|
144
162
|
gitMetadataSettings?: {
|
|
@@ -150,6 +168,7 @@ declare const loginSchema: z.ZodObject<{
|
|
|
150
168
|
appPublicUrl: string;
|
|
151
169
|
orgName: string;
|
|
152
170
|
logUrl: string;
|
|
171
|
+
proxyUrl: string;
|
|
153
172
|
loginToken: string;
|
|
154
173
|
orgId?: string | null | undefined;
|
|
155
174
|
gitMetadataSettings?: {
|
|
@@ -157,8 +176,8 @@ declare const loginSchema: z.ZodObject<{
|
|
|
157
176
|
fields?: ("dirty" | "tag" | "commit" | "branch" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
|
|
158
177
|
} | null | undefined;
|
|
159
178
|
}>;
|
|
160
|
-
|
|
161
|
-
|
|
179
|
+
type SerializedBraintrustState = z.infer<typeof loginSchema>;
|
|
180
|
+
declare class BraintrustState {
|
|
162
181
|
private loginParams;
|
|
163
182
|
id: string;
|
|
164
183
|
currentExperiment: Experiment | undefined;
|
|
@@ -171,11 +190,13 @@ export declare class BraintrustState {
|
|
|
171
190
|
orgId: string | null;
|
|
172
191
|
orgName: string | null;
|
|
173
192
|
logUrl: string | null;
|
|
193
|
+
proxyUrl: string | null;
|
|
174
194
|
loggedIn: boolean;
|
|
175
195
|
gitMetadataSettings?: GitMetadataSettings;
|
|
176
196
|
fetch: typeof globalThis.fetch;
|
|
177
197
|
private _apiConn;
|
|
178
198
|
private _logConn;
|
|
199
|
+
private _proxyConn;
|
|
179
200
|
constructor(loginParams: LoginOptions);
|
|
180
201
|
resetLoginInfo(): void;
|
|
181
202
|
copyLoginInfo(other: BraintrustState): void;
|
|
@@ -187,11 +208,12 @@ export declare class BraintrustState {
|
|
|
187
208
|
}): Promise<void>;
|
|
188
209
|
apiConn(): HTTPConnection;
|
|
189
210
|
logConn(): HTTPConnection;
|
|
211
|
+
proxyConn(): HTTPConnection;
|
|
190
212
|
bgLogger(): BackgroundLogger;
|
|
191
213
|
loginReplaceLogConn(logConn: HTTPConnection): void;
|
|
192
214
|
}
|
|
193
|
-
|
|
194
|
-
|
|
215
|
+
declare function _internalSetInitialState(): void;
|
|
216
|
+
declare const _internalGetGlobalState: () => BraintrustState;
|
|
195
217
|
declare class HTTPConnection {
|
|
196
218
|
base_url: string;
|
|
197
219
|
token: string | null;
|
|
@@ -209,7 +231,7 @@ declare class HTTPConnection {
|
|
|
209
231
|
get_json(object_type: string, args?: Record<string, string | undefined> | undefined, retries?: number): Promise<any>;
|
|
210
232
|
post_json(object_type: string, args?: Record<string, unknown> | string | undefined): Promise<any>;
|
|
211
233
|
}
|
|
212
|
-
|
|
234
|
+
interface ObjectMetadata {
|
|
213
235
|
id: string;
|
|
214
236
|
name: string;
|
|
215
237
|
fullInfo: Record<string, unknown>;
|
|
@@ -226,16 +248,16 @@ interface OrgProjectMetadata {
|
|
|
226
248
|
org_id: string;
|
|
227
249
|
project: ObjectMetadata;
|
|
228
250
|
}
|
|
229
|
-
|
|
251
|
+
interface LogOptions<IsAsyncFlush> {
|
|
230
252
|
asyncFlush?: IsAsyncFlush;
|
|
231
253
|
computeMetadataArgs?: Record<string, any>;
|
|
232
254
|
}
|
|
233
|
-
|
|
255
|
+
type PromiseUnless<B, R> = B extends true ? R : Promise<Awaited<R>>;
|
|
234
256
|
interface ParentSpanIds {
|
|
235
257
|
spanId: string;
|
|
236
258
|
rootSpanId: string;
|
|
237
259
|
}
|
|
238
|
-
|
|
260
|
+
declare class Logger<IsAsyncFlush extends boolean> implements Exportable {
|
|
239
261
|
private state;
|
|
240
262
|
private lazyMetadata;
|
|
241
263
|
private _asyncFlush;
|
|
@@ -249,6 +271,7 @@ export declare class Logger<IsAsyncFlush extends boolean> {
|
|
|
249
271
|
get project(): Promise<ObjectMetadata>;
|
|
250
272
|
get id(): Promise<string>;
|
|
251
273
|
private parentObjectType;
|
|
274
|
+
private triggerWaitUntilFlush;
|
|
252
275
|
/**
|
|
253
276
|
* Log a single event. The event will be batched and uploaded behind the scenes if `logOptions.asyncFlush` is true.
|
|
254
277
|
*
|
|
@@ -332,16 +355,13 @@ declare class BackgroundLogger {
|
|
|
332
355
|
type InitOpenOption<IsOpen extends boolean> = {
|
|
333
356
|
open?: IsOpen;
|
|
334
357
|
};
|
|
335
|
-
|
|
358
|
+
type InitOptions<IsOpen extends boolean> = FullLoginOptions & {
|
|
336
359
|
experiment?: string;
|
|
337
360
|
description?: string;
|
|
338
361
|
dataset?: AnyDataset;
|
|
339
362
|
update?: boolean;
|
|
340
363
|
baseExperiment?: string;
|
|
341
364
|
isPublic?: boolean;
|
|
342
|
-
appUrl?: string;
|
|
343
|
-
apiKey?: string;
|
|
344
|
-
orgName?: string;
|
|
345
365
|
metadata?: Record<string, unknown>;
|
|
346
366
|
gitMetadataSettings?: GitMetadataSettings;
|
|
347
367
|
projectId?: string;
|
|
@@ -350,7 +370,7 @@ export type InitOptions<IsOpen extends boolean> = {
|
|
|
350
370
|
setCurrent?: boolean;
|
|
351
371
|
state?: BraintrustState;
|
|
352
372
|
} & InitOpenOption<IsOpen>;
|
|
353
|
-
|
|
373
|
+
type FullInitOptions<IsOpen extends boolean> = {
|
|
354
374
|
project?: string;
|
|
355
375
|
} & InitOptions<IsOpen>;
|
|
356
376
|
type InitializedExperiment<IsOpen extends boolean | undefined> = IsOpen extends true ? ReadonlyExperiment : Experiment;
|
|
@@ -377,38 +397,35 @@ type InitializedExperiment<IsOpen extends boolean | undefined> = IsOpen extends
|
|
|
377
397
|
* @param options.repoInfo (Optional) Explicitly specify the git metadata for this experiment. This takes precedence over `gitMetadataSettings` if specified.
|
|
378
398
|
* @returns The newly created Experiment.
|
|
379
399
|
*/
|
|
380
|
-
|
|
400
|
+
declare function init<IsOpen extends boolean = false>(options: Readonly<FullInitOptions<IsOpen>>): InitializedExperiment<IsOpen>;
|
|
381
401
|
/**
|
|
382
402
|
* Legacy form of `init` which accepts the project name as the first parameter,
|
|
383
403
|
* separately from the remaining options. See `init(options)` for full details.
|
|
384
404
|
*/
|
|
385
|
-
|
|
405
|
+
declare function init<IsOpen extends boolean = false>(project: string, options?: Readonly<InitOptions<IsOpen>>): InitializedExperiment<IsOpen>;
|
|
386
406
|
/**
|
|
387
407
|
* Alias for init(options).
|
|
388
408
|
*/
|
|
389
|
-
|
|
409
|
+
declare function initExperiment<IsOpen extends boolean = false>(options: Readonly<InitOptions<IsOpen>>): InitializedExperiment<IsOpen>;
|
|
390
410
|
/**
|
|
391
411
|
* Alias for init(project, options).
|
|
392
412
|
*/
|
|
393
|
-
|
|
413
|
+
declare function initExperiment<IsOpen extends boolean = false>(project: string, options?: Readonly<InitOptions<IsOpen>>): InitializedExperiment<IsOpen>;
|
|
394
414
|
/**
|
|
395
415
|
* This function is deprecated. Use `init` instead.
|
|
396
416
|
*/
|
|
397
|
-
|
|
417
|
+
declare function withExperiment<R>(project: string, callback: (experiment: Experiment) => R, options?: Readonly<InitOptions<false> & SetCurrentArg>): R;
|
|
398
418
|
/**
|
|
399
419
|
* This function is deprecated. Use `initLogger` instead.
|
|
400
420
|
*/
|
|
401
|
-
|
|
421
|
+
declare function withLogger<IsAsyncFlush extends boolean = false, R = void>(callback: (logger: Logger<IsAsyncFlush>) => R, options?: Readonly<InitLoggerOptions<IsAsyncFlush> & SetCurrentArg>): R;
|
|
402
422
|
type UseOutputOption<IsLegacyDataset extends boolean> = {
|
|
403
423
|
useOutput?: IsLegacyDataset;
|
|
404
424
|
};
|
|
405
|
-
type InitDatasetOptions<IsLegacyDataset extends boolean> = {
|
|
425
|
+
type InitDatasetOptions<IsLegacyDataset extends boolean> = FullLoginOptions & {
|
|
406
426
|
dataset?: string;
|
|
407
427
|
description?: string;
|
|
408
428
|
version?: string;
|
|
409
|
-
appUrl?: string;
|
|
410
|
-
apiKey?: string;
|
|
411
|
-
orgName?: string;
|
|
412
429
|
projectId?: string;
|
|
413
430
|
state?: BraintrustState;
|
|
414
431
|
} & UseOutputOption<IsLegacyDataset>;
|
|
@@ -429,27 +446,23 @@ type FullInitDatasetOptions<IsLegacyDataset extends boolean> = {
|
|
|
429
446
|
* @param options.useOutput (Deprecated) If true, records will be fetched from this dataset in the legacy format, with the "expected" field renamed to "output". This option will be removed in a future version of Braintrust.
|
|
430
447
|
* @returns The newly created Dataset.
|
|
431
448
|
*/
|
|
432
|
-
|
|
449
|
+
declare function initDataset<IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY_DATASET>(options: Readonly<FullInitDatasetOptions<IsLegacyDataset>>): Dataset<IsLegacyDataset>;
|
|
433
450
|
/**
|
|
434
451
|
* Legacy form of `initDataset` which accepts the project name as the first
|
|
435
452
|
* parameter, separately from the remaining options. See
|
|
436
453
|
* `initDataset(options)` for full details.
|
|
437
454
|
*/
|
|
438
|
-
|
|
455
|
+
declare function initDataset<IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY_DATASET>(project: string, options?: Readonly<InitDatasetOptions<IsLegacyDataset>>): Dataset<IsLegacyDataset>;
|
|
439
456
|
/**
|
|
440
457
|
* This function is deprecated. Use `initDataset` instead.
|
|
441
458
|
*/
|
|
442
|
-
|
|
459
|
+
declare function withDataset<R, IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY_DATASET>(project: string, callback: (dataset: Dataset<IsLegacyDataset>) => R, options?: Readonly<InitDatasetOptions<IsLegacyDataset>>): R;
|
|
443
460
|
type AsyncFlushArg<IsAsyncFlush> = {
|
|
444
461
|
asyncFlush?: IsAsyncFlush;
|
|
445
462
|
};
|
|
446
|
-
type InitLoggerOptions<IsAsyncFlush> = {
|
|
463
|
+
type InitLoggerOptions<IsAsyncFlush> = FullLoginOptions & {
|
|
447
464
|
projectName?: string;
|
|
448
465
|
projectId?: string;
|
|
449
|
-
appUrl?: string;
|
|
450
|
-
apiKey?: string;
|
|
451
|
-
orgName?: string;
|
|
452
|
-
forceLogin?: boolean;
|
|
453
466
|
setCurrent?: boolean;
|
|
454
467
|
state?: BraintrustState;
|
|
455
468
|
} & AsyncFlushArg<IsAsyncFlush>;
|
|
@@ -468,19 +481,16 @@ type InitLoggerOptions<IsAsyncFlush> = {
|
|
|
468
481
|
* @param setCurrent If true (the default), set the global current-experiment to the newly-created one.
|
|
469
482
|
* @returns The newly created Logger.
|
|
470
483
|
*/
|
|
471
|
-
|
|
472
|
-
|
|
484
|
+
declare function initLogger<IsAsyncFlush extends boolean = false>(options?: Readonly<InitLoggerOptions<IsAsyncFlush>>): Logger<IsAsyncFlush>;
|
|
485
|
+
type LoadPromptOptions = FullLoginOptions & {
|
|
473
486
|
projectName?: string;
|
|
474
487
|
projectId?: string;
|
|
475
488
|
slug?: string;
|
|
476
489
|
version?: string;
|
|
477
490
|
defaults?: DefaultPromptArgs;
|
|
478
491
|
noTrace?: boolean;
|
|
479
|
-
appUrl?: string;
|
|
480
|
-
apiKey?: string;
|
|
481
|
-
orgName?: string;
|
|
482
492
|
state?: BraintrustState;
|
|
483
|
-
}
|
|
493
|
+
};
|
|
484
494
|
/**
|
|
485
495
|
* Load a prompt from the specified project.
|
|
486
496
|
*
|
|
@@ -507,11 +517,11 @@ interface LoadPromptOptions {
|
|
|
507
517
|
* });
|
|
508
518
|
* ```
|
|
509
519
|
*/
|
|
510
|
-
|
|
520
|
+
declare function loadPrompt({ projectName, projectId, slug, version, defaults, noTrace, appUrl, apiKey, orgName, fetch, forceLogin, state: stateArg, }: LoadPromptOptions): Promise<Prompt>;
|
|
511
521
|
/**
|
|
512
522
|
* Options for logging in to Braintrust.
|
|
513
523
|
*/
|
|
514
|
-
|
|
524
|
+
interface LoginOptions {
|
|
515
525
|
/**
|
|
516
526
|
* The URL of the Braintrust App. Defaults to https://www.braintrust.dev. You should not need
|
|
517
527
|
* to change this unless you are doing the "Full" deployment.
|
|
@@ -531,6 +541,9 @@ export interface LoginOptions {
|
|
|
531
541
|
*/
|
|
532
542
|
fetch?: typeof globalThis.fetch;
|
|
533
543
|
}
|
|
544
|
+
type FullLoginOptions = LoginOptions & {
|
|
545
|
+
forceLogin?: boolean;
|
|
546
|
+
};
|
|
534
547
|
/**
|
|
535
548
|
* Log into Braintrust. This will prompt you for your API token, which you can find at
|
|
536
549
|
* https://www.braintrust.dev/app/token. This method is called automatically by `init()`.
|
|
@@ -542,17 +555,17 @@ export interface LoginOptions {
|
|
|
542
555
|
* @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
|
|
543
556
|
* @param options.forceLogin Login again, even if you have already logged in (by default, this function will exit quickly if you have already logged in)
|
|
544
557
|
*/
|
|
545
|
-
|
|
558
|
+
declare function login(options?: LoginOptions & {
|
|
546
559
|
forceLogin?: boolean;
|
|
547
|
-
}): Promise<BraintrustState
|
|
548
|
-
|
|
560
|
+
}): Promise<BraintrustState>;
|
|
561
|
+
declare function loginToState(options?: LoginOptions): Promise<BraintrustState>;
|
|
549
562
|
/**
|
|
550
563
|
* Log a single event to the current experiment. The event will be batched and uploaded behind the scenes.
|
|
551
564
|
*
|
|
552
565
|
* @param event The event to log. See `Experiment.log` for full details.
|
|
553
566
|
* @returns The `id` of the logged event.
|
|
554
567
|
*/
|
|
555
|
-
|
|
568
|
+
declare function log(event: ExperimentLogFullArgs): string;
|
|
556
569
|
/**
|
|
557
570
|
* Summarize the current experiment, including the scores (compared to the closest reference experiment) and metadata.
|
|
558
571
|
*
|
|
@@ -561,7 +574,7 @@ export declare function log(event: ExperimentLogFullArgs): string;
|
|
|
561
574
|
* @param options.comparisonExperimentId The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
|
|
562
575
|
* @returns A summary of the experiment, including the scores (compared to the closest reference experiment) and metadata.
|
|
563
576
|
*/
|
|
564
|
-
|
|
577
|
+
declare function summarize(options?: {
|
|
565
578
|
readonly summarizeScores?: boolean;
|
|
566
579
|
readonly comparisonExperimentId?: string;
|
|
567
580
|
}): Promise<ExperimentSummary>;
|
|
@@ -571,21 +584,21 @@ type OptionalStateArg = {
|
|
|
571
584
|
/**
|
|
572
585
|
* Returns the currently-active experiment (set by `braintrust.init`). Returns undefined if no current experiment has been set.
|
|
573
586
|
*/
|
|
574
|
-
|
|
587
|
+
declare function currentExperiment(options?: OptionalStateArg): Experiment | undefined;
|
|
575
588
|
/**
|
|
576
589
|
* Returns the currently-active logger (set by `braintrust.initLogger`). Returns undefined if no current logger has been set.
|
|
577
590
|
*/
|
|
578
|
-
|
|
591
|
+
declare function currentLogger<IsAsyncFlush extends boolean>(options?: AsyncFlushArg<IsAsyncFlush> & OptionalStateArg): Logger<IsAsyncFlush> | undefined;
|
|
579
592
|
/**
|
|
580
593
|
* Return the currently-active span for logging (set by one of the `traced` methods). If there is no active span, returns a no-op span object, which supports the same interface as spans but does no logging.
|
|
581
594
|
*
|
|
582
595
|
* See `Span` for full details.
|
|
583
596
|
*/
|
|
584
|
-
|
|
597
|
+
declare function currentSpan(options?: OptionalStateArg): Span;
|
|
585
598
|
/**
|
|
586
599
|
* Mainly for internal use. Return the parent object for starting a span in a global context.
|
|
587
600
|
*/
|
|
588
|
-
|
|
601
|
+
declare function getSpanParentObject<IsAsyncFlush extends boolean>(options?: AsyncFlushArg<IsAsyncFlush> & OptionalStateArg): Span | Experiment | Logger<IsAsyncFlush>;
|
|
589
602
|
/**
|
|
590
603
|
* Toplevel function for starting a span. It checks the following (in precedence order):
|
|
591
604
|
* * Currently-active span
|
|
@@ -596,7 +609,7 @@ export declare function getSpanParentObject<IsAsyncFlush extends boolean>(option
|
|
|
596
609
|
*
|
|
597
610
|
* See `Span.traced` for full details.
|
|
598
611
|
*/
|
|
599
|
-
|
|
612
|
+
declare function traced<IsAsyncFlush extends boolean = false, R = void>(callback: (span: Span) => R, args?: StartSpanArgs & SetCurrentArg & AsyncFlushArg<IsAsyncFlush>): PromiseUnless<IsAsyncFlush, R>;
|
|
600
613
|
/**
|
|
601
614
|
* Wrap a function with `traced`, using the arguments as `input` and return value as `output`.
|
|
602
615
|
* Any functions wrapped this way will automatically be traced, similar to the `@traced` decorator
|
|
@@ -619,12 +632,12 @@ export declare function traced<IsAsyncFlush extends boolean = false, R = void>(c
|
|
|
619
632
|
* @param args Span-level arguments (e.g. a custom name or type) to pass to `traced`.
|
|
620
633
|
* @returns The wrapped function.
|
|
621
634
|
*/
|
|
622
|
-
|
|
635
|
+
declare function wrapTraced<F extends (...args: any[]) => any, IsAsyncFlush extends boolean = false>(fn: F, args?: StartSpanArgs & SetCurrentArg & AsyncFlushArg<IsAsyncFlush>): IsAsyncFlush extends false ? (...args: Parameters<F>) => Promise<Awaited<ReturnType<F>>> : F;
|
|
623
636
|
/**
|
|
624
637
|
* A synonym for `wrapTraced`. If you're porting from systems that use `traceable`, you can use this to
|
|
625
638
|
* make your codebase more consistent.
|
|
626
639
|
*/
|
|
627
|
-
|
|
640
|
+
declare const traceable: typeof wrapTraced;
|
|
628
641
|
/**
|
|
629
642
|
* Lower-level alternative to `traced`. This allows you to start a span yourself, and can be useful in situations
|
|
630
643
|
* where you cannot use callbacks. However, spans started with `startSpan` will not be marked as the "current span",
|
|
@@ -632,19 +645,19 @@ export declare const traceable: typeof wrapTraced;
|
|
|
632
645
|
*
|
|
633
646
|
* See `traced` for full details.
|
|
634
647
|
*/
|
|
635
|
-
|
|
648
|
+
declare function startSpan<IsAsyncFlush extends boolean = false>(args?: StartSpanArgs & AsyncFlushArg<IsAsyncFlush> & OptionalStateArg): Span;
|
|
636
649
|
/**
|
|
637
650
|
* Flush any pending rows to the server.
|
|
638
651
|
*/
|
|
639
|
-
|
|
652
|
+
declare function flush(options?: OptionalStateArg): Promise<void>;
|
|
640
653
|
/**
|
|
641
654
|
* Set the fetch implementation to use for requests. You can specify it here,
|
|
642
655
|
* or when you call `login`.
|
|
643
656
|
*
|
|
644
657
|
* @param fetch The fetch implementation to use.
|
|
645
658
|
*/
|
|
646
|
-
|
|
647
|
-
|
|
659
|
+
declare function setFetch(fetch: typeof globalThis.fetch): void;
|
|
660
|
+
type WithTransactionId<R> = R & {
|
|
648
661
|
[TRANSACTION_ID_FIELD]: TransactionId;
|
|
649
662
|
};
|
|
650
663
|
declare class ObjectFetcher<RecordType> implements AsyncIterable<WithTransactionId<RecordType>> {
|
|
@@ -661,9 +674,9 @@ declare class ObjectFetcher<RecordType> implements AsyncIterable<WithTransaction
|
|
|
661
674
|
clearCache(): void;
|
|
662
675
|
version(): Promise<string | undefined>;
|
|
663
676
|
}
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
677
|
+
type BaseMetadata = Record<string, unknown> | void;
|
|
678
|
+
type DefaultMetadataType = void;
|
|
679
|
+
type EvalCase<Input, Expected, Metadata> = {
|
|
667
680
|
input: Input;
|
|
668
681
|
tags?: string[];
|
|
669
682
|
} & (Expected extends void ? {} : {
|
|
@@ -683,7 +696,7 @@ export type EvalCase<Input, Expected, Metadata> = {
|
|
|
683
696
|
*
|
|
684
697
|
* You should not create `Experiment` objects directly. Instead, use the `braintrust.init()` method.
|
|
685
698
|
*/
|
|
686
|
-
|
|
699
|
+
declare class Experiment extends ObjectFetcher<ExperimentEvent> implements Exportable {
|
|
687
700
|
private readonly lazyMetadata;
|
|
688
701
|
readonly dataset?: AnyDataset;
|
|
689
702
|
private lastStartTime;
|
|
@@ -776,7 +789,7 @@ export declare class Experiment extends ObjectFetcher<ExperimentEvent> {
|
|
|
776
789
|
/**
|
|
777
790
|
* A read-only view of an experiment, initialized by passing `open: true` to `init()`.
|
|
778
791
|
*/
|
|
779
|
-
|
|
792
|
+
declare class ReadonlyExperiment extends ObjectFetcher<ExperimentEvent> {
|
|
780
793
|
private state;
|
|
781
794
|
private readonly lazyMetadata;
|
|
782
795
|
constructor(state: BraintrustState, lazyMetadata: LazyValue<ProjectExperimentMetadata>);
|
|
@@ -785,13 +798,13 @@ export declare class ReadonlyExperiment extends ObjectFetcher<ExperimentEvent> {
|
|
|
785
798
|
protected getState(): Promise<BraintrustState>;
|
|
786
799
|
asDataset<Input, Expected>(): AsyncGenerator<EvalCase<Input, Expected, void>>;
|
|
787
800
|
}
|
|
788
|
-
|
|
801
|
+
declare function newId(): string;
|
|
789
802
|
/**
|
|
790
803
|
* Primary implementation of the `Span` interface. See the `Span` interface for full details on each method.
|
|
791
804
|
*
|
|
792
805
|
* We suggest using one of the various `traced` methods, instead of creating Spans directly. See `Span.startSpan` for full details.
|
|
793
806
|
*/
|
|
794
|
-
|
|
807
|
+
declare class SpanImpl implements Span {
|
|
795
808
|
private state;
|
|
796
809
|
private isMerge;
|
|
797
810
|
private loggedEndTime;
|
|
@@ -830,7 +843,7 @@ export declare class SpanImpl implements Span {
|
|
|
830
843
|
*
|
|
831
844
|
* You should not create `Dataset` objects directly. Instead, use the `braintrust.initDataset()` method.
|
|
832
845
|
*/
|
|
833
|
-
|
|
846
|
+
declare class Dataset<IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY_DATASET> extends ObjectFetcher<DatasetRecord<IsLegacyDataset>> {
|
|
834
847
|
private state;
|
|
835
848
|
private readonly lazyMetadata;
|
|
836
849
|
constructor(state: BraintrustState, lazyMetadata: LazyValue<ProjectDatasetMetadata>, pinnedVersion?: string, legacy?: IsLegacyDataset);
|
|
@@ -881,17 +894,17 @@ export declare class Dataset<IsLegacyDataset extends boolean = typeof DEFAULT_IS
|
|
|
881
894
|
*/
|
|
882
895
|
close(): Promise<string>;
|
|
883
896
|
}
|
|
884
|
-
|
|
897
|
+
type CompiledPromptParams = Omit<NonNullable<PromptData["options"]>["params"], "use_cache"> & {
|
|
885
898
|
model: NonNullable<NonNullable<PromptData["options"]>["model"]>;
|
|
886
899
|
};
|
|
887
|
-
|
|
900
|
+
type ChatPrompt = {
|
|
888
901
|
messages: OpenAIMessage[];
|
|
889
902
|
tools?: Tools;
|
|
890
903
|
};
|
|
891
|
-
|
|
904
|
+
type CompletionPrompt = {
|
|
892
905
|
prompt: string;
|
|
893
906
|
};
|
|
894
|
-
|
|
907
|
+
type CompiledPrompt<Flavor extends "chat" | "completion"> = CompiledPromptParams & {
|
|
895
908
|
span_info?: {
|
|
896
909
|
name?: string;
|
|
897
910
|
spanAttributes?: Record<any, any>;
|
|
@@ -905,12 +918,12 @@ export type CompiledPrompt<Flavor extends "chat" | "completion"> = CompiledPromp
|
|
|
905
918
|
};
|
|
906
919
|
};
|
|
907
920
|
} & (Flavor extends "chat" ? ChatPrompt : Flavor extends "completion" ? CompletionPrompt : {});
|
|
908
|
-
|
|
909
|
-
|
|
921
|
+
type DefaultPromptArgs = Partial<CompiledPromptParams & AnyModelParam & ChatPrompt & CompletionPrompt>;
|
|
922
|
+
declare class Prompt {
|
|
910
923
|
private metadata;
|
|
911
924
|
private defaults;
|
|
912
925
|
private noTrace;
|
|
913
|
-
constructor(metadata: Omit<
|
|
926
|
+
constructor(metadata: Omit<Prompt$1, "log_id"> | PromptSessionEvent, defaults: DefaultPromptArgs, noTrace: boolean);
|
|
914
927
|
get id(): string;
|
|
915
928
|
get projectId(): string;
|
|
916
929
|
get name(): string;
|
|
@@ -930,7 +943,7 @@ export declare class Prompt {
|
|
|
930
943
|
}): CompiledPrompt<Flavor>;
|
|
931
944
|
private runBuild;
|
|
932
945
|
}
|
|
933
|
-
|
|
946
|
+
type AnyDataset = Dataset<boolean>;
|
|
934
947
|
/**
|
|
935
948
|
* Summary of a score's performance.
|
|
936
949
|
* @property name Name of the score.
|
|
@@ -939,7 +952,7 @@ export type AnyDataset = Dataset<boolean>;
|
|
|
939
952
|
* @property improvements Number of improvements in the score.
|
|
940
953
|
* @property regressions Number of regressions in the score.
|
|
941
954
|
*/
|
|
942
|
-
|
|
955
|
+
interface ScoreSummary {
|
|
943
956
|
name: string;
|
|
944
957
|
score: number;
|
|
945
958
|
diff?: number;
|
|
@@ -955,7 +968,7 @@ export interface ScoreSummary {
|
|
|
955
968
|
* @property improvements Number of improvements in the metric.
|
|
956
969
|
* @property regressions Number of regressions in the metric.
|
|
957
970
|
*/
|
|
958
|
-
|
|
971
|
+
interface MetricSummary {
|
|
959
972
|
name: string;
|
|
960
973
|
metric: number;
|
|
961
974
|
unit: string;
|
|
@@ -973,7 +986,7 @@ export interface MetricSummary {
|
|
|
973
986
|
* @property comparisonExperimentName The experiment scores are baselined against.
|
|
974
987
|
* @property scores Summary of the experiment's scores.
|
|
975
988
|
*/
|
|
976
|
-
|
|
989
|
+
interface ExperimentSummary {
|
|
977
990
|
projectName: string;
|
|
978
991
|
experimentName: string;
|
|
979
992
|
projectId?: string;
|
|
@@ -990,7 +1003,7 @@ export interface ExperimentSummary {
|
|
|
990
1003
|
* @property newRecords New or updated records added in this session.
|
|
991
1004
|
* @property totalRecords Total records in the dataset.
|
|
992
1005
|
*/
|
|
993
|
-
|
|
1006
|
+
interface DataSummary {
|
|
994
1007
|
newRecords: number;
|
|
995
1008
|
totalRecords: number;
|
|
996
1009
|
}
|
|
@@ -1003,11 +1016,74 @@ export interface DataSummary {
|
|
|
1003
1016
|
* @property datasetUrl URL to the experiment's page in the Braintrust app.
|
|
1004
1017
|
* @property dataSummary Summary of the dataset's data.
|
|
1005
1018
|
*/
|
|
1006
|
-
|
|
1019
|
+
interface DatasetSummary {
|
|
1007
1020
|
projectName: string;
|
|
1008
1021
|
datasetName: string;
|
|
1009
1022
|
projectUrl: string;
|
|
1010
1023
|
datasetUrl: string;
|
|
1011
1024
|
dataSummary: DataSummary;
|
|
1012
1025
|
}
|
|
1013
|
-
|
|
1026
|
+
|
|
1027
|
+
type BraintrustStreamChunk = {
|
|
1028
|
+
type: "text_delta";
|
|
1029
|
+
data: string;
|
|
1030
|
+
} | {
|
|
1031
|
+
type: "json_delta";
|
|
1032
|
+
data: string;
|
|
1033
|
+
};
|
|
1034
|
+
declare class BraintrustStream {
|
|
1035
|
+
private stream;
|
|
1036
|
+
constructor(baseStream: ReadableStream<Uint8Array>);
|
|
1037
|
+
constructor(stream: ReadableStream<string>);
|
|
1038
|
+
constructor(stream: ReadableStream<BraintrustStreamChunk>);
|
|
1039
|
+
copy(): BraintrustStream;
|
|
1040
|
+
toReadableStream(): ReadableStream<BraintrustStreamChunk>;
|
|
1041
|
+
}
|
|
1042
|
+
declare function createFinalValuePassThroughStream<T extends BraintrustStreamChunk | string | Uint8Array>(onFinal: (result: unknown) => void): TransformStream<T, BraintrustStreamChunk>;
|
|
1043
|
+
declare function devNullWritableStream(): WritableStream;
|
|
1044
|
+
|
|
1045
|
+
type InvokeReturn<Stream extends boolean, Return> = Stream extends true ? BraintrustStream : Return;
|
|
1046
|
+
type InvokeFunctionArgs<Arg, Return, Stream extends boolean = false> = FunctionId & FullLoginOptions & {
|
|
1047
|
+
arg: Arg;
|
|
1048
|
+
parent?: Exportable | string;
|
|
1049
|
+
state?: BraintrustState;
|
|
1050
|
+
stream?: Stream;
|
|
1051
|
+
schema?: z.ZodSchema<Return>;
|
|
1052
|
+
};
|
|
1053
|
+
declare function invoke<Arg, Return, Stream extends boolean = false>(args: InvokeFunctionArgs<Arg, Return, Stream>): Promise<InvokeReturn<Stream, Return>>;
|
|
1054
|
+
|
|
1055
|
+
interface BetaLike {
|
|
1056
|
+
chat: {
|
|
1057
|
+
completions: {
|
|
1058
|
+
stream: any;
|
|
1059
|
+
};
|
|
1060
|
+
};
|
|
1061
|
+
embeddings: any;
|
|
1062
|
+
}
|
|
1063
|
+
interface ChatLike {
|
|
1064
|
+
completions: any;
|
|
1065
|
+
}
|
|
1066
|
+
interface OpenAILike {
|
|
1067
|
+
chat: ChatLike;
|
|
1068
|
+
embeddings: any;
|
|
1069
|
+
beta?: BetaLike;
|
|
1070
|
+
}
|
|
1071
|
+
declare global {
|
|
1072
|
+
var __inherited_braintrust_wrap_openai: ((openai: any) => any) | undefined;
|
|
1073
|
+
}
|
|
1074
|
+
/**
|
|
1075
|
+
* Wrap an `OpenAI` object (created with `new OpenAI(...)`) to add tracing. If Braintrust is
|
|
1076
|
+
* not configured, this is a no-op
|
|
1077
|
+
*
|
|
1078
|
+
* Currently, this only supports the `v4` API.
|
|
1079
|
+
*
|
|
1080
|
+
* @param openai
|
|
1081
|
+
* @returns The wrapped `OpenAI` object.
|
|
1082
|
+
*/
|
|
1083
|
+
declare function wrapOpenAI<T extends object>(openai: T): T;
|
|
1084
|
+
declare function wrapOpenAIv4<T extends OpenAILike>(openai: T): T;
|
|
1085
|
+
declare const LEGACY_CACHED_HEADER = "x-cached";
|
|
1086
|
+
declare const X_CACHED_HEADER = "x-bt-cached";
|
|
1087
|
+
declare function parseCachedHeader(value: string | null | undefined): number | undefined;
|
|
1088
|
+
|
|
1089
|
+
export { type AnyDataset, type BaseMetadata, BraintrustState, BraintrustStream, type BraintrustStreamChunk, type ChatPrompt, type CompiledPrompt, type CompiledPromptParams, type CompletionPrompt, type DataSummary, Dataset, type DatasetSummary, type DefaultMetadataType, type DefaultPromptArgs, type EndSpanArgs, type EvalCase, Experiment, type ExperimentSummary, type Exportable, type FullInitOptions, type FullLoginOptions, type InitOptions, type InvokeFunctionArgs, type InvokeReturn, LEGACY_CACHED_HEADER, type LogOptions, Logger, type LoginOptions, type MetricSummary, NOOP_SPAN, NoopSpan, type ObjectMetadata, type PromiseUnless, Prompt, ReadonlyExperiment, type ScoreSummary, type SerializedBraintrustState, type SetCurrentArg, type Span, SpanImpl, type StartSpanArgs, type WithTransactionId, X_CACHED_HEADER, _internalGetGlobalState, _internalSetInitialState, createFinalValuePassThroughStream, currentExperiment, currentLogger, currentSpan, devNullWritableStream, flush, getSpanParentObject, init, initDataset, initExperiment, initLogger, invoke, loadPrompt, log, login, loginToState, newId, parseCachedHeader, setFetch, startSpan, summarize, traceable, traced, withDataset, withExperiment, withLogger, wrapOpenAI, wrapOpenAIv4, wrapTraced };
|