braintrust 0.0.139 → 0.0.141-dev
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +42 -0
- package/.turbo/turbo-watch.log +1843 -0
- package/LICENSE +201 -0
- package/braintrust-0.0.141.tgz +0 -0
- package/dist/{logger.d.ts → browser.d.mts} +194 -90
- package/dist/browser.d.ts +1089 -2
- package/dist/browser.js +444 -9231
- package/dist/browser.mjs +3203 -0
- package/dist/cli.js +3831 -27083
- package/dist/index.d.mts +1250 -0
- package/dist/index.d.ts +1236 -41
- package/dist/index.js +1769 -18971
- package/dist/index.mjs +5210 -0
- package/package.json +40 -23
- package/tsup.config.ts +23 -0
- package/turbo.json +8 -0
- package/vitest.config.js +6 -0
- package/dist/browser-config.d.ts +0 -6
- package/dist/cli.d.ts +0 -35
- package/dist/framework.d.ts +0 -180
- package/dist/framework.test.d.ts +0 -1
- package/dist/functions.d.ts +0 -15
- package/dist/gitutil.d.ts +0 -17
- package/dist/isomorph.d.ts +0 -27
- package/dist/jest/nodeModulesPaths.d.ts +0 -15
- package/dist/jest/tryRealpath.d.ts +0 -7
- package/dist/node.d.ts +0 -1
- package/dist/progress.d.ts +0 -18
- package/dist/stackutil.d.ts +0 -8
- package/dist/tsconfig.tsbuildinfo +0 -1
- package/dist/util.d.ts +0 -11
- package/dist/wrappers/ai-sdk.d.ts +0 -8
- package/dist/wrappers/oai.d.ts +0 -34
- package/jest.config.js +0 -5
|
@@ -1,14 +1,28 @@
|
|
|
1
|
+
import { GitMetadataSettings, LogFeedbackFullArgs, BackgroundLogEvent, ExperimentEvent, ExperimentLogFullArgs, ExperimentLogPartialArgs, IdField, SpanType, RepoInfo, DEFAULT_IS_LEGACY_DATASET, TRANSACTION_ID_FIELD, TransactionId, SpanObjectTypeV2, DatasetRecord } from '@braintrust/core';
|
|
2
|
+
import { PromptData, OpenAIMessage, Tools, AnyModelParam, Prompt as Prompt$1, PromptSessionEvent, FunctionId } from '@braintrust/core/typespecs';
|
|
3
|
+
import { z } from 'zod';
|
|
4
|
+
|
|
5
|
+
interface IsoAsyncLocalStorage<T> {
|
|
6
|
+
enterWith(store: T): void;
|
|
7
|
+
run<R>(store: T | undefined, callback: () => R): R;
|
|
8
|
+
getStore(): T | undefined;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
declare class LazyValue<T> {
|
|
12
|
+
private callable;
|
|
13
|
+
private value;
|
|
14
|
+
constructor(callable: () => Promise<T>);
|
|
15
|
+
get(): Promise<T>;
|
|
16
|
+
get hasComputed(): boolean;
|
|
17
|
+
}
|
|
18
|
+
|
|
1
19
|
/// <reference lib="dom" />
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
import { IsoAsyncLocalStorage } from "./isomorph";
|
|
5
|
-
import { LazyValue } from "./util";
|
|
6
|
-
import { z } from "zod";
|
|
7
|
-
export type SetCurrentArg = {
|
|
20
|
+
|
|
21
|
+
type SetCurrentArg = {
|
|
8
22
|
setCurrent?: boolean;
|
|
9
23
|
};
|
|
10
24
|
type StartSpanEventArgs = ExperimentLogPartialArgs & Partial<IdField>;
|
|
11
|
-
|
|
25
|
+
type StartSpanArgs = {
|
|
12
26
|
name?: string;
|
|
13
27
|
type?: SpanType;
|
|
14
28
|
spanAttributes?: Record<any, any>;
|
|
@@ -16,9 +30,15 @@ export type StartSpanArgs = {
|
|
|
16
30
|
parent?: string;
|
|
17
31
|
event?: StartSpanEventArgs;
|
|
18
32
|
};
|
|
19
|
-
|
|
33
|
+
type EndSpanArgs = {
|
|
20
34
|
endTime?: number;
|
|
21
35
|
};
|
|
36
|
+
interface Exportable {
|
|
37
|
+
/**
|
|
38
|
+
* Return a serialized representation of the object that can be used to start subspans in other places. See `Span.traced` for more details.
|
|
39
|
+
*/
|
|
40
|
+
export(): Promise<string>;
|
|
41
|
+
}
|
|
22
42
|
/**
|
|
23
43
|
* A Span encapsulates logged data and metrics for a unit of work. This interface is shared by all span implementations.
|
|
24
44
|
*
|
|
@@ -26,7 +46,7 @@ export type EndSpanArgs = {
|
|
|
26
46
|
*
|
|
27
47
|
* See `Span.traced` for full details.
|
|
28
48
|
*/
|
|
29
|
-
|
|
49
|
+
interface Span extends Exportable {
|
|
30
50
|
/**
|
|
31
51
|
* Row ID of the span.
|
|
32
52
|
*/
|
|
@@ -78,10 +98,6 @@ export interface Span {
|
|
|
78
98
|
* @returns The end time logged to the span metrics.
|
|
79
99
|
*/
|
|
80
100
|
end(args?: EndSpanArgs): number;
|
|
81
|
-
/**
|
|
82
|
-
* Return a serialized representation of the span that can be used to start subspans in other places. See `Span.traced` for more details.
|
|
83
|
-
*/
|
|
84
|
-
export(): Promise<string>;
|
|
85
101
|
/**
|
|
86
102
|
* Flush any pending rows to the server.
|
|
87
103
|
*/
|
|
@@ -99,7 +115,7 @@ export interface Span {
|
|
|
99
115
|
/**
|
|
100
116
|
* A fake implementation of the Span API which does nothing. This can be used as the default span.
|
|
101
117
|
*/
|
|
102
|
-
|
|
118
|
+
declare class NoopSpan implements Span {
|
|
103
119
|
id: string;
|
|
104
120
|
kind: "span";
|
|
105
121
|
constructor();
|
|
@@ -113,7 +129,7 @@ export declare class NoopSpan implements Span {
|
|
|
113
129
|
close(args?: EndSpanArgs): number;
|
|
114
130
|
setAttributes(_args: Omit<StartSpanArgs, "event">): void;
|
|
115
131
|
}
|
|
116
|
-
|
|
132
|
+
declare const NOOP_SPAN: NoopSpan;
|
|
117
133
|
declare global {
|
|
118
134
|
var __inherited_braintrust_state: BraintrustState;
|
|
119
135
|
}
|
|
@@ -122,6 +138,7 @@ declare const loginSchema: z.ZodObject<{
|
|
|
122
138
|
appPublicUrl: z.ZodString;
|
|
123
139
|
orgName: z.ZodString;
|
|
124
140
|
logUrl: z.ZodString;
|
|
141
|
+
proxyUrl: z.ZodString;
|
|
125
142
|
loginToken: z.ZodString;
|
|
126
143
|
orgId: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
127
144
|
gitMetadataSettings: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
@@ -139,6 +156,7 @@ declare const loginSchema: z.ZodObject<{
|
|
|
139
156
|
appPublicUrl: string;
|
|
140
157
|
orgName: string;
|
|
141
158
|
logUrl: string;
|
|
159
|
+
proxyUrl: string;
|
|
142
160
|
loginToken: string;
|
|
143
161
|
orgId?: string | null | undefined;
|
|
144
162
|
gitMetadataSettings?: {
|
|
@@ -150,6 +168,7 @@ declare const loginSchema: z.ZodObject<{
|
|
|
150
168
|
appPublicUrl: string;
|
|
151
169
|
orgName: string;
|
|
152
170
|
logUrl: string;
|
|
171
|
+
proxyUrl: string;
|
|
153
172
|
loginToken: string;
|
|
154
173
|
orgId?: string | null | undefined;
|
|
155
174
|
gitMetadataSettings?: {
|
|
@@ -157,8 +176,8 @@ declare const loginSchema: z.ZodObject<{
|
|
|
157
176
|
fields?: ("dirty" | "tag" | "commit" | "branch" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
|
|
158
177
|
} | null | undefined;
|
|
159
178
|
}>;
|
|
160
|
-
|
|
161
|
-
|
|
179
|
+
type SerializedBraintrustState = z.infer<typeof loginSchema>;
|
|
180
|
+
declare class BraintrustState {
|
|
162
181
|
private loginParams;
|
|
163
182
|
id: string;
|
|
164
183
|
currentExperiment: Experiment | undefined;
|
|
@@ -171,11 +190,13 @@ export declare class BraintrustState {
|
|
|
171
190
|
orgId: string | null;
|
|
172
191
|
orgName: string | null;
|
|
173
192
|
logUrl: string | null;
|
|
193
|
+
proxyUrl: string | null;
|
|
174
194
|
loggedIn: boolean;
|
|
175
195
|
gitMetadataSettings?: GitMetadataSettings;
|
|
176
196
|
fetch: typeof globalThis.fetch;
|
|
177
197
|
private _apiConn;
|
|
178
198
|
private _logConn;
|
|
199
|
+
private _proxyConn;
|
|
179
200
|
constructor(loginParams: LoginOptions);
|
|
180
201
|
resetLoginInfo(): void;
|
|
181
202
|
copyLoginInfo(other: BraintrustState): void;
|
|
@@ -187,11 +208,12 @@ export declare class BraintrustState {
|
|
|
187
208
|
}): Promise<void>;
|
|
188
209
|
apiConn(): HTTPConnection;
|
|
189
210
|
logConn(): HTTPConnection;
|
|
211
|
+
proxyConn(): HTTPConnection;
|
|
190
212
|
bgLogger(): BackgroundLogger;
|
|
191
213
|
loginReplaceLogConn(logConn: HTTPConnection): void;
|
|
192
214
|
}
|
|
193
|
-
|
|
194
|
-
|
|
215
|
+
declare function _internalSetInitialState(): void;
|
|
216
|
+
declare const _internalGetGlobalState: () => BraintrustState;
|
|
195
217
|
declare class HTTPConnection {
|
|
196
218
|
base_url: string;
|
|
197
219
|
token: string | null;
|
|
@@ -209,7 +231,7 @@ declare class HTTPConnection {
|
|
|
209
231
|
get_json(object_type: string, args?: Record<string, string | undefined> | undefined, retries?: number): Promise<any>;
|
|
210
232
|
post_json(object_type: string, args?: Record<string, unknown> | string | undefined): Promise<any>;
|
|
211
233
|
}
|
|
212
|
-
|
|
234
|
+
interface ObjectMetadata {
|
|
213
235
|
id: string;
|
|
214
236
|
name: string;
|
|
215
237
|
fullInfo: Record<string, unknown>;
|
|
@@ -226,16 +248,16 @@ interface OrgProjectMetadata {
|
|
|
226
248
|
org_id: string;
|
|
227
249
|
project: ObjectMetadata;
|
|
228
250
|
}
|
|
229
|
-
|
|
251
|
+
interface LogOptions<IsAsyncFlush> {
|
|
230
252
|
asyncFlush?: IsAsyncFlush;
|
|
231
253
|
computeMetadataArgs?: Record<string, any>;
|
|
232
254
|
}
|
|
233
|
-
|
|
255
|
+
type PromiseUnless<B, R> = B extends true ? R : Promise<Awaited<R>>;
|
|
234
256
|
interface ParentSpanIds {
|
|
235
257
|
spanId: string;
|
|
236
258
|
rootSpanId: string;
|
|
237
259
|
}
|
|
238
|
-
|
|
260
|
+
declare class Logger<IsAsyncFlush extends boolean> implements Exportable {
|
|
239
261
|
private state;
|
|
240
262
|
private lazyMetadata;
|
|
241
263
|
private _asyncFlush;
|
|
@@ -249,6 +271,7 @@ export declare class Logger<IsAsyncFlush extends boolean> {
|
|
|
249
271
|
get project(): Promise<ObjectMetadata>;
|
|
250
272
|
get id(): Promise<string>;
|
|
251
273
|
private parentObjectType;
|
|
274
|
+
private triggerWaitUntilFlush;
|
|
252
275
|
/**
|
|
253
276
|
* Log a single event. The event will be batched and uploaded behind the scenes if `logOptions.asyncFlush` is true.
|
|
254
277
|
*
|
|
@@ -332,16 +355,13 @@ declare class BackgroundLogger {
|
|
|
332
355
|
type InitOpenOption<IsOpen extends boolean> = {
|
|
333
356
|
open?: IsOpen;
|
|
334
357
|
};
|
|
335
|
-
|
|
358
|
+
type InitOptions<IsOpen extends boolean> = FullLoginOptions & {
|
|
336
359
|
experiment?: string;
|
|
337
360
|
description?: string;
|
|
338
361
|
dataset?: AnyDataset;
|
|
339
362
|
update?: boolean;
|
|
340
363
|
baseExperiment?: string;
|
|
341
364
|
isPublic?: boolean;
|
|
342
|
-
appUrl?: string;
|
|
343
|
-
apiKey?: string;
|
|
344
|
-
orgName?: string;
|
|
345
365
|
metadata?: Record<string, unknown>;
|
|
346
366
|
gitMetadataSettings?: GitMetadataSettings;
|
|
347
367
|
projectId?: string;
|
|
@@ -350,7 +370,7 @@ export type InitOptions<IsOpen extends boolean> = {
|
|
|
350
370
|
setCurrent?: boolean;
|
|
351
371
|
state?: BraintrustState;
|
|
352
372
|
} & InitOpenOption<IsOpen>;
|
|
353
|
-
|
|
373
|
+
type FullInitOptions<IsOpen extends boolean> = {
|
|
354
374
|
project?: string;
|
|
355
375
|
} & InitOptions<IsOpen>;
|
|
356
376
|
type InitializedExperiment<IsOpen extends boolean | undefined> = IsOpen extends true ? ReadonlyExperiment : Experiment;
|
|
@@ -377,38 +397,35 @@ type InitializedExperiment<IsOpen extends boolean | undefined> = IsOpen extends
|
|
|
377
397
|
* @param options.repoInfo (Optional) Explicitly specify the git metadata for this experiment. This takes precedence over `gitMetadataSettings` if specified.
|
|
378
398
|
* @returns The newly created Experiment.
|
|
379
399
|
*/
|
|
380
|
-
|
|
400
|
+
declare function init<IsOpen extends boolean = false>(options: Readonly<FullInitOptions<IsOpen>>): InitializedExperiment<IsOpen>;
|
|
381
401
|
/**
|
|
382
402
|
* Legacy form of `init` which accepts the project name as the first parameter,
|
|
383
403
|
* separately from the remaining options. See `init(options)` for full details.
|
|
384
404
|
*/
|
|
385
|
-
|
|
405
|
+
declare function init<IsOpen extends boolean = false>(project: string, options?: Readonly<InitOptions<IsOpen>>): InitializedExperiment<IsOpen>;
|
|
386
406
|
/**
|
|
387
407
|
* Alias for init(options).
|
|
388
408
|
*/
|
|
389
|
-
|
|
409
|
+
declare function initExperiment<IsOpen extends boolean = false>(options: Readonly<InitOptions<IsOpen>>): InitializedExperiment<IsOpen>;
|
|
390
410
|
/**
|
|
391
411
|
* Alias for init(project, options).
|
|
392
412
|
*/
|
|
393
|
-
|
|
413
|
+
declare function initExperiment<IsOpen extends boolean = false>(project: string, options?: Readonly<InitOptions<IsOpen>>): InitializedExperiment<IsOpen>;
|
|
394
414
|
/**
|
|
395
415
|
* This function is deprecated. Use `init` instead.
|
|
396
416
|
*/
|
|
397
|
-
|
|
417
|
+
declare function withExperiment<R>(project: string, callback: (experiment: Experiment) => R, options?: Readonly<InitOptions<false> & SetCurrentArg>): R;
|
|
398
418
|
/**
|
|
399
419
|
* This function is deprecated. Use `initLogger` instead.
|
|
400
420
|
*/
|
|
401
|
-
|
|
421
|
+
declare function withLogger<IsAsyncFlush extends boolean = false, R = void>(callback: (logger: Logger<IsAsyncFlush>) => R, options?: Readonly<InitLoggerOptions<IsAsyncFlush> & SetCurrentArg>): R;
|
|
402
422
|
type UseOutputOption<IsLegacyDataset extends boolean> = {
|
|
403
423
|
useOutput?: IsLegacyDataset;
|
|
404
424
|
};
|
|
405
|
-
type InitDatasetOptions<IsLegacyDataset extends boolean> = {
|
|
425
|
+
type InitDatasetOptions<IsLegacyDataset extends boolean> = FullLoginOptions & {
|
|
406
426
|
dataset?: string;
|
|
407
427
|
description?: string;
|
|
408
428
|
version?: string;
|
|
409
|
-
appUrl?: string;
|
|
410
|
-
apiKey?: string;
|
|
411
|
-
orgName?: string;
|
|
412
429
|
projectId?: string;
|
|
413
430
|
state?: BraintrustState;
|
|
414
431
|
} & UseOutputOption<IsLegacyDataset>;
|
|
@@ -429,27 +446,23 @@ type FullInitDatasetOptions<IsLegacyDataset extends boolean> = {
|
|
|
429
446
|
* @param options.useOutput (Deprecated) If true, records will be fetched from this dataset in the legacy format, with the "expected" field renamed to "output". This option will be removed in a future version of Braintrust.
|
|
430
447
|
* @returns The newly created Dataset.
|
|
431
448
|
*/
|
|
432
|
-
|
|
449
|
+
declare function initDataset<IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY_DATASET>(options: Readonly<FullInitDatasetOptions<IsLegacyDataset>>): Dataset<IsLegacyDataset>;
|
|
433
450
|
/**
|
|
434
451
|
* Legacy form of `initDataset` which accepts the project name as the first
|
|
435
452
|
* parameter, separately from the remaining options. See
|
|
436
453
|
* `initDataset(options)` for full details.
|
|
437
454
|
*/
|
|
438
|
-
|
|
455
|
+
declare function initDataset<IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY_DATASET>(project: string, options?: Readonly<InitDatasetOptions<IsLegacyDataset>>): Dataset<IsLegacyDataset>;
|
|
439
456
|
/**
|
|
440
457
|
* This function is deprecated. Use `initDataset` instead.
|
|
441
458
|
*/
|
|
442
|
-
|
|
459
|
+
declare function withDataset<R, IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY_DATASET>(project: string, callback: (dataset: Dataset<IsLegacyDataset>) => R, options?: Readonly<InitDatasetOptions<IsLegacyDataset>>): R;
|
|
443
460
|
type AsyncFlushArg<IsAsyncFlush> = {
|
|
444
461
|
asyncFlush?: IsAsyncFlush;
|
|
445
462
|
};
|
|
446
|
-
type InitLoggerOptions<IsAsyncFlush> = {
|
|
463
|
+
type InitLoggerOptions<IsAsyncFlush> = FullLoginOptions & {
|
|
447
464
|
projectName?: string;
|
|
448
465
|
projectId?: string;
|
|
449
|
-
appUrl?: string;
|
|
450
|
-
apiKey?: string;
|
|
451
|
-
orgName?: string;
|
|
452
|
-
forceLogin?: boolean;
|
|
453
466
|
setCurrent?: boolean;
|
|
454
467
|
state?: BraintrustState;
|
|
455
468
|
} & AsyncFlushArg<IsAsyncFlush>;
|
|
@@ -468,19 +481,16 @@ type InitLoggerOptions<IsAsyncFlush> = {
|
|
|
468
481
|
* @param setCurrent If true (the default), set the global current-experiment to the newly-created one.
|
|
469
482
|
* @returns The newly created Logger.
|
|
470
483
|
*/
|
|
471
|
-
|
|
472
|
-
|
|
484
|
+
declare function initLogger<IsAsyncFlush extends boolean = false>(options?: Readonly<InitLoggerOptions<IsAsyncFlush>>): Logger<IsAsyncFlush>;
|
|
485
|
+
type LoadPromptOptions = FullLoginOptions & {
|
|
473
486
|
projectName?: string;
|
|
474
487
|
projectId?: string;
|
|
475
488
|
slug?: string;
|
|
476
489
|
version?: string;
|
|
477
490
|
defaults?: DefaultPromptArgs;
|
|
478
491
|
noTrace?: boolean;
|
|
479
|
-
appUrl?: string;
|
|
480
|
-
apiKey?: string;
|
|
481
|
-
orgName?: string;
|
|
482
492
|
state?: BraintrustState;
|
|
483
|
-
}
|
|
493
|
+
};
|
|
484
494
|
/**
|
|
485
495
|
* Load a prompt from the specified project.
|
|
486
496
|
*
|
|
@@ -507,11 +517,11 @@ interface LoadPromptOptions {
|
|
|
507
517
|
* });
|
|
508
518
|
* ```
|
|
509
519
|
*/
|
|
510
|
-
|
|
520
|
+
declare function loadPrompt({ projectName, projectId, slug, version, defaults, noTrace, appUrl, apiKey, orgName, fetch, forceLogin, state: stateArg, }: LoadPromptOptions): Promise<Prompt>;
|
|
511
521
|
/**
|
|
512
522
|
* Options for logging in to Braintrust.
|
|
513
523
|
*/
|
|
514
|
-
|
|
524
|
+
interface LoginOptions {
|
|
515
525
|
/**
|
|
516
526
|
* The URL of the Braintrust App. Defaults to https://www.braintrust.dev. You should not need
|
|
517
527
|
* to change this unless you are doing the "Full" deployment.
|
|
@@ -531,6 +541,9 @@ export interface LoginOptions {
|
|
|
531
541
|
*/
|
|
532
542
|
fetch?: typeof globalThis.fetch;
|
|
533
543
|
}
|
|
544
|
+
type FullLoginOptions = LoginOptions & {
|
|
545
|
+
forceLogin?: boolean;
|
|
546
|
+
};
|
|
534
547
|
/**
|
|
535
548
|
* Log into Braintrust. This will prompt you for your API token, which you can find at
|
|
536
549
|
* https://www.braintrust.dev/app/token. This method is called automatically by `init()`.
|
|
@@ -542,17 +555,17 @@ export interface LoginOptions {
|
|
|
542
555
|
* @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
|
|
543
556
|
* @param options.forceLogin Login again, even if you have already logged in (by default, this function will exit quickly if you have already logged in)
|
|
544
557
|
*/
|
|
545
|
-
|
|
558
|
+
declare function login(options?: LoginOptions & {
|
|
546
559
|
forceLogin?: boolean;
|
|
547
|
-
}): Promise<BraintrustState
|
|
548
|
-
|
|
560
|
+
}): Promise<BraintrustState>;
|
|
561
|
+
declare function loginToState(options?: LoginOptions): Promise<BraintrustState>;
|
|
549
562
|
/**
|
|
550
563
|
* Log a single event to the current experiment. The event will be batched and uploaded behind the scenes.
|
|
551
564
|
*
|
|
552
565
|
* @param event The event to log. See `Experiment.log` for full details.
|
|
553
566
|
* @returns The `id` of the logged event.
|
|
554
567
|
*/
|
|
555
|
-
|
|
568
|
+
declare function log(event: ExperimentLogFullArgs): string;
|
|
556
569
|
/**
|
|
557
570
|
* Summarize the current experiment, including the scores (compared to the closest reference experiment) and metadata.
|
|
558
571
|
*
|
|
@@ -561,7 +574,7 @@ export declare function log(event: ExperimentLogFullArgs): string;
|
|
|
561
574
|
* @param options.comparisonExperimentId The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
|
|
562
575
|
* @returns A summary of the experiment, including the scores (compared to the closest reference experiment) and metadata.
|
|
563
576
|
*/
|
|
564
|
-
|
|
577
|
+
declare function summarize(options?: {
|
|
565
578
|
readonly summarizeScores?: boolean;
|
|
566
579
|
readonly comparisonExperimentId?: string;
|
|
567
580
|
}): Promise<ExperimentSummary>;
|
|
@@ -571,21 +584,21 @@ type OptionalStateArg = {
|
|
|
571
584
|
/**
|
|
572
585
|
* Returns the currently-active experiment (set by `braintrust.init`). Returns undefined if no current experiment has been set.
|
|
573
586
|
*/
|
|
574
|
-
|
|
587
|
+
declare function currentExperiment(options?: OptionalStateArg): Experiment | undefined;
|
|
575
588
|
/**
|
|
576
589
|
* Returns the currently-active logger (set by `braintrust.initLogger`). Returns undefined if no current logger has been set.
|
|
577
590
|
*/
|
|
578
|
-
|
|
591
|
+
declare function currentLogger<IsAsyncFlush extends boolean>(options?: AsyncFlushArg<IsAsyncFlush> & OptionalStateArg): Logger<IsAsyncFlush> | undefined;
|
|
579
592
|
/**
|
|
580
593
|
* Return the currently-active span for logging (set by one of the `traced` methods). If there is no active span, returns a no-op span object, which supports the same interface as spans but does no logging.
|
|
581
594
|
*
|
|
582
595
|
* See `Span` for full details.
|
|
583
596
|
*/
|
|
584
|
-
|
|
597
|
+
declare function currentSpan(options?: OptionalStateArg): Span;
|
|
585
598
|
/**
|
|
586
599
|
* Mainly for internal use. Return the parent object for starting a span in a global context.
|
|
587
600
|
*/
|
|
588
|
-
|
|
601
|
+
declare function getSpanParentObject<IsAsyncFlush extends boolean>(options?: AsyncFlushArg<IsAsyncFlush> & OptionalStateArg): Span | Experiment | Logger<IsAsyncFlush>;
|
|
589
602
|
/**
|
|
590
603
|
* Toplevel function for starting a span. It checks the following (in precedence order):
|
|
591
604
|
* * Currently-active span
|
|
@@ -596,7 +609,35 @@ export declare function getSpanParentObject<IsAsyncFlush extends boolean>(option
|
|
|
596
609
|
*
|
|
597
610
|
* See `Span.traced` for full details.
|
|
598
611
|
*/
|
|
599
|
-
|
|
612
|
+
declare function traced<IsAsyncFlush extends boolean = false, R = void>(callback: (span: Span) => R, args?: StartSpanArgs & SetCurrentArg & AsyncFlushArg<IsAsyncFlush>): PromiseUnless<IsAsyncFlush, R>;
|
|
613
|
+
/**
|
|
614
|
+
* Wrap a function with `traced`, using the arguments as `input` and return value as `output`.
|
|
615
|
+
* Any functions wrapped this way will automatically be traced, similar to the `@traced` decorator
|
|
616
|
+
* in Python. If you want to correctly propagate the function's name and define it in one go, then
|
|
617
|
+
* you can do so like this:
|
|
618
|
+
*
|
|
619
|
+
* ```ts
|
|
620
|
+
* const myFunc = wrapTraced(async function myFunc(input) {
|
|
621
|
+
* const result = await client.chat.completions.create({
|
|
622
|
+
* model: "gpt-3.5-turbo",
|
|
623
|
+
* messages: [{ role: "user", content: input }],
|
|
624
|
+
* });
|
|
625
|
+
* return result.choices[0].message.content ?? "unknown";
|
|
626
|
+
* });
|
|
627
|
+
* ```
|
|
628
|
+
* Now, any calls to `myFunc` will be traced, and the input and output will be logged automatically.
|
|
629
|
+
* If tracing is inactive, i.e. there is no active logger or experiment, it's just a no-op.
|
|
630
|
+
*
|
|
631
|
+
* @param fn The function to wrap.
|
|
632
|
+
* @param args Span-level arguments (e.g. a custom name or type) to pass to `traced`.
|
|
633
|
+
* @returns The wrapped function.
|
|
634
|
+
*/
|
|
635
|
+
declare function wrapTraced<F extends (...args: any[]) => any, IsAsyncFlush extends boolean = false>(fn: F, args?: StartSpanArgs & SetCurrentArg & AsyncFlushArg<IsAsyncFlush>): IsAsyncFlush extends false ? (...args: Parameters<F>) => Promise<Awaited<ReturnType<F>>> : F;
|
|
636
|
+
/**
|
|
637
|
+
* A synonym for `wrapTraced`. If you're porting from systems that use `traceable`, you can use this to
|
|
638
|
+
* make your codebase more consistent.
|
|
639
|
+
*/
|
|
640
|
+
declare const traceable: typeof wrapTraced;
|
|
600
641
|
/**
|
|
601
642
|
* Lower-level alternative to `traced`. This allows you to start a span yourself, and can be useful in situations
|
|
602
643
|
* where you cannot use callbacks. However, spans started with `startSpan` will not be marked as the "current span",
|
|
@@ -604,19 +645,19 @@ export declare function traced<IsAsyncFlush extends boolean = false, R = void>(c
|
|
|
604
645
|
*
|
|
605
646
|
* See `traced` for full details.
|
|
606
647
|
*/
|
|
607
|
-
|
|
648
|
+
declare function startSpan<IsAsyncFlush extends boolean = false>(args?: StartSpanArgs & AsyncFlushArg<IsAsyncFlush> & OptionalStateArg): Span;
|
|
608
649
|
/**
|
|
609
650
|
* Flush any pending rows to the server.
|
|
610
651
|
*/
|
|
611
|
-
|
|
652
|
+
declare function flush(options?: OptionalStateArg): Promise<void>;
|
|
612
653
|
/**
|
|
613
654
|
* Set the fetch implementation to use for requests. You can specify it here,
|
|
614
655
|
* or when you call `login`.
|
|
615
656
|
*
|
|
616
657
|
* @param fetch The fetch implementation to use.
|
|
617
658
|
*/
|
|
618
|
-
|
|
619
|
-
|
|
659
|
+
declare function setFetch(fetch: typeof globalThis.fetch): void;
|
|
660
|
+
type WithTransactionId<R> = R & {
|
|
620
661
|
[TRANSACTION_ID_FIELD]: TransactionId;
|
|
621
662
|
};
|
|
622
663
|
declare class ObjectFetcher<RecordType> implements AsyncIterable<WithTransactionId<RecordType>> {
|
|
@@ -633,9 +674,9 @@ declare class ObjectFetcher<RecordType> implements AsyncIterable<WithTransaction
|
|
|
633
674
|
clearCache(): void;
|
|
634
675
|
version(): Promise<string | undefined>;
|
|
635
676
|
}
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
677
|
+
type BaseMetadata = Record<string, unknown> | void;
|
|
678
|
+
type DefaultMetadataType = void;
|
|
679
|
+
type EvalCase<Input, Expected, Metadata> = {
|
|
639
680
|
input: Input;
|
|
640
681
|
tags?: string[];
|
|
641
682
|
} & (Expected extends void ? {} : {
|
|
@@ -655,7 +696,7 @@ export type EvalCase<Input, Expected, Metadata> = {
|
|
|
655
696
|
*
|
|
656
697
|
* You should not create `Experiment` objects directly. Instead, use the `braintrust.init()` method.
|
|
657
698
|
*/
|
|
658
|
-
|
|
699
|
+
declare class Experiment extends ObjectFetcher<ExperimentEvent> implements Exportable {
|
|
659
700
|
private readonly lazyMetadata;
|
|
660
701
|
readonly dataset?: AnyDataset;
|
|
661
702
|
private lastStartTime;
|
|
@@ -748,7 +789,7 @@ export declare class Experiment extends ObjectFetcher<ExperimentEvent> {
|
|
|
748
789
|
/**
|
|
749
790
|
* A read-only view of an experiment, initialized by passing `open: true` to `init()`.
|
|
750
791
|
*/
|
|
751
|
-
|
|
792
|
+
declare class ReadonlyExperiment extends ObjectFetcher<ExperimentEvent> {
|
|
752
793
|
private state;
|
|
753
794
|
private readonly lazyMetadata;
|
|
754
795
|
constructor(state: BraintrustState, lazyMetadata: LazyValue<ProjectExperimentMetadata>);
|
|
@@ -757,13 +798,13 @@ export declare class ReadonlyExperiment extends ObjectFetcher<ExperimentEvent> {
|
|
|
757
798
|
protected getState(): Promise<BraintrustState>;
|
|
758
799
|
asDataset<Input, Expected>(): AsyncGenerator<EvalCase<Input, Expected, void>>;
|
|
759
800
|
}
|
|
760
|
-
|
|
801
|
+
declare function newId(): string;
|
|
761
802
|
/**
|
|
762
803
|
* Primary implementation of the `Span` interface. See the `Span` interface for full details on each method.
|
|
763
804
|
*
|
|
764
805
|
* We suggest using one of the various `traced` methods, instead of creating Spans directly. See `Span.startSpan` for full details.
|
|
765
806
|
*/
|
|
766
|
-
|
|
807
|
+
declare class SpanImpl implements Span {
|
|
767
808
|
private state;
|
|
768
809
|
private isMerge;
|
|
769
810
|
private loggedEndTime;
|
|
@@ -802,7 +843,7 @@ export declare class SpanImpl implements Span {
|
|
|
802
843
|
*
|
|
803
844
|
* You should not create `Dataset` objects directly. Instead, use the `braintrust.initDataset()` method.
|
|
804
845
|
*/
|
|
805
|
-
|
|
846
|
+
declare class Dataset<IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY_DATASET> extends ObjectFetcher<DatasetRecord<IsLegacyDataset>> {
|
|
806
847
|
private state;
|
|
807
848
|
private readonly lazyMetadata;
|
|
808
849
|
constructor(state: BraintrustState, lazyMetadata: LazyValue<ProjectDatasetMetadata>, pinnedVersion?: string, legacy?: IsLegacyDataset);
|
|
@@ -853,17 +894,17 @@ export declare class Dataset<IsLegacyDataset extends boolean = typeof DEFAULT_IS
|
|
|
853
894
|
*/
|
|
854
895
|
close(): Promise<string>;
|
|
855
896
|
}
|
|
856
|
-
|
|
897
|
+
type CompiledPromptParams = Omit<NonNullable<PromptData["options"]>["params"], "use_cache"> & {
|
|
857
898
|
model: NonNullable<NonNullable<PromptData["options"]>["model"]>;
|
|
858
899
|
};
|
|
859
|
-
|
|
900
|
+
type ChatPrompt = {
|
|
860
901
|
messages: OpenAIMessage[];
|
|
861
902
|
tools?: Tools;
|
|
862
903
|
};
|
|
863
|
-
|
|
904
|
+
type CompletionPrompt = {
|
|
864
905
|
prompt: string;
|
|
865
906
|
};
|
|
866
|
-
|
|
907
|
+
type CompiledPrompt<Flavor extends "chat" | "completion"> = CompiledPromptParams & {
|
|
867
908
|
span_info?: {
|
|
868
909
|
name?: string;
|
|
869
910
|
spanAttributes?: Record<any, any>;
|
|
@@ -877,12 +918,12 @@ export type CompiledPrompt<Flavor extends "chat" | "completion"> = CompiledPromp
|
|
|
877
918
|
};
|
|
878
919
|
};
|
|
879
920
|
} & (Flavor extends "chat" ? ChatPrompt : Flavor extends "completion" ? CompletionPrompt : {});
|
|
880
|
-
|
|
881
|
-
|
|
921
|
+
type DefaultPromptArgs = Partial<CompiledPromptParams & AnyModelParam & ChatPrompt & CompletionPrompt>;
|
|
922
|
+
declare class Prompt {
|
|
882
923
|
private metadata;
|
|
883
924
|
private defaults;
|
|
884
925
|
private noTrace;
|
|
885
|
-
constructor(metadata: Omit<
|
|
926
|
+
constructor(metadata: Omit<Prompt$1, "log_id"> | PromptSessionEvent, defaults: DefaultPromptArgs, noTrace: boolean);
|
|
886
927
|
get id(): string;
|
|
887
928
|
get projectId(): string;
|
|
888
929
|
get name(): string;
|
|
@@ -902,7 +943,7 @@ export declare class Prompt {
|
|
|
902
943
|
}): CompiledPrompt<Flavor>;
|
|
903
944
|
private runBuild;
|
|
904
945
|
}
|
|
905
|
-
|
|
946
|
+
type AnyDataset = Dataset<boolean>;
|
|
906
947
|
/**
|
|
907
948
|
* Summary of a score's performance.
|
|
908
949
|
* @property name Name of the score.
|
|
@@ -911,7 +952,7 @@ export type AnyDataset = Dataset<boolean>;
|
|
|
911
952
|
* @property improvements Number of improvements in the score.
|
|
912
953
|
* @property regressions Number of regressions in the score.
|
|
913
954
|
*/
|
|
914
|
-
|
|
955
|
+
interface ScoreSummary {
|
|
915
956
|
name: string;
|
|
916
957
|
score: number;
|
|
917
958
|
diff?: number;
|
|
@@ -927,7 +968,7 @@ export interface ScoreSummary {
|
|
|
927
968
|
* @property improvements Number of improvements in the metric.
|
|
928
969
|
* @property regressions Number of regressions in the metric.
|
|
929
970
|
*/
|
|
930
|
-
|
|
971
|
+
interface MetricSummary {
|
|
931
972
|
name: string;
|
|
932
973
|
metric: number;
|
|
933
974
|
unit: string;
|
|
@@ -945,7 +986,7 @@ export interface MetricSummary {
|
|
|
945
986
|
* @property comparisonExperimentName The experiment scores are baselined against.
|
|
946
987
|
* @property scores Summary of the experiment's scores.
|
|
947
988
|
*/
|
|
948
|
-
|
|
989
|
+
interface ExperimentSummary {
|
|
949
990
|
projectName: string;
|
|
950
991
|
experimentName: string;
|
|
951
992
|
projectId?: string;
|
|
@@ -962,7 +1003,7 @@ export interface ExperimentSummary {
|
|
|
962
1003
|
* @property newRecords New or updated records added in this session.
|
|
963
1004
|
* @property totalRecords Total records in the dataset.
|
|
964
1005
|
*/
|
|
965
|
-
|
|
1006
|
+
interface DataSummary {
|
|
966
1007
|
newRecords: number;
|
|
967
1008
|
totalRecords: number;
|
|
968
1009
|
}
|
|
@@ -975,11 +1016,74 @@ export interface DataSummary {
|
|
|
975
1016
|
* @property datasetUrl URL to the experiment's page in the Braintrust app.
|
|
976
1017
|
* @property dataSummary Summary of the dataset's data.
|
|
977
1018
|
*/
|
|
978
|
-
|
|
1019
|
+
interface DatasetSummary {
|
|
979
1020
|
projectName: string;
|
|
980
1021
|
datasetName: string;
|
|
981
1022
|
projectUrl: string;
|
|
982
1023
|
datasetUrl: string;
|
|
983
1024
|
dataSummary: DataSummary;
|
|
984
1025
|
}
|
|
985
|
-
|
|
1026
|
+
|
|
1027
|
+
type BraintrustStreamChunk = {
|
|
1028
|
+
type: "text_delta";
|
|
1029
|
+
data: string;
|
|
1030
|
+
} | {
|
|
1031
|
+
type: "json_delta";
|
|
1032
|
+
data: string;
|
|
1033
|
+
};
|
|
1034
|
+
declare class BraintrustStream {
|
|
1035
|
+
private stream;
|
|
1036
|
+
constructor(baseStream: ReadableStream<Uint8Array>);
|
|
1037
|
+
constructor(stream: ReadableStream<string>);
|
|
1038
|
+
constructor(stream: ReadableStream<BraintrustStreamChunk>);
|
|
1039
|
+
copy(): BraintrustStream;
|
|
1040
|
+
toReadableStream(): ReadableStream<BraintrustStreamChunk>;
|
|
1041
|
+
}
|
|
1042
|
+
declare function createFinalValuePassThroughStream<T extends BraintrustStreamChunk | string | Uint8Array>(onFinal: (result: unknown) => void): TransformStream<T, BraintrustStreamChunk>;
|
|
1043
|
+
declare function devNullWritableStream(): WritableStream;
|
|
1044
|
+
|
|
1045
|
+
type InvokeReturn<Stream extends boolean, Return> = Stream extends true ? BraintrustStream : Return;
|
|
1046
|
+
type InvokeFunctionArgs<Arg, Return, Stream extends boolean = false> = FunctionId & FullLoginOptions & {
|
|
1047
|
+
arg: Arg;
|
|
1048
|
+
parent?: Exportable | string;
|
|
1049
|
+
state?: BraintrustState;
|
|
1050
|
+
stream?: Stream;
|
|
1051
|
+
schema?: z.ZodSchema<Return>;
|
|
1052
|
+
};
|
|
1053
|
+
declare function invoke<Arg, Return, Stream extends boolean = false>(args: InvokeFunctionArgs<Arg, Return, Stream>): Promise<InvokeReturn<Stream, Return>>;
|
|
1054
|
+
|
|
1055
|
+
interface BetaLike {
|
|
1056
|
+
chat: {
|
|
1057
|
+
completions: {
|
|
1058
|
+
stream: any;
|
|
1059
|
+
};
|
|
1060
|
+
};
|
|
1061
|
+
embeddings: any;
|
|
1062
|
+
}
|
|
1063
|
+
interface ChatLike {
|
|
1064
|
+
completions: any;
|
|
1065
|
+
}
|
|
1066
|
+
interface OpenAILike {
|
|
1067
|
+
chat: ChatLike;
|
|
1068
|
+
embeddings: any;
|
|
1069
|
+
beta?: BetaLike;
|
|
1070
|
+
}
|
|
1071
|
+
declare global {
|
|
1072
|
+
var __inherited_braintrust_wrap_openai: ((openai: any) => any) | undefined;
|
|
1073
|
+
}
|
|
1074
|
+
/**
|
|
1075
|
+
* Wrap an `OpenAI` object (created with `new OpenAI(...)`) to add tracing. If Braintrust is
|
|
1076
|
+
* not configured, this is a no-op
|
|
1077
|
+
*
|
|
1078
|
+
* Currently, this only supports the `v4` API.
|
|
1079
|
+
*
|
|
1080
|
+
* @param openai
|
|
1081
|
+
* @returns The wrapped `OpenAI` object.
|
|
1082
|
+
*/
|
|
1083
|
+
declare function wrapOpenAI<T extends object>(openai: T): T;
|
|
1084
|
+
declare function wrapOpenAIv4<T extends OpenAILike>(openai: T): T;
|
|
1085
|
+
declare const LEGACY_CACHED_HEADER = "x-cached";
|
|
1086
|
+
declare const X_CACHED_HEADER = "x-bt-cached";
|
|
1087
|
+
declare function parseCachedHeader(value: string | null | undefined): number | undefined;
|
|
1088
|
+
|
|
1089
|
+
export { type AnyDataset, type BaseMetadata, BraintrustState, BraintrustStream, type BraintrustStreamChunk, type ChatPrompt, type CompiledPrompt, type CompiledPromptParams, type CompletionPrompt, type DataSummary, Dataset, type DatasetSummary, type DefaultMetadataType, type DefaultPromptArgs, type EndSpanArgs, type EvalCase, Experiment, type ExperimentSummary, type Exportable, type FullInitOptions, type FullLoginOptions, type InitOptions, type InvokeFunctionArgs, type InvokeReturn, LEGACY_CACHED_HEADER, type LogOptions, Logger, type LoginOptions, type MetricSummary, NOOP_SPAN, NoopSpan, type ObjectMetadata, type PromiseUnless, Prompt, ReadonlyExperiment, type ScoreSummary, type SerializedBraintrustState, type SetCurrentArg, type Span, SpanImpl, type StartSpanArgs, type WithTransactionId, X_CACHED_HEADER, _internalGetGlobalState, _internalSetInitialState, createFinalValuePassThroughStream, currentExperiment, currentLogger, currentSpan, devNullWritableStream, flush, getSpanParentObject, init, initDataset, initExperiment, initLogger, invoke, loadPrompt, log, login, loginToState, newId, parseCachedHeader, setFetch, startSpan, summarize, traceable, traced, withDataset, withExperiment, withLogger, wrapOpenAI, wrapOpenAIv4, wrapTraced };
|