braintrust 0.0.139 → 0.0.141-dev

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,3203 @@
1
+ // src/isomorph.ts
2
+ var DefaultAsyncLocalStorage = class {
3
+ constructor() {
4
+ }
5
+ enterWith(_) {
6
+ }
7
+ run(_, callback) {
8
+ return callback();
9
+ }
10
+ getStore() {
11
+ return void 0;
12
+ }
13
+ };
14
+ var iso = {
15
+ getRepoInfo: async (_settings) => void 0,
16
+ getPastNAncestors: async () => [],
17
+ getEnv: (_name) => void 0,
18
+ getCallerLocation: () => void 0,
19
+ newAsyncLocalStorage: () => new DefaultAsyncLocalStorage(),
20
+ processOn: (_0, _1) => {
21
+ }
22
+ };
23
+ var isomorph_default = iso;
24
+
25
+ // src/logger.ts
26
+ import { v4 as uuidv4 } from "uuid";
27
+ import {
28
+ TRANSACTION_ID_FIELD,
29
+ IS_MERGE_FIELD,
30
+ mergeDicts,
31
+ mergeRowBatch,
32
+ VALID_SOURCES,
33
+ AUDIT_SOURCE_FIELD,
34
+ AUDIT_METADATA_FIELD,
35
+ mergeGitMetadataSettings,
36
+ DEFAULT_IS_LEGACY_DATASET,
37
+ ensureDatasetRecord,
38
+ makeLegacyEvent,
39
+ constructJsonArray,
40
+ SpanTypeAttribute,
41
+ batchItems,
42
+ SpanComponentsV2,
43
+ SpanObjectTypeV2,
44
+ SpanRowIdsV2,
45
+ gitMetadataSettingsSchema
46
+ } from "@braintrust/core";
47
+ import {
48
+ BRAINTRUST_PARAMS,
49
+ promptSchema,
50
+ toolsSchema
51
+ } from "@braintrust/core/typespecs";
52
+
53
+ // src/util.ts
54
+ var GLOBAL_PROJECT = "Global";
55
+ function runFinally(f, finallyF) {
56
+ let runSyncCleanup = true;
57
+ try {
58
+ const ret = f();
59
+ if (ret instanceof Promise) {
60
+ runSyncCleanup = false;
61
+ return ret.finally(finallyF);
62
+ } else {
63
+ return ret;
64
+ }
65
+ } finally {
66
+ if (runSyncCleanup) {
67
+ finallyF();
68
+ }
69
+ }
70
+ }
71
+ function getCurrentUnixTimestamp() {
72
+ return (/* @__PURE__ */ new Date()).getTime() / 1e3;
73
+ }
74
+ function isEmpty(a) {
75
+ return a === void 0 || a === null;
76
+ }
77
+ var LazyValue = class {
78
+ callable;
79
+ value = {
80
+ hasComputed: false
81
+ };
82
+ constructor(callable) {
83
+ this.callable = callable;
84
+ }
85
+ get() {
86
+ if (this.value.hasComputed) {
87
+ return this.value.val;
88
+ }
89
+ this.value = { hasComputed: true, val: this.callable() };
90
+ return this.value.val;
91
+ }
92
+ get hasComputed() {
93
+ return this.value.hasComputed;
94
+ }
95
+ };
96
+
97
+ // src/logger.ts
98
+ import Mustache from "mustache";
99
+ import { z } from "zod";
100
+
101
+ // src/stream.ts
102
+ import { callEventSchema } from "@braintrust/core/typespecs";
103
+ import {
104
+ createParser
105
+ } from "eventsource-parser";
106
+ var BraintrustStream = class _BraintrustStream {
107
+ stream;
108
+ constructor(baseStream) {
109
+ this.stream = baseStream.pipeThrough(btStreamParser());
110
+ }
111
+ copy() {
112
+ const [newStream, copyStream] = this.stream.tee();
113
+ this.stream = copyStream;
114
+ return new _BraintrustStream(newStream);
115
+ }
116
+ toReadableStream() {
117
+ return this.stream;
118
+ }
119
+ };
120
+ function btStreamParser() {
121
+ const decoder = new TextDecoder();
122
+ let parser;
123
+ return new TransformStream({
124
+ async start(controller) {
125
+ parser = createParser((event) => {
126
+ if (event.type === "reconnect-interval") {
127
+ return;
128
+ }
129
+ const parsed = callEventSchema.safeParse(event);
130
+ if (!parsed.success) {
131
+ throw new Error(`Failed to parse event: ${parsed.error}`);
132
+ }
133
+ switch (event.event) {
134
+ case "text_delta":
135
+ controller.enqueue({
136
+ type: "text_delta",
137
+ data: JSON.parse(event.data)
138
+ });
139
+ break;
140
+ case "json_delta":
141
+ controller.enqueue({
142
+ type: "json_delta",
143
+ data: event.data
144
+ });
145
+ break;
146
+ case "done":
147
+ break;
148
+ }
149
+ });
150
+ },
151
+ async transform(chunk, controller) {
152
+ if (chunk instanceof Uint8Array) {
153
+ parser.feed(decoder.decode(chunk));
154
+ } else if (typeof chunk === "string") {
155
+ parser.feed(chunk);
156
+ } else {
157
+ controller.enqueue(chunk);
158
+ }
159
+ },
160
+ async flush(controller) {
161
+ controller.terminate();
162
+ }
163
+ });
164
+ }
165
+ function createFinalValuePassThroughStream(onFinal) {
166
+ const decoder = new TextDecoder();
167
+ const textChunks = [];
168
+ const jsonChunks = [];
169
+ const transformStream = new TransformStream({
170
+ transform(chunk, controller) {
171
+ if (typeof chunk === "string") {
172
+ textChunks.push(chunk);
173
+ } else if (chunk instanceof Uint8Array) {
174
+ textChunks.push(decoder.decode(chunk));
175
+ } else {
176
+ const chunkType = chunk.type;
177
+ switch (chunkType) {
178
+ case "text_delta":
179
+ textChunks.push(chunk.data);
180
+ break;
181
+ case "json_delta":
182
+ jsonChunks.push(chunk.data);
183
+ break;
184
+ default:
185
+ const _type = chunkType;
186
+ throw new Error(`Unknown chunk type ${_type}`);
187
+ }
188
+ controller.enqueue(chunk);
189
+ }
190
+ },
191
+ flush(controller) {
192
+ if (jsonChunks.length > 0) {
193
+ onFinal(JSON.parse(jsonChunks.join("")));
194
+ } else if (textChunks.length > 0) {
195
+ onFinal(textChunks.join(""));
196
+ } else {
197
+ onFinal(void 0);
198
+ }
199
+ controller.terminate();
200
+ }
201
+ });
202
+ return transformStream;
203
+ }
204
+ function devNullWritableStream() {
205
+ return new WritableStream({
206
+ write(chunk) {
207
+ },
208
+ close() {
209
+ },
210
+ abort(reason) {
211
+ },
212
+ start(controller) {
213
+ }
214
+ });
215
+ }
216
+
217
+ // src/logger.ts
218
+ import { waitUntil } from "@vercel/functions";
219
+ var NoopSpan = class {
220
+ id;
221
+ kind = "span";
222
+ constructor() {
223
+ this.id = "";
224
+ }
225
+ log(_) {
226
+ }
227
+ logFeedback(_event) {
228
+ }
229
+ traced(callback, _1) {
230
+ return callback(this);
231
+ }
232
+ startSpan(_1) {
233
+ return this;
234
+ }
235
+ end(args) {
236
+ return args?.endTime ?? getCurrentUnixTimestamp();
237
+ }
238
+ async export() {
239
+ return "";
240
+ }
241
+ async flush() {
242
+ }
243
+ close(args) {
244
+ return this.end(args);
245
+ }
246
+ setAttributes(_args) {
247
+ }
248
+ };
249
+ var NOOP_SPAN = new NoopSpan();
250
+ var loginSchema = z.strictObject({
251
+ appUrl: z.string(),
252
+ appPublicUrl: z.string(),
253
+ orgName: z.string(),
254
+ logUrl: z.string(),
255
+ proxyUrl: z.string(),
256
+ loginToken: z.string(),
257
+ orgId: z.string().nullish(),
258
+ gitMetadataSettings: gitMetadataSettingsSchema.nullish()
259
+ });
260
+ var BraintrustState = class _BraintrustState {
261
+ constructor(loginParams) {
262
+ this.loginParams = loginParams;
263
+ this.id = (/* @__PURE__ */ new Date()).toLocaleString();
264
+ this.currentExperiment = void 0;
265
+ this.currentLogger = void 0;
266
+ this.currentSpan = isomorph_default.newAsyncLocalStorage();
267
+ if (loginParams.fetch) {
268
+ this.fetch = loginParams.fetch;
269
+ }
270
+ const defaultGetLogConn = async () => {
271
+ await this.login({});
272
+ return this.logConn();
273
+ };
274
+ this._bgLogger = new BackgroundLogger(new LazyValue(defaultGetLogConn));
275
+ this.resetLoginInfo();
276
+ }
277
+ id;
278
+ currentExperiment;
279
+ // Note: the value of IsAsyncFlush doesn't really matter here, since we
280
+ // (safely) dynamically cast it whenever retrieving the logger.
281
+ currentLogger;
282
+ currentSpan;
283
+ // Any time we re-log in, we directly update the logConn inside the logger.
284
+ // This is preferable to replacing the whole logger, which would create the
285
+ // possibility of multiple loggers floating around, which may not log in a
286
+ // deterministic order.
287
+ _bgLogger;
288
+ appUrl = null;
289
+ appPublicUrl = null;
290
+ loginToken = null;
291
+ orgId = null;
292
+ orgName = null;
293
+ logUrl = null;
294
+ proxyUrl = null;
295
+ loggedIn = false;
296
+ gitMetadataSettings;
297
+ fetch = globalThis.fetch;
298
+ _apiConn = null;
299
+ _logConn = null;
300
+ _proxyConn = null;
301
+ resetLoginInfo() {
302
+ this.appUrl = null;
303
+ this.appPublicUrl = null;
304
+ this.loginToken = null;
305
+ this.orgId = null;
306
+ this.orgName = null;
307
+ this.logUrl = null;
308
+ this.proxyUrl = null;
309
+ this.loggedIn = false;
310
+ this.gitMetadataSettings = void 0;
311
+ this._apiConn = null;
312
+ this._logConn = null;
313
+ this._proxyConn = null;
314
+ }
315
+ copyLoginInfo(other) {
316
+ this.appUrl = other.appUrl;
317
+ this.appPublicUrl = other.appPublicUrl;
318
+ this.loginToken = other.loginToken;
319
+ this.orgId = other.orgId;
320
+ this.orgName = other.orgName;
321
+ this.logUrl = other.logUrl;
322
+ this.proxyUrl = other.proxyUrl;
323
+ this.loggedIn = other.loggedIn;
324
+ this.gitMetadataSettings = other.gitMetadataSettings;
325
+ this._apiConn = other._apiConn;
326
+ this._logConn = other._logConn;
327
+ this._proxyConn = other._proxyConn;
328
+ }
329
+ serialize() {
330
+ if (!this.loggedIn) {
331
+ throw new Error(
332
+ "Cannot serialize BraintrustState without being logged in"
333
+ );
334
+ }
335
+ if (!this.appUrl || !this.appPublicUrl || !this.logUrl || !this.proxyUrl || !this.orgName || !this.loginToken || !this.loggedIn) {
336
+ throw new Error(
337
+ "Cannot serialize BraintrustState without all login attributes"
338
+ );
339
+ }
340
+ return {
341
+ appUrl: this.appUrl,
342
+ appPublicUrl: this.appPublicUrl,
343
+ loginToken: this.loginToken,
344
+ orgId: this.orgId,
345
+ orgName: this.orgName,
346
+ logUrl: this.logUrl,
347
+ proxyUrl: this.proxyUrl,
348
+ gitMetadataSettings: this.gitMetadataSettings
349
+ };
350
+ }
351
+ static deserialize(serialized) {
352
+ const serializedParsed = loginSchema.safeParse(serialized);
353
+ if (!serializedParsed.success) {
354
+ throw new Error(
355
+ `Cannot deserialize BraintrustState: ${serializedParsed.error.errors}`
356
+ );
357
+ }
358
+ const state = new _BraintrustState({});
359
+ for (const key of Object.keys(loginSchema.shape)) {
360
+ state[key] = serializedParsed.data[key];
361
+ }
362
+ if (!state.loginToken) {
363
+ throw new Error(
364
+ "Cannot deserialize BraintrustState without a login token"
365
+ );
366
+ }
367
+ state.logConn().set_token(state.loginToken);
368
+ state.logConn().make_long_lived();
369
+ state.apiConn().set_token(state.loginToken);
370
+ state.proxyConn().set_token(state.loginToken);
371
+ state.loggedIn = true;
372
+ state.loginReplaceLogConn(state.logConn());
373
+ return state;
374
+ }
375
+ setFetch(fetch) {
376
+ this.loginParams.fetch = fetch;
377
+ this.fetch = fetch;
378
+ this._logConn?.setFetch(fetch);
379
+ this._apiConn?.setFetch(fetch);
380
+ }
381
+ async login(loginParams) {
382
+ if (this.logUrl && !loginParams.forceLogin) {
383
+ return;
384
+ }
385
+ const newState = await loginToState({
386
+ ...this.loginParams,
387
+ ...loginParams
388
+ });
389
+ this.copyLoginInfo(newState);
390
+ }
391
+ apiConn() {
392
+ if (!this._apiConn) {
393
+ if (!this.appUrl) {
394
+ throw new Error("Must initialize appUrl before requesting apiConn");
395
+ }
396
+ this._apiConn = new HTTPConnection(this.appUrl, this.fetch);
397
+ }
398
+ return this._apiConn;
399
+ }
400
+ logConn() {
401
+ if (!this._logConn) {
402
+ if (!this.logUrl) {
403
+ throw new Error("Must initialize logUrl before requesting logConn");
404
+ }
405
+ this._logConn = new HTTPConnection(this.logUrl, this.fetch);
406
+ }
407
+ return this._logConn;
408
+ }
409
+ proxyConn() {
410
+ if (!this._proxyConn) {
411
+ if (!this.proxyUrl) {
412
+ throw new Error("Must initialize proxyUrl before requesting proxyConn");
413
+ }
414
+ this._proxyConn = new HTTPConnection(this.proxyUrl, this.fetch);
415
+ }
416
+ return this._proxyConn;
417
+ }
418
+ bgLogger() {
419
+ return this._bgLogger;
420
+ }
421
+ // Should only be called by the login function.
422
+ loginReplaceLogConn(logConn) {
423
+ this._bgLogger.internalReplaceLogConn(logConn);
424
+ }
425
+ };
426
+ var _globalState;
427
+ function _internalSetInitialState() {
428
+ if (_globalState) {
429
+ throw new Error("Cannot set initial state more than once");
430
+ }
431
+ _globalState = globalThis.__inherited_braintrust_state || new BraintrustState({
432
+ /*empty login options*/
433
+ });
434
+ }
435
+ var _internalGetGlobalState = () => _globalState;
436
+ var FailedHTTPResponse = class extends Error {
437
+ status;
438
+ text;
439
+ data;
440
+ constructor(status, text, data = null) {
441
+ super(`${status}: ${text}`);
442
+ this.status = status;
443
+ this.text = text;
444
+ this.data = data;
445
+ }
446
+ };
447
+ async function checkResponse(resp) {
448
+ if (resp.ok) {
449
+ return resp;
450
+ } else {
451
+ throw new FailedHTTPResponse(
452
+ resp.status,
453
+ resp.statusText,
454
+ await resp.text()
455
+ );
456
+ }
457
+ }
458
+ var HTTPConnection = class _HTTPConnection {
459
+ base_url;
460
+ token;
461
+ headers;
462
+ fetch;
463
+ constructor(base_url, fetch) {
464
+ this.base_url = base_url;
465
+ this.token = null;
466
+ this.headers = {};
467
+ this._reset();
468
+ this.fetch = fetch;
469
+ }
470
+ setFetch(fetch) {
471
+ this.fetch = fetch;
472
+ }
473
+ async ping() {
474
+ try {
475
+ const resp = await this.get("ping");
476
+ return resp.status === 200;
477
+ } catch (e) {
478
+ return false;
479
+ }
480
+ }
481
+ make_long_lived() {
482
+ this._reset();
483
+ }
484
+ static sanitize_token(token) {
485
+ return token.trim();
486
+ }
487
+ set_token(token) {
488
+ token = _HTTPConnection.sanitize_token(token);
489
+ this.token = token;
490
+ this._reset();
491
+ }
492
+ // As far as I can tell, you cannot set the retry/backoff factor here
493
+ _reset() {
494
+ this.headers = {};
495
+ if (this.token) {
496
+ this.headers["Authorization"] = `Bearer ${this.token}`;
497
+ }
498
+ }
499
+ async get(path, params = void 0, config) {
500
+ const { headers, ...rest } = config || {};
501
+ const url = new URL(_urljoin(this.base_url, path));
502
+ url.search = new URLSearchParams(
503
+ params ? Object.fromEntries(
504
+ Object.entries(params).filter(([_, v]) => v !== void 0)
505
+ ) : {}
506
+ ).toString();
507
+ return await checkResponse(
508
+ // Using toString() here makes it work with isomorphic fetch
509
+ await this.fetch(url.toString(), {
510
+ headers: {
511
+ Accept: "application/json",
512
+ ...this.headers,
513
+ ...headers
514
+ },
515
+ keepalive: true,
516
+ ...rest
517
+ })
518
+ );
519
+ }
520
+ async post(path, params, config) {
521
+ const { headers, ...rest } = config || {};
522
+ return await checkResponse(
523
+ await this.fetch(_urljoin(this.base_url, path), {
524
+ method: "POST",
525
+ headers: {
526
+ Accept: "application/json",
527
+ "Content-Type": "application/json",
528
+ ...this.headers,
529
+ ...headers
530
+ },
531
+ body: typeof params === "string" ? params : params ? JSON.stringify(params) : void 0,
532
+ keepalive: true,
533
+ ...rest
534
+ })
535
+ );
536
+ }
537
+ async get_json(object_type, args = void 0, retries = 0) {
538
+ const tries = retries + 1;
539
+ for (let i = 0; i < tries; i++) {
540
+ try {
541
+ const resp = await this.get(`${object_type}`, args);
542
+ return await resp.json();
543
+ } catch (e) {
544
+ if (i < tries - 1) {
545
+ console.log(
546
+ `Retrying API request ${object_type} ${JSON.stringify(args)} ${e.status} ${e.text}`
547
+ );
548
+ continue;
549
+ }
550
+ throw e;
551
+ }
552
+ }
553
+ }
554
+ async post_json(object_type, args = void 0) {
555
+ const resp = await this.post(`${object_type}`, args, {
556
+ headers: { "Content-Type": "application/json" }
557
+ });
558
+ return await resp.json();
559
+ }
560
+ };
561
+ function logFeedbackImpl(state, parentObjectType, parentObjectId, {
562
+ id,
563
+ expected,
564
+ scores,
565
+ metadata: inputMetadata,
566
+ tags,
567
+ comment,
568
+ source: inputSource
569
+ }) {
570
+ const source = inputSource ?? "external";
571
+ if (!VALID_SOURCES.includes(source)) {
572
+ throw new Error(`source must be one of ${VALID_SOURCES}`);
573
+ }
574
+ if (isEmpty(scores) && isEmpty(expected) && isEmpty(tags) && isEmpty(comment)) {
575
+ throw new Error(
576
+ "At least one of scores, expected, tags, or comment must be specified"
577
+ );
578
+ }
579
+ const validatedEvent = validateAndSanitizeExperimentLogPartialArgs({
580
+ scores,
581
+ metadata: inputMetadata,
582
+ expected,
583
+ tags
584
+ });
585
+ let { metadata, ...updateEvent } = validatedEvent;
586
+ updateEvent = Object.fromEntries(
587
+ Object.entries(updateEvent).filter(([_, v]) => !isEmpty(v))
588
+ );
589
+ const parentIds = async () => new SpanComponentsV2({
590
+ objectType: parentObjectType,
591
+ objectId: await parentObjectId.get()
592
+ }).objectIdFields();
593
+ if (Object.keys(updateEvent).length > 0) {
594
+ const record = new LazyValue(async () => {
595
+ return {
596
+ id,
597
+ ...updateEvent,
598
+ ...await parentIds(),
599
+ [AUDIT_SOURCE_FIELD]: source,
600
+ [AUDIT_METADATA_FIELD]: metadata,
601
+ [IS_MERGE_FIELD]: true
602
+ };
603
+ });
604
+ state.bgLogger().log([record]);
605
+ }
606
+ if (!isEmpty(comment)) {
607
+ const record = new LazyValue(async () => {
608
+ return {
609
+ id: uuidv4(),
610
+ created: (/* @__PURE__ */ new Date()).toISOString(),
611
+ origin: {
612
+ // NOTE: We do not know (or care?) what the transaction id of the row that
613
+ // we're commenting on is here, so we omit it.
614
+ id
615
+ },
616
+ comment: {
617
+ text: comment
618
+ },
619
+ ...await parentIds(),
620
+ [AUDIT_SOURCE_FIELD]: source,
621
+ [AUDIT_METADATA_FIELD]: metadata
622
+ };
623
+ });
624
+ state.bgLogger().log([record]);
625
+ }
626
+ }
627
+ function spanComponentsToObjectIdLambda(state, components) {
628
+ if (components.objectId) {
629
+ const ret = components.objectId;
630
+ return async () => ret;
631
+ }
632
+ if (!components.computeObjectMetadataArgs) {
633
+ throw new Error(
634
+ "Impossible: must provide either objectId or computeObjectMetadataArgs"
635
+ );
636
+ }
637
+ switch (components.objectType) {
638
+ case SpanObjectTypeV2.EXPERIMENT:
639
+ throw new Error(
640
+ "Impossible: computeObjectMetadataArgs not supported for experiments"
641
+ );
642
+ case SpanObjectTypeV2.PROJECT_LOGS:
643
+ return async () => (await computeLoggerMetadata(state, {
644
+ ...components.computeObjectMetadataArgs
645
+ })).project.id;
646
+ default:
647
+ const x = components.objectType;
648
+ throw new Error(`Unknown object type: ${x}`);
649
+ }
650
+ }
651
+ function startSpanParentArgs(args) {
652
+ let argParentObjectId = void 0;
653
+ let argParentSpanIds = void 0;
654
+ if (args.parent) {
655
+ if (args.parentSpanIds) {
656
+ throw new Error("Cannot specify both parent and parentSpanIds");
657
+ }
658
+ const parentComponents = SpanComponentsV2.fromStr(args.parent);
659
+ if (args.parentObjectType !== parentComponents.objectType) {
660
+ throw new Error(
661
+ `Mismatch between expected span parent object type ${args.parentObjectType} and provided type ${parentComponents.objectType}`
662
+ );
663
+ }
664
+ const parentComponentsObjectIdLambda = spanComponentsToObjectIdLambda(
665
+ args.state,
666
+ parentComponents
667
+ );
668
+ const computeParentObjectId = async () => {
669
+ const parentComponentsObjectId = await parentComponentsObjectIdLambda();
670
+ if (await args.parentObjectId.get() !== parentComponentsObjectId) {
671
+ throw new Error(
672
+ `Mismatch between expected span parent object id ${await args.parentObjectId.get()} and provided id ${parentComponentsObjectId}`
673
+ );
674
+ }
675
+ return await args.parentObjectId.get();
676
+ };
677
+ argParentObjectId = new LazyValue(computeParentObjectId);
678
+ if (parentComponents.rowIds) {
679
+ argParentSpanIds = {
680
+ spanId: parentComponents.rowIds.spanId,
681
+ rootSpanId: parentComponents.rowIds.rootSpanId
682
+ };
683
+ }
684
+ } else {
685
+ argParentObjectId = args.parentObjectId;
686
+ argParentSpanIds = args.parentSpanIds;
687
+ }
688
+ return {
689
+ parentObjectType: args.parentObjectType,
690
+ parentObjectId: argParentObjectId,
691
+ parentComputeObjectMetadataArgs: args.parentComputeObjectMetadataArgs,
692
+ parentSpanIds: argParentSpanIds
693
+ };
694
+ }
695
+ var Logger = class {
696
+ state;
697
+ lazyMetadata;
698
+ _asyncFlush;
699
+ computeMetadataArgs;
700
+ lastStartTime;
701
+ lazyId;
702
+ calledStartSpan;
703
+ // For type identification.
704
+ kind = "logger";
705
+ constructor(state, lazyMetadata, logOptions = {}) {
706
+ this.lazyMetadata = lazyMetadata;
707
+ this._asyncFlush = logOptions.asyncFlush;
708
+ this.computeMetadataArgs = logOptions.computeMetadataArgs;
709
+ this.lastStartTime = getCurrentUnixTimestamp();
710
+ this.lazyId = new LazyValue(async () => await this.id);
711
+ this.calledStartSpan = false;
712
+ this.state = state;
713
+ }
714
+ get org_id() {
715
+ return (async () => {
716
+ return (await this.lazyMetadata.get()).org_id;
717
+ })();
718
+ }
719
+ get project() {
720
+ return (async () => {
721
+ return (await this.lazyMetadata.get()).project;
722
+ })();
723
+ }
724
+ get id() {
725
+ return (async () => (await this.project).id)();
726
+ }
727
+ parentObjectType() {
728
+ return SpanObjectTypeV2.PROJECT_LOGS;
729
+ }
730
+ triggerWaitUntilFlush() {
731
+ if (!this.state.bgLogger().syncFlush) {
732
+ return waitUntil(this.state.bgLogger().flush());
733
+ }
734
+ }
735
+ /**
736
+ * Log a single event. The event will be batched and uploaded behind the scenes if `logOptions.asyncFlush` is true.
737
+ *
738
+ * @param event The event to log.
739
+ * @param event.input: (Optional) the arguments that uniquely define a user input (an arbitrary, JSON serializable object).
740
+ * @param event.output: (Optional) the output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question.
741
+ * @param event.expected: (Optional) the ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models.
742
+ * @param event.scores: (Optional) a dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs.
743
+ * @param event.metadata: (Optional) a dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings.
744
+ * @param event.metrics: (Optional) a dictionary of metrics to log. The following keys are populated automatically: "start", "end".
745
+ * @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
746
+ * @param options Additional logging options
747
+ * @param options.allowConcurrentWithSpans in rare cases where you need to log at the top level separately from spans on the logger elsewhere, set this to true.
748
+ * :returns: The `id` of the logged event.
749
+ */
750
+ log(event, options) {
751
+ if (this.calledStartSpan && !options?.allowConcurrentWithSpans) {
752
+ throw new Error(
753
+ "Cannot run toplevel `log` method while using spans. To log to the span, call `logger.traced` and then log with `span.log`"
754
+ );
755
+ }
756
+ const span = this.startSpanImpl({ startTime: this.lastStartTime, event });
757
+ this.lastStartTime = span.end();
758
+ const ret = span.id;
759
+ if (this.asyncFlush === true) {
760
+ this.triggerWaitUntilFlush();
761
+ return ret;
762
+ } else {
763
+ return (async () => {
764
+ await this.flush();
765
+ return ret;
766
+ })();
767
+ }
768
+ }
769
+ /**
770
+ * Create a new toplevel span underneath the logger. The name defaults to "root".
771
+ *
772
+ * See `Span.traced` for full details.
773
+ */
774
+ traced(callback, args) {
775
+ const { setCurrent, ...argsRest } = args ?? {};
776
+ const span = this.startSpan(argsRest);
777
+ const ret = runFinally(
778
+ () => {
779
+ if (setCurrent ?? true) {
780
+ return withCurrent(span, callback);
781
+ } else {
782
+ return callback(span);
783
+ }
784
+ },
785
+ () => span.end()
786
+ );
787
+ if (this.asyncFlush) {
788
+ this.triggerWaitUntilFlush();
789
+ return ret;
790
+ } else {
791
+ return (async () => {
792
+ const awaitedRet = await ret;
793
+ await this.flush();
794
+ return awaitedRet;
795
+ })();
796
+ }
797
+ }
798
+ /**
799
+ * Lower-level alternative to `traced`. This allows you to start a span yourself, and can be useful in situations
800
+ * where you cannot use callbacks. However, spans started with `startSpan` will not be marked as the "current span",
801
+ * so `currentSpan()` and `traced()` will be no-ops. If you want to mark a span as current, use `traced` instead.
802
+ *
803
+ * See `traced` for full details.
804
+ */
805
+ startSpan(args) {
806
+ this.calledStartSpan = true;
807
+ return this.startSpanImpl(args);
808
+ }
809
+ startSpanImpl(args) {
810
+ return new SpanImpl({
811
+ state: this.state,
812
+ ...startSpanParentArgs({
813
+ state: this.state,
814
+ parent: args?.parent,
815
+ parentObjectType: this.parentObjectType(),
816
+ parentObjectId: this.lazyId,
817
+ parentComputeObjectMetadataArgs: this.computeMetadataArgs,
818
+ parentSpanIds: void 0
819
+ }),
820
+ ...args,
821
+ defaultRootType: SpanTypeAttribute.TASK
822
+ });
823
+ }
824
+ /**
825
+ * Log feedback to an event. Feedback is used to save feedback scores, set an expected value, or add a comment.
826
+ *
827
+ * @param event
828
+ * @param event.id The id of the event to log feedback for. This is the `id` returned by `log` or accessible as the `id` field of a span.
829
+ * @param event.scores (Optional) a dictionary of numeric values (between 0 and 1) to log. These scores will be merged into the existing scores for the event.
830
+ * @param event.expected (Optional) the ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not.
831
+ * @param event.comment (Optional) an optional comment string to log about the event.
832
+ * @param event.metadata (Optional) a dictionary with additional data about the feedback. If you have a `user_id`, you can log it here and access it in the Braintrust UI.
833
+ * @param event.source (Optional) the source of the feedback. Must be one of "external" (default), "app", or "api".
834
+ */
835
+ logFeedback(event) {
836
+ logFeedbackImpl(this.state, this.parentObjectType(), this.lazyId, event);
837
+ }
838
+ /**
839
+ * Return a serialized representation of the logger that can be used to start subspans in other places. See `Span.start_span` for more details.
840
+ */
841
+ async export() {
842
+ let objectId = void 0;
843
+ let computeObjectMetadataArgs = void 0;
844
+ if (this.computeMetadataArgs && !this.lazyId.hasComputed) {
845
+ computeObjectMetadataArgs = this.computeMetadataArgs;
846
+ } else {
847
+ objectId = await this.lazyId.get();
848
+ }
849
+ return new SpanComponentsV2({
850
+ objectType: this.parentObjectType(),
851
+ objectId,
852
+ computeObjectMetadataArgs
853
+ }).toStr();
854
+ }
855
+ /*
856
+ * Flush any pending logs to the server.
857
+ */
858
+ async flush() {
859
+ return await this.state.bgLogger().flush();
860
+ }
861
+ get asyncFlush() {
862
+ return this._asyncFlush;
863
+ }
864
+ };
865
+ function castLogger(logger, asyncFlush) {
866
+ if (logger === void 0)
867
+ return void 0;
868
+ if (asyncFlush && !!asyncFlush !== !!logger.asyncFlush) {
869
+ throw new Error(
870
+ `Asserted asyncFlush setting ${asyncFlush} does not match stored logger's setting ${logger.asyncFlush}`
871
+ );
872
+ }
873
+ return logger;
874
+ }
875
+ function constructLogs3Data(items) {
876
+ return `{"rows": ${constructJsonArray(items)}, "api_version": 2}`;
877
+ }
878
+ function now() {
879
+ return (/* @__PURE__ */ new Date()).getTime();
880
+ }
881
+ var BackgroundLogger = class _BackgroundLogger {
882
+ logConn;
883
+ items = [];
884
+ activeFlush = Promise.resolve();
885
+ activeFlushResolved = true;
886
+ activeFlushError = void 0;
887
+ syncFlush = false;
888
+ // 6 MB for the AWS lambda gateway (from our own testing).
889
+ maxRequestSize = 6 * 1024 * 1024;
890
+ defaultBatchSize = 100;
891
+ numTries = 3;
892
+ queueDropExceedingMaxsize = void 0;
893
+ queueDropLoggingPeriod = 60;
894
+ failedPublishPayloadsDir = void 0;
895
+ allPublishPayloadsDir = void 0;
896
+ queueDropLoggingState = {
897
+ numDropped: 0,
898
+ lastLoggedTimestamp: 0
899
+ };
900
+ constructor(logConn) {
901
+ this.logConn = logConn;
902
+ const syncFlushEnv = Number(isomorph_default.getEnv("BRAINTRUST_SYNC_FLUSH"));
903
+ if (!isNaN(syncFlushEnv)) {
904
+ this.syncFlush = Boolean(syncFlushEnv);
905
+ }
906
+ const defaultBatchSizeEnv = Number(
907
+ isomorph_default.getEnv("BRAINTRUST_DEFAULT_BATCH_SIZE")
908
+ );
909
+ if (!isNaN(defaultBatchSizeEnv)) {
910
+ this.defaultBatchSize = defaultBatchSizeEnv;
911
+ }
912
+ const maxRequestSizeEnv = Number(isomorph_default.getEnv("BRAINTRUST_MAX_REQUEST_SIZE"));
913
+ if (!isNaN(maxRequestSizeEnv)) {
914
+ this.maxRequestSize = maxRequestSizeEnv;
915
+ }
916
+ const numTriesEnv = Number(isomorph_default.getEnv("BRAINTRUST_NUM_RETRIES"));
917
+ if (!isNaN(numTriesEnv)) {
918
+ this.numTries = numTriesEnv + 1;
919
+ }
920
+ const queueDropExceedingMaxsizeEnv = Number(
921
+ isomorph_default.getEnv("BRAINTRUST_QUEUE_DROP_EXCEEDING_MAXSIZE")
922
+ );
923
+ if (!isNaN(queueDropExceedingMaxsizeEnv)) {
924
+ this.queueDropExceedingMaxsize = queueDropExceedingMaxsizeEnv;
925
+ }
926
+ const queueDropLoggingPeriodEnv = Number(
927
+ isomorph_default.getEnv("BRAINTRUST_QUEUE_DROP_LOGGING_PERIOD")
928
+ );
929
+ if (!isNaN(queueDropLoggingPeriodEnv)) {
930
+ this.queueDropLoggingPeriod = queueDropLoggingPeriodEnv;
931
+ }
932
+ const failedPublishPayloadsDirEnv = isomorph_default.getEnv(
933
+ "BRAINTRUST_FAILED_PUBLISH_PAYLOADS_DIR"
934
+ );
935
+ if (failedPublishPayloadsDirEnv) {
936
+ this.failedPublishPayloadsDir = failedPublishPayloadsDirEnv;
937
+ }
938
+ const allPublishPayloadsDirEnv = isomorph_default.getEnv(
939
+ "BRAINTRUST_ALL_PUBLISH_PAYLOADS_DIR"
940
+ );
941
+ if (allPublishPayloadsDirEnv) {
942
+ this.allPublishPayloadsDir = allPublishPayloadsDirEnv;
943
+ }
944
+ isomorph_default.processOn("beforeExit", async () => {
945
+ await this.flush();
946
+ });
947
+ }
948
+ log(items) {
949
+ const [addedItems, droppedItems] = (() => {
950
+ if (this.queueDropExceedingMaxsize === void 0) {
951
+ return [items, []];
952
+ }
953
+ const numElementsToAdd = Math.min(
954
+ Math.max(this.queueDropExceedingMaxsize - this.items.length, 0),
955
+ items.length
956
+ );
957
+ return [items.slice(0, numElementsToAdd), items.slice(numElementsToAdd)];
958
+ })();
959
+ this.items.push(...addedItems);
960
+ if (!this.syncFlush) {
961
+ this.triggerActiveFlush();
962
+ }
963
+ if (droppedItems.length) {
964
+ this.registerDroppedItemCount(droppedItems.length);
965
+ if (this.allPublishPayloadsDir || this.failedPublishPayloadsDir) {
966
+ this.dumpDroppedEvents(droppedItems);
967
+ }
968
+ }
969
+ }
970
+ async flush() {
971
+ if (this.syncFlush) {
972
+ this.triggerActiveFlush();
973
+ }
974
+ await this.activeFlush;
975
+ if (this.activeFlushError) {
976
+ const err = this.activeFlushError;
977
+ this.activeFlushError = void 0;
978
+ throw err;
979
+ }
980
+ }
981
+ async flushOnce(args) {
982
+ const batchSize = args?.batchSize ?? this.defaultBatchSize;
983
+ const wrappedItems = this.items;
984
+ this.items = [];
985
+ const allItems = await this.unwrapLazyValues(wrappedItems);
986
+ if (allItems.length === 0) {
987
+ return;
988
+ }
989
+ const allItemsStr = allItems.map(
990
+ (bucket) => bucket.map((item) => JSON.stringify(item))
991
+ );
992
+ const batchSets = batchItems({
993
+ items: allItemsStr,
994
+ batchMaxNumItems: batchSize,
995
+ batchMaxNumBytes: this.maxRequestSize / 2
996
+ });
997
+ for (const batchSet of batchSets) {
998
+ const postPromises = batchSet.map(
999
+ (batch) => (async () => {
1000
+ try {
1001
+ await this.submitLogsRequest(batch);
1002
+ return { type: "success" };
1003
+ } catch (e) {
1004
+ return { type: "error", value: e };
1005
+ }
1006
+ })()
1007
+ );
1008
+ const results = await Promise.all(postPromises);
1009
+ const failingResultErrors = results.map((r) => r.type === "success" ? void 0 : r.value).filter((r) => r !== void 0);
1010
+ if (failingResultErrors.length) {
1011
+ throw new AggregateError(
1012
+ failingResultErrors,
1013
+ `Encountered the following errors while logging:`
1014
+ );
1015
+ }
1016
+ }
1017
+ if (this.items.length > 0) {
1018
+ await this.flushOnce(args);
1019
+ }
1020
+ }
1021
+ async unwrapLazyValues(wrappedItems) {
1022
+ for (let i = 0; i < this.numTries; ++i) {
1023
+ try {
1024
+ const itemPromises = wrappedItems.map((x) => x.get());
1025
+ return mergeRowBatch(await Promise.all(itemPromises));
1026
+ } catch (e) {
1027
+ let errmsg = "Encountered error when constructing records to flush";
1028
+ const isRetrying = i + 1 < this.numTries;
1029
+ if (isRetrying) {
1030
+ errmsg += ". Retrying";
1031
+ }
1032
+ console.warn(errmsg);
1033
+ if (!isRetrying && this.syncFlush) {
1034
+ throw e;
1035
+ } else {
1036
+ console.warn(e);
1037
+ await new Promise((resolve) => setTimeout(resolve, 100));
1038
+ }
1039
+ }
1040
+ }
1041
+ console.warn(
1042
+ `Failed to construct log records to flush after ${this.numTries} attempts. Dropping batch`
1043
+ );
1044
+ return [];
1045
+ }
1046
+ async submitLogsRequest(items) {
1047
+ const conn = await this.logConn.get();
1048
+ const dataStr = constructLogs3Data(items);
1049
+ if (this.allPublishPayloadsDir) {
1050
+ await _BackgroundLogger.writePayloadToDir({
1051
+ payloadDir: this.allPublishPayloadsDir,
1052
+ payload: dataStr
1053
+ });
1054
+ }
1055
+ for (let i = 0; i < this.numTries; i++) {
1056
+ const startTime = now();
1057
+ let error = void 0;
1058
+ try {
1059
+ await conn.post_json("logs3", dataStr);
1060
+ } catch (e) {
1061
+ try {
1062
+ const legacyDataS = constructJsonArray(
1063
+ items.map(
1064
+ (r) => JSON.stringify(makeLegacyEvent(JSON.parse(r)))
1065
+ )
1066
+ );
1067
+ await conn.post_json("logs", legacyDataS);
1068
+ } catch (e2) {
1069
+ error = e2;
1070
+ }
1071
+ }
1072
+ if (error === void 0) {
1073
+ return;
1074
+ }
1075
+ const isRetrying = i + 1 < this.numTries;
1076
+ const retryingText = isRetrying ? "" : " Retrying";
1077
+ const errorText = (() => {
1078
+ if (error instanceof FailedHTTPResponse) {
1079
+ return `${error.status} (${error.text}): ${error.data}`;
1080
+ } else {
1081
+ return `${error}`;
1082
+ }
1083
+ })();
1084
+ const errMsg = `log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${dataStr.length}.${retryingText}
1085
+ Error: ${errorText}`;
1086
+ if (!isRetrying && this.failedPublishPayloadsDir) {
1087
+ await _BackgroundLogger.writePayloadToDir({
1088
+ payloadDir: this.failedPublishPayloadsDir,
1089
+ payload: dataStr
1090
+ });
1091
+ this.logFailedPayloadsDir();
1092
+ }
1093
+ if (!isRetrying && this.syncFlush) {
1094
+ throw new Error(errMsg);
1095
+ } else {
1096
+ console.warn(errMsg);
1097
+ if (isRetrying) {
1098
+ await new Promise((resolve) => setTimeout(resolve, 100));
1099
+ }
1100
+ }
1101
+ }
1102
+ console.warn(
1103
+ `log request failed after ${this.numTries} retries. Dropping batch`
1104
+ );
1105
+ return;
1106
+ }
1107
+ registerDroppedItemCount(numItems) {
1108
+ if (numItems <= 0) {
1109
+ return;
1110
+ }
1111
+ this.queueDropLoggingState.numDropped += numItems;
1112
+ const timeNow = getCurrentUnixTimestamp();
1113
+ if (timeNow - this.queueDropLoggingState.lastLoggedTimestamp > this.queueDropLoggingPeriod) {
1114
+ console.warn(
1115
+ `Dropped ${this.queueDropLoggingState.numDropped} elements due to full queue`
1116
+ );
1117
+ if (this.failedPublishPayloadsDir) {
1118
+ this.logFailedPayloadsDir();
1119
+ }
1120
+ this.queueDropLoggingState.numDropped = 0;
1121
+ this.queueDropLoggingState.lastLoggedTimestamp = timeNow;
1122
+ }
1123
+ }
1124
+ async dumpDroppedEvents(wrappedItems) {
1125
+ const publishPayloadsDir = [
1126
+ this.allPublishPayloadsDir,
1127
+ this.failedPublishPayloadsDir
1128
+ ].reduce((acc, x) => x ? acc.concat([x]) : acc, new Array());
1129
+ if (!(wrappedItems.length && publishPayloadsDir.length)) {
1130
+ return;
1131
+ }
1132
+ try {
1133
+ const allItems = await this.unwrapLazyValues(wrappedItems);
1134
+ const dataStr = constructLogs3Data(
1135
+ allItems.map((x) => JSON.stringify(x))
1136
+ );
1137
+ for (const payloadDir of publishPayloadsDir) {
1138
+ await _BackgroundLogger.writePayloadToDir({
1139
+ payloadDir,
1140
+ payload: dataStr
1141
+ });
1142
+ }
1143
+ } catch (e) {
1144
+ console.error(e);
1145
+ }
1146
+ }
1147
+ static async writePayloadToDir({
1148
+ payloadDir,
1149
+ payload
1150
+ }) {
1151
+ if (!(isomorph_default.pathJoin && isomorph_default.mkdir && isomorph_default.writeFile)) {
1152
+ console.warn(
1153
+ "Cannot dump payloads: filesystem-operations not supported on this platform"
1154
+ );
1155
+ return;
1156
+ }
1157
+ const payloadFile = isomorph_default.pathJoin(
1158
+ payloadDir,
1159
+ `payload_${getCurrentUnixTimestamp()}_${uuidv4().slice(0, 8)}.json`
1160
+ );
1161
+ try {
1162
+ await isomorph_default.mkdir(payloadDir, { recursive: true });
1163
+ await isomorph_default.writeFile(payloadFile, payload);
1164
+ } catch (e) {
1165
+ console.error(
1166
+ `Failed to write failed payload to output file ${payloadFile}:
1167
+ `,
1168
+ e
1169
+ );
1170
+ }
1171
+ }
1172
+ triggerActiveFlush() {
1173
+ if (this.activeFlushResolved) {
1174
+ this.activeFlushResolved = false;
1175
+ this.activeFlushError = void 0;
1176
+ this.activeFlush = (async () => {
1177
+ try {
1178
+ await this.flushOnce();
1179
+ } catch (err) {
1180
+ this.activeFlushError = err;
1181
+ } finally {
1182
+ this.activeFlushResolved = true;
1183
+ }
1184
+ })();
1185
+ }
1186
+ }
1187
+ logFailedPayloadsDir() {
1188
+ console.warn(`Logging failed payloads to ${this.failedPublishPayloadsDir}`);
1189
+ }
1190
+ // Should only be called by BraintrustState.
1191
+ internalReplaceLogConn(logConn) {
1192
+ this.logConn = new LazyValue(async () => logConn);
1193
+ }
1194
+ };
1195
+ function init(projectOrOptions, optionalOptions) {
1196
+ const options = (() => {
1197
+ if (typeof projectOrOptions === "string") {
1198
+ return { ...optionalOptions, project: projectOrOptions };
1199
+ } else {
1200
+ if (optionalOptions !== void 0) {
1201
+ throw new Error(
1202
+ "Cannot specify options struct as both parameters. Must call either init(project, options) or init(options)."
1203
+ );
1204
+ }
1205
+ return projectOrOptions;
1206
+ }
1207
+ })();
1208
+ const {
1209
+ project,
1210
+ experiment,
1211
+ description,
1212
+ dataset,
1213
+ baseExperiment,
1214
+ isPublic,
1215
+ open,
1216
+ update,
1217
+ appUrl,
1218
+ apiKey,
1219
+ orgName,
1220
+ forceLogin,
1221
+ fetch,
1222
+ metadata,
1223
+ gitMetadataSettings,
1224
+ projectId,
1225
+ baseExperimentId,
1226
+ repoInfo,
1227
+ state: stateArg
1228
+ } = options;
1229
+ if (open && update) {
1230
+ throw new Error("Cannot open and update an experiment at the same time");
1231
+ }
1232
+ const state = stateArg ?? _globalState;
1233
+ if (open) {
1234
+ if (isEmpty(experiment)) {
1235
+ throw new Error(`Cannot open an experiment without specifying its name`);
1236
+ }
1237
+ const lazyMetadata2 = new LazyValue(
1238
+ async () => {
1239
+ await state.login({ apiKey, appUrl, orgName, fetch, forceLogin });
1240
+ const args = {
1241
+ project_name: project,
1242
+ project_id: projectId,
1243
+ org_name: state.orgName,
1244
+ experiment_name: experiment
1245
+ };
1246
+ const response = await state.apiConn().post_json("api/experiment/get", args);
1247
+ if (response.length === 0) {
1248
+ throw new Error(
1249
+ `Experiment ${experiment} not found in project ${projectId ?? project}.`
1250
+ );
1251
+ }
1252
+ const info = response[0];
1253
+ return {
1254
+ project: {
1255
+ id: info.project_id,
1256
+ name: project ?? "UNKNOWN_PROJECT",
1257
+ fullInfo: {}
1258
+ },
1259
+ experiment: {
1260
+ id: info.id,
1261
+ name: info.name,
1262
+ fullInfo: info
1263
+ }
1264
+ };
1265
+ }
1266
+ );
1267
+ return new ReadonlyExperiment(
1268
+ stateArg ?? _globalState,
1269
+ lazyMetadata2
1270
+ );
1271
+ }
1272
+ const lazyMetadata = new LazyValue(
1273
+ async () => {
1274
+ await state.login({ apiKey, appUrl, orgName });
1275
+ const args = {
1276
+ project_name: project,
1277
+ project_id: projectId,
1278
+ org_id: state.orgId,
1279
+ update
1280
+ };
1281
+ if (experiment) {
1282
+ args["experiment_name"] = experiment;
1283
+ }
1284
+ if (description) {
1285
+ args["description"] = description;
1286
+ }
1287
+ const repoInfoArg = await (async () => {
1288
+ if (repoInfo) {
1289
+ return repoInfo;
1290
+ }
1291
+ let mergedGitMetadataSettings = {
1292
+ ...state.gitMetadataSettings || {
1293
+ collect: "all"
1294
+ }
1295
+ };
1296
+ if (gitMetadataSettings) {
1297
+ mergedGitMetadataSettings = mergeGitMetadataSettings(
1298
+ mergedGitMetadataSettings,
1299
+ gitMetadataSettings
1300
+ );
1301
+ }
1302
+ return await isomorph_default.getRepoInfo(mergedGitMetadataSettings);
1303
+ })();
1304
+ if (repoInfoArg) {
1305
+ args["repo_info"] = repoInfoArg;
1306
+ }
1307
+ if (baseExperimentId) {
1308
+ args["base_exp_id"] = baseExperimentId;
1309
+ } else if (baseExperiment) {
1310
+ args["base_experiment"] = baseExperiment;
1311
+ } else {
1312
+ args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
1313
+ }
1314
+ if (dataset !== void 0) {
1315
+ args["dataset_id"] = await dataset.id;
1316
+ args["dataset_version"] = await dataset.version();
1317
+ }
1318
+ if (isPublic !== void 0) {
1319
+ args["public"] = isPublic;
1320
+ }
1321
+ if (metadata) {
1322
+ args["metadata"] = metadata;
1323
+ }
1324
+ let response = null;
1325
+ while (true) {
1326
+ try {
1327
+ response = await state.apiConn().post_json("api/experiment/register", args);
1328
+ break;
1329
+ } catch (e) {
1330
+ if (args["base_experiment"] && `${"data" in e && e.data}`.includes("base experiment")) {
1331
+ console.warn(
1332
+ `Base experiment ${args["base_experiment"]} not found.`
1333
+ );
1334
+ delete args["base_experiment"];
1335
+ } else {
1336
+ throw e;
1337
+ }
1338
+ }
1339
+ }
1340
+ return {
1341
+ project: {
1342
+ id: response.project.id,
1343
+ name: response.project.name,
1344
+ fullInfo: response.project
1345
+ },
1346
+ experiment: {
1347
+ id: response.experiment.id,
1348
+ name: response.experiment.name,
1349
+ fullInfo: response.experiment
1350
+ }
1351
+ };
1352
+ }
1353
+ );
1354
+ const ret = new Experiment(state, lazyMetadata, dataset);
1355
+ if (options.setCurrent ?? true) {
1356
+ state.currentExperiment = ret;
1357
+ }
1358
+ return ret;
1359
+ }
1360
+ function initExperiment(projectOrOptions, optionalOptions) {
1361
+ const options = (() => {
1362
+ if (typeof projectOrOptions === "string") {
1363
+ return { ...optionalOptions, project: projectOrOptions };
1364
+ } else {
1365
+ if (optionalOptions !== void 0) {
1366
+ throw new Error(
1367
+ "Cannot specify options struct as both parameters. Must call either init(project, options) or init(options)."
1368
+ );
1369
+ }
1370
+ return projectOrOptions;
1371
+ }
1372
+ })();
1373
+ return init(options);
1374
+ }
1375
+ function withExperiment(project, callback, options = {}) {
1376
+ console.warn(
1377
+ "withExperiment is deprecated and will be removed in a future version of braintrust. Simply create the experiment with `init`."
1378
+ );
1379
+ const experiment = init(project, options);
1380
+ return callback(experiment);
1381
+ }
1382
+ function withLogger(callback, options = {}) {
1383
+ console.warn(
1384
+ "withLogger is deprecated and will be removed in a future version of braintrust. Simply create the logger with `initLogger`."
1385
+ );
1386
+ const logger = initLogger(options);
1387
+ return callback(logger);
1388
+ }
1389
+ function initDataset(projectOrOptions, optionalOptions) {
1390
+ const options = (() => {
1391
+ if (typeof projectOrOptions === "string") {
1392
+ return { ...optionalOptions, project: projectOrOptions };
1393
+ } else {
1394
+ if (optionalOptions !== void 0) {
1395
+ throw new Error(
1396
+ "Cannot specify options struct as both parameters. Must call either initDataset(project, options) or initDataset(options)."
1397
+ );
1398
+ }
1399
+ return projectOrOptions;
1400
+ }
1401
+ })();
1402
+ const {
1403
+ project,
1404
+ dataset,
1405
+ description,
1406
+ version,
1407
+ appUrl,
1408
+ apiKey,
1409
+ orgName,
1410
+ fetch,
1411
+ forceLogin,
1412
+ projectId,
1413
+ useOutput: legacy,
1414
+ state: stateArg
1415
+ } = options;
1416
+ const state = stateArg ?? _globalState;
1417
+ const lazyMetadata = new LazyValue(
1418
+ async () => {
1419
+ await state.login({
1420
+ orgName,
1421
+ apiKey,
1422
+ appUrl,
1423
+ fetch,
1424
+ forceLogin
1425
+ });
1426
+ const args = {
1427
+ org_id: state.orgId,
1428
+ project_name: project,
1429
+ project_id: projectId,
1430
+ dataset_name: dataset,
1431
+ description
1432
+ };
1433
+ const response = await state.apiConn().post_json("api/dataset/register", args);
1434
+ return {
1435
+ project: {
1436
+ id: response.project.id,
1437
+ name: response.project.name,
1438
+ fullInfo: response.project
1439
+ },
1440
+ dataset: {
1441
+ id: response.dataset.id,
1442
+ name: response.dataset.name,
1443
+ fullInfo: response.dataset
1444
+ }
1445
+ };
1446
+ }
1447
+ );
1448
+ return new Dataset(stateArg ?? _globalState, lazyMetadata, version, legacy);
1449
+ }
1450
+ function withDataset(project, callback, options = {}) {
1451
+ console.warn(
1452
+ "withDataset is deprecated and will be removed in a future version of braintrust. Simply create the dataset with `initDataset`."
1453
+ );
1454
+ const dataset = initDataset(project, options);
1455
+ return callback(dataset);
1456
+ }
1457
+ async function computeLoggerMetadata(state, {
1458
+ project_name,
1459
+ project_id
1460
+ }) {
1461
+ const org_id = state.orgId;
1462
+ if (isEmpty(project_id)) {
1463
+ const response = await state.apiConn().post_json("api/project/register", {
1464
+ project_name: project_name || GLOBAL_PROJECT,
1465
+ org_id
1466
+ });
1467
+ return {
1468
+ org_id,
1469
+ project: {
1470
+ id: response.project.id,
1471
+ name: response.project.name,
1472
+ fullInfo: response.project
1473
+ }
1474
+ };
1475
+ } else if (isEmpty(project_name)) {
1476
+ const response = await state.apiConn().get_json("api/project", {
1477
+ id: project_id
1478
+ });
1479
+ return {
1480
+ org_id,
1481
+ project: {
1482
+ id: project_id,
1483
+ name: response.name,
1484
+ fullInfo: response.project
1485
+ }
1486
+ };
1487
+ } else {
1488
+ return {
1489
+ org_id,
1490
+ project: { id: project_id, name: project_name, fullInfo: {} }
1491
+ };
1492
+ }
1493
+ }
1494
+ function initLogger(options = {}) {
1495
+ const {
1496
+ projectName,
1497
+ projectId,
1498
+ asyncFlush,
1499
+ appUrl,
1500
+ apiKey,
1501
+ orgName,
1502
+ forceLogin,
1503
+ fetch,
1504
+ state: stateArg
1505
+ } = options || {};
1506
+ const computeMetadataArgs = {
1507
+ project_name: projectName,
1508
+ project_id: projectId
1509
+ };
1510
+ const state = stateArg ?? _globalState;
1511
+ const lazyMetadata = new LazyValue(
1512
+ async () => {
1513
+ await state.login({
1514
+ orgName,
1515
+ apiKey,
1516
+ appUrl,
1517
+ forceLogin,
1518
+ fetch
1519
+ });
1520
+ return computeLoggerMetadata(state, computeMetadataArgs);
1521
+ }
1522
+ );
1523
+ const ret = new Logger(state, lazyMetadata, {
1524
+ asyncFlush,
1525
+ computeMetadataArgs
1526
+ });
1527
+ if (options.setCurrent ?? true) {
1528
+ state.currentLogger = ret;
1529
+ }
1530
+ return ret;
1531
+ }
1532
+ async function loadPrompt({
1533
+ projectName,
1534
+ projectId,
1535
+ slug,
1536
+ version,
1537
+ defaults,
1538
+ noTrace = false,
1539
+ appUrl,
1540
+ apiKey,
1541
+ orgName,
1542
+ fetch,
1543
+ forceLogin,
1544
+ state: stateArg
1545
+ }) {
1546
+ if (isEmpty(projectName) && isEmpty(projectId)) {
1547
+ throw new Error("Must specify either projectName or projectId");
1548
+ }
1549
+ if (isEmpty(slug)) {
1550
+ throw new Error("Must specify slug");
1551
+ }
1552
+ const state = stateArg ?? _globalState;
1553
+ state.login({
1554
+ orgName,
1555
+ apiKey,
1556
+ appUrl,
1557
+ fetch,
1558
+ forceLogin
1559
+ });
1560
+ const args = {
1561
+ project_name: projectName,
1562
+ project_id: projectId,
1563
+ slug,
1564
+ version
1565
+ };
1566
+ const response = await state.logConn().get_json("v1/prompt", args);
1567
+ if (!("objects" in response) || response.objects.length === 0) {
1568
+ throw new Error(
1569
+ `Prompt ${slug} not found in ${[projectName ?? projectId]}`
1570
+ );
1571
+ } else if (response.objects.length > 1) {
1572
+ throw new Error(
1573
+ `Multiple prompts found with slug ${slug} in project ${projectName ?? projectId}. This should never happen.`
1574
+ );
1575
+ }
1576
+ const metadata = promptSchema.parse(response["objects"][0]);
1577
+ return new Prompt(metadata, defaults || {}, noTrace);
1578
+ }
1579
+ async function login(options = {}) {
1580
+ let { forceLogin = false } = options || {};
1581
+ if (_globalState.loggedIn && !forceLogin) {
1582
+ let checkUpdatedParam2 = function(varname, arg, orig) {
1583
+ if (!isEmpty(arg) && !isEmpty(orig) && arg !== orig) {
1584
+ throw new Error(
1585
+ `Re-logging in with different ${varname} (${arg}) than original (${orig}). To force re-login, pass \`forceLogin: true\``
1586
+ );
1587
+ }
1588
+ };
1589
+ var checkUpdatedParam = checkUpdatedParam2;
1590
+ checkUpdatedParam2("appUrl", options.appUrl, _globalState.appUrl);
1591
+ checkUpdatedParam2(
1592
+ "apiKey",
1593
+ options.apiKey ? HTTPConnection.sanitize_token(options.apiKey) : void 0,
1594
+ _globalState.loginToken
1595
+ );
1596
+ checkUpdatedParam2("orgName", options.orgName, _globalState.orgName);
1597
+ return _globalState;
1598
+ }
1599
+ await _globalState.login(options);
1600
+ globalThis.__inherited_braintrust_state = _globalState;
1601
+ return _globalState;
1602
+ }
1603
+ async function loginToState(options = {}) {
1604
+ const {
1605
+ appUrl = isomorph_default.getEnv("BRAINTRUST_APP_URL") || "https://www.braintrust.dev",
1606
+ apiKey = isomorph_default.getEnv("BRAINTRUST_API_KEY"),
1607
+ orgName = isomorph_default.getEnv("BRAINTRUST_ORG_NAME"),
1608
+ fetch = globalThis.fetch
1609
+ } = options || {};
1610
+ const appPublicUrl = isomorph_default.getEnv("BRAINTRUST_APP_PUBLIC_URL") || appUrl;
1611
+ const state = new BraintrustState(options);
1612
+ state.resetLoginInfo();
1613
+ state.appUrl = appUrl;
1614
+ state.appPublicUrl = appPublicUrl;
1615
+ let conn = null;
1616
+ if (apiKey !== void 0) {
1617
+ const resp = await checkResponse(
1618
+ await fetch(_urljoin(state.appUrl, `/api/apikey/login`), {
1619
+ method: "POST",
1620
+ headers: {
1621
+ "Content-Type": "application/json",
1622
+ Authorization: `Bearer ${apiKey}`
1623
+ }
1624
+ })
1625
+ );
1626
+ const info = await resp.json();
1627
+ _check_org_info(state, info.org_info, orgName);
1628
+ conn = state.logConn();
1629
+ conn.set_token(apiKey);
1630
+ } else {
1631
+ throw new Error(
1632
+ "Please specify an api key (e.g. by setting BRAINTRUST_API_KEY)."
1633
+ );
1634
+ }
1635
+ if (!conn) {
1636
+ throw new Error("Conn should be set at this point (a bug)");
1637
+ }
1638
+ conn.make_long_lived();
1639
+ state.apiConn().set_token(apiKey);
1640
+ state.proxyConn().set_token(apiKey);
1641
+ state.loginToken = conn.token;
1642
+ state.loggedIn = true;
1643
+ state.loginReplaceLogConn(conn);
1644
+ return state;
1645
+ }
1646
+ function log(event) {
1647
+ console.warn(
1648
+ "braintrust.log is deprecated and will be removed in a future version of braintrust. Use `experiment.log` instead."
1649
+ );
1650
+ const e = currentExperiment();
1651
+ if (!e) {
1652
+ throw new Error("Not initialized. Please call init() first");
1653
+ }
1654
+ return e.log(event);
1655
+ }
1656
+ async function summarize(options = {}) {
1657
+ console.warn(
1658
+ "braintrust.summarize is deprecated and will be removed in a future version of braintrust. Use `experiment.summarize` instead."
1659
+ );
1660
+ const e = currentExperiment();
1661
+ if (!e) {
1662
+ throw new Error("Not initialized. Please call init() first");
1663
+ }
1664
+ return await e.summarize(options);
1665
+ }
1666
+ function currentExperiment(options) {
1667
+ const state = options?.state ?? _globalState;
1668
+ return state.currentExperiment;
1669
+ }
1670
+ function currentLogger(options) {
1671
+ const state = options?.state ?? _globalState;
1672
+ return castLogger(state.currentLogger, options?.asyncFlush);
1673
+ }
1674
+ function currentSpan(options) {
1675
+ const state = options?.state ?? _globalState;
1676
+ return state.currentSpan.getStore() ?? NOOP_SPAN;
1677
+ }
1678
+ function getSpanParentObject(options) {
1679
+ const state = options?.state ?? _globalState;
1680
+ const parentSpan = currentSpan({ state });
1681
+ if (!Object.is(parentSpan, NOOP_SPAN)) {
1682
+ return parentSpan;
1683
+ }
1684
+ const experiment = currentExperiment();
1685
+ if (experiment) {
1686
+ return experiment;
1687
+ }
1688
+ const logger = currentLogger(options);
1689
+ if (logger) {
1690
+ return logger;
1691
+ }
1692
+ return NOOP_SPAN;
1693
+ }
1694
+ function traced(callback, args) {
1695
+ const { span, isLogger } = startSpanAndIsLogger(args);
1696
+ const ret = runFinally(
1697
+ () => {
1698
+ if (args?.setCurrent ?? true) {
1699
+ return withCurrent(span, callback);
1700
+ } else {
1701
+ return callback(span);
1702
+ }
1703
+ },
1704
+ () => span.end()
1705
+ );
1706
+ if (args?.asyncFlush) {
1707
+ return ret;
1708
+ } else {
1709
+ return (async () => {
1710
+ const awaitedRet = await ret;
1711
+ if (isLogger) {
1712
+ await span.flush();
1713
+ }
1714
+ return awaitedRet;
1715
+ })();
1716
+ }
1717
+ }
1718
+ function wrapTraced(fn, args) {
1719
+ const spanArgs = {
1720
+ name: fn.name,
1721
+ type: "function",
1722
+ ...args
1723
+ };
1724
+ const hasExplicitInput = args && args.event && "input" in args.event && args.event.input !== void 0;
1725
+ const hasExplicitOutput = args && args.event && args.event.output !== void 0;
1726
+ if (args?.asyncFlush) {
1727
+ return (...fnArgs) => traced((span) => {
1728
+ if (!hasExplicitInput) {
1729
+ span.log({ input: fnArgs });
1730
+ }
1731
+ const output = fn(...fnArgs);
1732
+ if (!hasExplicitOutput) {
1733
+ if (output instanceof Promise) {
1734
+ return (async () => {
1735
+ const result = await output;
1736
+ span.log({ output: result });
1737
+ return result;
1738
+ })();
1739
+ } else {
1740
+ span.log({ output });
1741
+ }
1742
+ }
1743
+ return output;
1744
+ }, spanArgs);
1745
+ } else {
1746
+ return (...fnArgs) => traced(async (span) => {
1747
+ if (!hasExplicitInput) {
1748
+ span.log({ input: fnArgs });
1749
+ }
1750
+ const outputResult = fn(...fnArgs);
1751
+ const output = await outputResult;
1752
+ if (!hasExplicitOutput) {
1753
+ span.log({ output });
1754
+ }
1755
+ return output;
1756
+ }, spanArgs);
1757
+ }
1758
+ }
1759
+ var traceable = wrapTraced;
1760
+ function startSpan(args) {
1761
+ return startSpanAndIsLogger(args).span;
1762
+ }
1763
+ async function flush(options) {
1764
+ const state = options?.state ?? _globalState;
1765
+ return await state.bgLogger().flush();
1766
+ }
1767
+ function setFetch(fetch) {
1768
+ _globalState.setFetch(fetch);
1769
+ }
1770
+ function startSpanAndIsLogger(args) {
1771
+ const state = args?.state ?? _globalState;
1772
+ if (args?.parent) {
1773
+ const components = SpanComponentsV2.fromStr(args?.parent);
1774
+ const parentSpanIds = components.rowIds ? {
1775
+ spanId: components.rowIds.spanId,
1776
+ rootSpanId: components.rowIds.rootSpanId
1777
+ } : void 0;
1778
+ const span = new SpanImpl({
1779
+ state,
1780
+ ...args,
1781
+ parentObjectType: components.objectType,
1782
+ parentObjectId: new LazyValue(
1783
+ spanComponentsToObjectIdLambda(state, components)
1784
+ ),
1785
+ parentComputeObjectMetadataArgs: components.computeObjectMetadataArgs,
1786
+ parentSpanIds
1787
+ });
1788
+ return {
1789
+ span,
1790
+ isLogger: components.objectType === SpanObjectTypeV2.PROJECT_LOGS
1791
+ };
1792
+ } else {
1793
+ const parentObject = getSpanParentObject({
1794
+ asyncFlush: args?.asyncFlush
1795
+ });
1796
+ const span = parentObject.startSpan(args);
1797
+ return { span, isLogger: parentObject.kind === "logger" };
1798
+ }
1799
+ }
1800
+ function withCurrent(span, callback, state = _globalState) {
1801
+ return state.currentSpan.run(span, () => callback(span));
1802
+ }
1803
+ function _check_org_info(state, org_info, org_name) {
1804
+ if (org_info.length === 0) {
1805
+ throw new Error("This user is not part of any organizations.");
1806
+ }
1807
+ for (const org of org_info) {
1808
+ if (org_name === void 0 || org.name === org_name) {
1809
+ state.orgId = org.id;
1810
+ state.orgName = org.name;
1811
+ state.logUrl = isomorph_default.getEnv("BRAINTRUST_API_URL") ?? org.api_url;
1812
+ state.proxyUrl = isomorph_default.getEnv("BRAINTRUST_PROXY_URL") ?? org.proxy_url;
1813
+ if (state.proxyUrl) {
1814
+ const url = new URL(state.proxyUrl);
1815
+ url.pathname = "";
1816
+ state.proxyUrl = url.toString();
1817
+ }
1818
+ state.gitMetadataSettings = org.git_metadata || void 0;
1819
+ break;
1820
+ }
1821
+ }
1822
+ if (state.orgId === void 0) {
1823
+ throw new Error(
1824
+ `Organization ${org_name} not found. Must be one of ${org_info.map((x) => x.name).join(", ")}`
1825
+ );
1826
+ }
1827
+ }
1828
+ function _urljoin(...parts) {
1829
+ return parts.map(
1830
+ (x, i) => x.replace(/^\//, "").replace(i < parts.length - 1 ? /\/$/ : "", "")
1831
+ ).join("/");
1832
+ }
1833
+ function validateTags(tags) {
1834
+ const seen = /* @__PURE__ */ new Set();
1835
+ for (const tag of tags) {
1836
+ if (typeof tag !== "string") {
1837
+ throw new Error("tags must be strings");
1838
+ }
1839
+ if (seen.has(tag)) {
1840
+ throw new Error(`duplicate tag: ${tag}`);
1841
+ }
1842
+ }
1843
+ }
1844
+ function validateAndSanitizeExperimentLogPartialArgs(event) {
1845
+ if (event.scores) {
1846
+ if (Array.isArray(event.scores)) {
1847
+ throw new Error("scores must be an object, not an array");
1848
+ }
1849
+ for (let [name, score] of Object.entries(event.scores)) {
1850
+ if (typeof name !== "string") {
1851
+ throw new Error("score names must be strings");
1852
+ }
1853
+ if (score === null || score === void 0) {
1854
+ continue;
1855
+ }
1856
+ if (typeof score === "boolean") {
1857
+ score = score ? 1 : 0;
1858
+ event.scores[name] = score;
1859
+ }
1860
+ if (typeof score !== "number") {
1861
+ throw new Error("score values must be numbers");
1862
+ }
1863
+ if (score < 0 || score > 1) {
1864
+ throw new Error("score values must be between 0 and 1");
1865
+ }
1866
+ }
1867
+ }
1868
+ if (event.metadata) {
1869
+ for (const key of Object.keys(event.metadata)) {
1870
+ if (typeof key !== "string") {
1871
+ throw new Error("metadata keys must be strings");
1872
+ }
1873
+ }
1874
+ }
1875
+ if (event.metrics) {
1876
+ for (const [key, value] of Object.entries(event.metrics)) {
1877
+ if (typeof key !== "string") {
1878
+ throw new Error("metric keys must be strings");
1879
+ }
1880
+ if (value !== void 0 && typeof value !== "number") {
1881
+ throw new Error("metric values must be numbers");
1882
+ }
1883
+ }
1884
+ }
1885
+ if ("input" in event && event.input && "inputs" in event && event.inputs) {
1886
+ throw new Error(
1887
+ "Only one of input or inputs (deprecated) can be specified. Prefer input."
1888
+ );
1889
+ }
1890
+ if ("tags" in event && event.tags) {
1891
+ validateTags(event.tags);
1892
+ }
1893
+ if ("inputs" in event) {
1894
+ const { inputs, ...rest } = event;
1895
+ return { input: inputs, ...rest };
1896
+ } else {
1897
+ return { ...event };
1898
+ }
1899
+ }
1900
+ function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
1901
+ if ("input" in event && !isEmpty(event.input) && "inputs" in event && !isEmpty(event.inputs) || !("input" in event) && !("inputs" in event)) {
1902
+ throw new Error(
1903
+ "Exactly one of input or inputs (deprecated) must be specified. Prefer input."
1904
+ );
1905
+ }
1906
+ if (isEmpty(event.output)) {
1907
+ throw new Error("output must be specified");
1908
+ }
1909
+ if (isEmpty(event.scores)) {
1910
+ throw new Error("scores must be specified");
1911
+ }
1912
+ if (hasDataset && event.datasetRecordId === void 0) {
1913
+ throw new Error("datasetRecordId must be specified when using a dataset");
1914
+ } else if (!hasDataset && event.datasetRecordId !== void 0) {
1915
+ throw new Error(
1916
+ "datasetRecordId cannot be specified when not using a dataset"
1917
+ );
1918
+ }
1919
+ return event;
1920
+ }
1921
+ var ObjectFetcher = class {
1922
+ constructor(objectType, pinnedVersion, mutateRecord) {
1923
+ this.objectType = objectType;
1924
+ this.pinnedVersion = pinnedVersion;
1925
+ this.mutateRecord = mutateRecord;
1926
+ }
1927
+ _fetchedData = void 0;
1928
+ get id() {
1929
+ throw new Error("ObjectFetcher subclasses must have an 'id' attribute");
1930
+ }
1931
+ async getState() {
1932
+ throw new Error("ObjectFetcher subclasses must have a 'getState' method");
1933
+ }
1934
+ async *fetch() {
1935
+ const records = await this.fetchedData();
1936
+ for (const record of records) {
1937
+ yield record;
1938
+ }
1939
+ }
1940
+ [Symbol.asyncIterator]() {
1941
+ return this.fetch();
1942
+ }
1943
+ async fetchedData() {
1944
+ if (this._fetchedData === void 0) {
1945
+ const state = await this.getState();
1946
+ const resp = await state.logConn().get(
1947
+ `v1/${this.objectType}/${await this.id}/fetch`,
1948
+ {
1949
+ version: this.pinnedVersion
1950
+ },
1951
+ { headers: { "Accept-Encoding": "gzip" } }
1952
+ );
1953
+ const data = (await resp.json()).events;
1954
+ this._fetchedData = this.mutateRecord ? data?.map(this.mutateRecord) : data;
1955
+ }
1956
+ return this._fetchedData || [];
1957
+ }
1958
+ clearCache() {
1959
+ this._fetchedData = void 0;
1960
+ }
1961
+ async version() {
1962
+ if (this.pinnedVersion !== void 0) {
1963
+ return this.pinnedVersion;
1964
+ } else {
1965
+ const fetchedData = await this.fetchedData();
1966
+ let maxVersion = void 0;
1967
+ for (const record of fetchedData) {
1968
+ const xactId = String(record[TRANSACTION_ID_FIELD] ?? "0");
1969
+ if (maxVersion === void 0 || xactId > maxVersion) {
1970
+ maxVersion = xactId;
1971
+ }
1972
+ }
1973
+ return maxVersion;
1974
+ }
1975
+ }
1976
+ };
1977
+ var Experiment = class extends ObjectFetcher {
1978
+ lazyMetadata;
1979
+ dataset;
1980
+ lastStartTime;
1981
+ lazyId;
1982
+ calledStartSpan;
1983
+ state;
1984
+ // For type identification.
1985
+ kind = "experiment";
1986
+ constructor(state, lazyMetadata, dataset) {
1987
+ super("experiment", void 0);
1988
+ this.lazyMetadata = lazyMetadata;
1989
+ this.dataset = dataset;
1990
+ this.lastStartTime = getCurrentUnixTimestamp();
1991
+ this.lazyId = new LazyValue(async () => await this.id);
1992
+ this.calledStartSpan = false;
1993
+ this.state = state;
1994
+ }
1995
+ get id() {
1996
+ return (async () => {
1997
+ return (await this.lazyMetadata.get()).experiment.id;
1998
+ })();
1999
+ }
2000
+ get name() {
2001
+ return (async () => {
2002
+ return (await this.lazyMetadata.get()).experiment.name;
2003
+ })();
2004
+ }
2005
+ get project() {
2006
+ return (async () => {
2007
+ return (await this.lazyMetadata.get()).project;
2008
+ })();
2009
+ }
2010
+ parentObjectType() {
2011
+ return SpanObjectTypeV2.EXPERIMENT;
2012
+ }
2013
+ async getState() {
2014
+ await this.lazyMetadata.get();
2015
+ return this.state;
2016
+ }
2017
+ /**
2018
+ * Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
2019
+ *
2020
+ * @param event The event to log.
2021
+ * @param event.input: The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on, Braintrust will use the `input` to know whether two test cases are the same between experiments, so they should not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the `input` should be identical.
2022
+ * @param event.output: The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question.
2023
+ * @param event.expected: (Optional) The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models.
2024
+ * @param event.scores: A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
2025
+ * @param event.metadata: (Optional) a dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings.
2026
+ * @param event.metrics: (Optional) a dictionary of metrics to log. The following keys are populated automatically: "start", "end".
2027
+ * @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
2028
+ * @param event.dataset_record_id: (Optional) the id of the dataset record that this event is associated with. This field is required if and only if the experiment is associated with a dataset.
2029
+ * @param event.inputs: (Deprecated) the same as `input` (will be removed in a future version).
2030
+ * @param options Additional logging options
2031
+ * @param options.allowConcurrentWithSpans in rare cases where you need to log at the top level separately from spans on the experiment elsewhere, set this to true.
2032
+ * :returns: The `id` of the logged event.
2033
+ */
2034
+ log(event, options) {
2035
+ if (this.calledStartSpan && !options?.allowConcurrentWithSpans) {
2036
+ throw new Error(
2037
+ "Cannot run toplevel `log` method while using spans. To log to the span, call `experiment.traced` and then log with `span.log`"
2038
+ );
2039
+ }
2040
+ event = validateAndSanitizeExperimentLogFullArgs(event, !!this.dataset);
2041
+ const span = this.startSpanImpl({ startTime: this.lastStartTime, event });
2042
+ this.lastStartTime = span.end();
2043
+ return span.id;
2044
+ }
2045
+ /**
2046
+ * Create a new toplevel span underneath the experiment. The name defaults to "root".
2047
+ *
2048
+ * See `Span.traced` for full details.
2049
+ */
2050
+ traced(callback, args) {
2051
+ const { setCurrent, ...argsRest } = args ?? {};
2052
+ const span = this.startSpan(argsRest);
2053
+ return runFinally(
2054
+ () => {
2055
+ if (setCurrent ?? true) {
2056
+ return withCurrent(span, callback);
2057
+ } else {
2058
+ return callback(span);
2059
+ }
2060
+ },
2061
+ () => span.end()
2062
+ );
2063
+ }
2064
+ /**
2065
+ * Lower-level alternative to `traced`. This allows you to start a span yourself, and can be useful in situations
2066
+ * where you cannot use callbacks. However, spans started with `startSpan` will not be marked as the "current span",
2067
+ * so `currentSpan()` and `traced()` will be no-ops. If you want to mark a span as current, use `traced` instead.
2068
+ *
2069
+ * See `traced` for full details.
2070
+ */
2071
+ startSpan(args) {
2072
+ this.calledStartSpan = true;
2073
+ return this.startSpanImpl(args);
2074
+ }
2075
+ startSpanImpl(args) {
2076
+ return new SpanImpl({
2077
+ state: this.state,
2078
+ ...startSpanParentArgs({
2079
+ state: this.state,
2080
+ parent: args?.parent,
2081
+ parentObjectType: this.parentObjectType(),
2082
+ parentObjectId: this.lazyId,
2083
+ parentComputeObjectMetadataArgs: void 0,
2084
+ parentSpanIds: void 0
2085
+ }),
2086
+ ...args,
2087
+ defaultRootType: SpanTypeAttribute.EVAL
2088
+ });
2089
+ }
2090
+ async fetchBaseExperiment() {
2091
+ const state = await this.getState();
2092
+ const conn = state.apiConn();
2093
+ try {
2094
+ const resp = await conn.post("/api/base_experiment/get_id", {
2095
+ id: await this.id
2096
+ });
2097
+ const base = await resp.json();
2098
+ return {
2099
+ id: base["base_exp_id"],
2100
+ name: base["base_exp_name"]
2101
+ };
2102
+ } catch (e) {
2103
+ if (e instanceof FailedHTTPResponse && e.status === 400) {
2104
+ return null;
2105
+ } else {
2106
+ throw e;
2107
+ }
2108
+ }
2109
+ }
2110
+ /**
2111
+ * Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
2112
+ *
2113
+ * @param options Options for summarizing the experiment.
2114
+ * @param options.summarizeScores Whether to summarize the scores. If False, only the metadata will be returned.
2115
+ * @param options.comparisonExperimentId The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
2116
+ * @returns A summary of the experiment, including the scores (compared to the closest reference experiment) and metadata.
2117
+ */
2118
+ async summarize(options = {}) {
2119
+ let { summarizeScores = true, comparisonExperimentId = void 0 } = options || {};
2120
+ await this.flush();
2121
+ const state = await this.getState();
2122
+ const projectUrl = `${state.appPublicUrl}/app/${encodeURIComponent(
2123
+ state.orgName
2124
+ )}/p/${encodeURIComponent((await this.project).name)}`;
2125
+ const experimentUrl = `${projectUrl}/experiments/${encodeURIComponent(
2126
+ await this.name
2127
+ )}`;
2128
+ let scores = void 0;
2129
+ let metrics = void 0;
2130
+ let comparisonExperimentName = void 0;
2131
+ if (summarizeScores) {
2132
+ if (comparisonExperimentId === void 0) {
2133
+ const baseExperiment = await this.fetchBaseExperiment();
2134
+ if (baseExperiment !== null) {
2135
+ comparisonExperimentId = baseExperiment.id;
2136
+ comparisonExperimentName = baseExperiment.name;
2137
+ }
2138
+ }
2139
+ const results = await state.logConn().get_json(
2140
+ "/experiment-comparison2",
2141
+ {
2142
+ experiment_id: await this.id,
2143
+ base_experiment_id: comparisonExperimentId
2144
+ },
2145
+ 3
2146
+ );
2147
+ scores = results["scores"];
2148
+ metrics = results["metrics"];
2149
+ }
2150
+ return {
2151
+ projectName: (await this.project).name,
2152
+ experimentName: await this.name,
2153
+ projectId: (await this.project).id,
2154
+ experimentId: await this.id,
2155
+ projectUrl,
2156
+ experimentUrl,
2157
+ comparisonExperimentName,
2158
+ scores: scores ?? {},
2159
+ metrics
2160
+ };
2161
+ }
2162
+ /**
2163
+ * Log feedback to an event in the experiment. Feedback is used to save feedback scores, set an expected value, or add a comment.
2164
+ *
2165
+ * @param event
2166
+ * @param event.id The id of the event to log feedback for. This is the `id` returned by `log` or accessible as the `id` field of a span.
2167
+ * @param event.scores (Optional) a dictionary of numeric values (between 0 and 1) to log. These scores will be merged into the existing scores for the event.
2168
+ * @param event.expected (Optional) the ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not.
2169
+ * @param event.comment (Optional) an optional comment string to log about the event.
2170
+ * @param event.metadata (Optional) a dictionary with additional data about the feedback. If you have a `user_id`, you can log it here and access it in the Braintrust UI.
2171
+ * @param event.source (Optional) the source of the feedback. Must be one of "external" (default), "app", or "api".
2172
+ */
2173
+ logFeedback(event) {
2174
+ logFeedbackImpl(this.state, this.parentObjectType(), this.lazyId, event);
2175
+ }
2176
+ /**
2177
+ * Return a serialized representation of the experiment that can be used to start subspans in other places. See `Span.start_span` for more details.
2178
+ */
2179
+ async export() {
2180
+ return new SpanComponentsV2({
2181
+ objectType: this.parentObjectType(),
2182
+ objectId: await this.id
2183
+ }).toStr();
2184
+ }
2185
+ /**
2186
+ * Flush any pending rows to the server.
2187
+ */
2188
+ async flush() {
2189
+ return await this.state.bgLogger().flush();
2190
+ }
2191
+ /**
2192
+ * This function is deprecated. You can simply remove it from your code.
2193
+ */
2194
+ async close() {
2195
+ console.warn(
2196
+ "close is deprecated and will be removed in a future version of braintrust. It is now a no-op and can be removed"
2197
+ );
2198
+ return this.id;
2199
+ }
2200
+ };
2201
+ var ReadonlyExperiment = class extends ObjectFetcher {
2202
+ constructor(state, lazyMetadata) {
2203
+ super("experiment", void 0);
2204
+ this.state = state;
2205
+ this.lazyMetadata = lazyMetadata;
2206
+ }
2207
+ get id() {
2208
+ return (async () => {
2209
+ return (await this.lazyMetadata.get()).experiment.id;
2210
+ })();
2211
+ }
2212
+ get name() {
2213
+ return (async () => {
2214
+ return (await this.lazyMetadata.get()).experiment.name;
2215
+ })();
2216
+ }
2217
+ async getState() {
2218
+ await this.lazyMetadata.get();
2219
+ return this.state;
2220
+ }
2221
+ async *asDataset() {
2222
+ const records = this.fetch();
2223
+ for await (const record of records) {
2224
+ if (record.root_span_id !== record.span_id) {
2225
+ continue;
2226
+ }
2227
+ const { output, expected: expectedRecord } = record;
2228
+ const expected = expectedRecord ?? output;
2229
+ if (isEmpty(expected)) {
2230
+ yield {
2231
+ input: record.input,
2232
+ tags: record.tags
2233
+ };
2234
+ } else {
2235
+ yield {
2236
+ input: record.input,
2237
+ expected,
2238
+ tags: record.tags
2239
+ };
2240
+ }
2241
+ }
2242
+ }
2243
+ };
2244
+ var executionCounter = 0;
2245
+ function newId() {
2246
+ return uuidv4();
2247
+ }
2248
+ var SpanImpl = class _SpanImpl {
2249
+ state;
2250
+ // `internalData` contains fields that are not part of the "user-sanitized"
2251
+ // set of fields which we want to log in just one of the span rows.
2252
+ isMerge;
2253
+ loggedEndTime;
2254
+ // For internal use only.
2255
+ parentObjectType;
2256
+ parentObjectId;
2257
+ parentComputeObjectMetadataArgs;
2258
+ _id;
2259
+ spanId;
2260
+ rootSpanId;
2261
+ spanParents;
2262
+ kind = "span";
2263
+ constructor(args) {
2264
+ this.state = args.state;
2265
+ const spanAttributes = args.spanAttributes ?? {};
2266
+ const event = args.event ?? {};
2267
+ const type = args.type ?? (args.parentSpanIds ? void 0 : args.defaultRootType);
2268
+ this.loggedEndTime = void 0;
2269
+ this.parentObjectType = args.parentObjectType;
2270
+ this.parentObjectId = args.parentObjectId;
2271
+ this.parentComputeObjectMetadataArgs = args.parentComputeObjectMetadataArgs;
2272
+ const callerLocation = isomorph_default.getCallerLocation();
2273
+ const name = (() => {
2274
+ if (args.name)
2275
+ return args.name;
2276
+ if (!args.parentSpanIds)
2277
+ return "root";
2278
+ if (callerLocation) {
2279
+ const pathComponents = callerLocation.caller_filename.split("/");
2280
+ const filename = pathComponents[pathComponents.length - 1];
2281
+ return [callerLocation.caller_functionname].concat(
2282
+ filename ? [`${filename}:${callerLocation.caller_lineno}`] : []
2283
+ ).join(":");
2284
+ }
2285
+ return "subspan";
2286
+ })();
2287
+ const internalData = {
2288
+ metrics: {
2289
+ start: args.startTime ?? getCurrentUnixTimestamp()
2290
+ },
2291
+ context: { ...callerLocation },
2292
+ span_attributes: {
2293
+ name,
2294
+ type,
2295
+ ...spanAttributes,
2296
+ exec_counter: executionCounter++
2297
+ },
2298
+ created: (/* @__PURE__ */ new Date()).toISOString()
2299
+ };
2300
+ this._id = event.id ?? uuidv4();
2301
+ this.spanId = uuidv4();
2302
+ if (args.parentSpanIds) {
2303
+ this.rootSpanId = args.parentSpanIds.rootSpanId;
2304
+ this.spanParents = [args.parentSpanIds.spanId];
2305
+ } else {
2306
+ this.rootSpanId = this.spanId;
2307
+ this.spanParents = void 0;
2308
+ }
2309
+ this.isMerge = false;
2310
+ const { id: _id, ...eventRest } = event;
2311
+ this.logInternal({ event: eventRest, internalData });
2312
+ this.isMerge = true;
2313
+ }
2314
+ get id() {
2315
+ return this._id;
2316
+ }
2317
+ setAttributes(args) {
2318
+ this.logInternal({ internalData: { span_attributes: args } });
2319
+ }
2320
+ log(event) {
2321
+ this.logInternal({ event });
2322
+ }
2323
+ logInternal({
2324
+ event,
2325
+ internalData
2326
+ }) {
2327
+ const sanitized = validateAndSanitizeExperimentLogPartialArgs(event ?? {});
2328
+ let sanitizedAndInternalData = { ...internalData };
2329
+ mergeDicts(sanitizedAndInternalData, sanitized);
2330
+ const serializableInternalData = {};
2331
+ const lazyInternalData = {};
2332
+ for (const [key, value] of Object.entries(sanitizedAndInternalData)) {
2333
+ if (value instanceof BraintrustStream) {
2334
+ const streamCopy = value.copy();
2335
+ lazyInternalData[key] = new LazyValue(async () => {
2336
+ return await new Promise((resolve) => {
2337
+ streamCopy.toReadableStream().pipeThrough(createFinalValuePassThroughStream(resolve)).pipeTo(devNullWritableStream());
2338
+ });
2339
+ });
2340
+ } else if (value instanceof ReadableStream) {
2341
+ lazyInternalData[key] = new LazyValue(async () => {
2342
+ return await new Promise((resolve) => {
2343
+ value.pipeThrough(createFinalValuePassThroughStream(resolve)).pipeTo(devNullWritableStream());
2344
+ });
2345
+ });
2346
+ } else {
2347
+ serializableInternalData[key] = value;
2348
+ }
2349
+ }
2350
+ let partialRecord = {
2351
+ id: this.id,
2352
+ span_id: this.spanId,
2353
+ root_span_id: this.rootSpanId,
2354
+ span_parents: this.spanParents,
2355
+ ...serializableInternalData,
2356
+ [IS_MERGE_FIELD]: this.isMerge
2357
+ };
2358
+ const serializedPartialRecord = JSON.stringify(partialRecord, (k, v) => {
2359
+ if (v instanceof _SpanImpl) {
2360
+ return `<span>`;
2361
+ } else if (v instanceof Experiment) {
2362
+ return `<experiment>`;
2363
+ } else if (v instanceof Dataset) {
2364
+ return `<dataset>`;
2365
+ } else if (v instanceof Logger) {
2366
+ return `<logger>`;
2367
+ }
2368
+ return v;
2369
+ });
2370
+ partialRecord = JSON.parse(serializedPartialRecord);
2371
+ if (partialRecord.metrics?.end) {
2372
+ this.loggedEndTime = partialRecord.metrics?.end;
2373
+ }
2374
+ if ((partialRecord.tags ?? []).length > 0 && this.spanParents?.length) {
2375
+ throw new Error("Tags can only be logged to the root span");
2376
+ }
2377
+ const computeRecord = async () => ({
2378
+ ...partialRecord,
2379
+ ...Object.fromEntries(
2380
+ await Promise.all(
2381
+ Object.entries(lazyInternalData).map(async ([key, value]) => [
2382
+ key,
2383
+ await value.get()
2384
+ ])
2385
+ )
2386
+ ),
2387
+ ...new SpanComponentsV2({
2388
+ objectType: this.parentObjectType,
2389
+ objectId: await this.parentObjectId.get()
2390
+ }).objectIdFields()
2391
+ });
2392
+ this.state.bgLogger().log([new LazyValue(computeRecord)]);
2393
+ }
2394
+ logFeedback(event) {
2395
+ logFeedbackImpl(this.state, this.parentObjectType, this.parentObjectId, {
2396
+ ...event,
2397
+ id: this.id
2398
+ });
2399
+ }
2400
+ traced(callback, args) {
2401
+ const { setCurrent, ...argsRest } = args ?? {};
2402
+ const span = this.startSpan(argsRest);
2403
+ return runFinally(
2404
+ () => {
2405
+ if (setCurrent ?? true) {
2406
+ return withCurrent(span, callback);
2407
+ } else {
2408
+ return callback(span);
2409
+ }
2410
+ },
2411
+ () => span.end()
2412
+ );
2413
+ }
2414
+ startSpan(args) {
2415
+ const parentSpanIds = args?.parent ? void 0 : { spanId: this.spanId, rootSpanId: this.rootSpanId };
2416
+ return new _SpanImpl({
2417
+ state: this.state,
2418
+ ...args,
2419
+ ...startSpanParentArgs({
2420
+ state: this.state,
2421
+ parent: args?.parent,
2422
+ parentObjectType: this.parentObjectType,
2423
+ parentObjectId: this.parentObjectId,
2424
+ parentComputeObjectMetadataArgs: this.parentComputeObjectMetadataArgs,
2425
+ parentSpanIds
2426
+ })
2427
+ });
2428
+ }
2429
+ end(args) {
2430
+ let endTime;
2431
+ let internalData = {};
2432
+ if (!this.loggedEndTime) {
2433
+ endTime = args?.endTime ?? getCurrentUnixTimestamp();
2434
+ internalData = { metrics: { end: endTime } };
2435
+ } else {
2436
+ endTime = this.loggedEndTime;
2437
+ }
2438
+ this.logInternal({ internalData });
2439
+ return endTime;
2440
+ }
2441
+ async export() {
2442
+ let objectId = void 0;
2443
+ let computeObjectMetadataArgs = void 0;
2444
+ if (this.parentComputeObjectMetadataArgs && !this.parentObjectId.hasComputed) {
2445
+ computeObjectMetadataArgs = this.parentComputeObjectMetadataArgs;
2446
+ } else {
2447
+ objectId = await this.parentObjectId.get();
2448
+ }
2449
+ return new SpanComponentsV2({
2450
+ objectType: this.parentObjectType,
2451
+ objectId,
2452
+ computeObjectMetadataArgs,
2453
+ rowIds: new SpanRowIdsV2({
2454
+ rowId: this.id,
2455
+ spanId: this.spanId,
2456
+ rootSpanId: this.rootSpanId
2457
+ })
2458
+ }).toStr();
2459
+ }
2460
+ async flush() {
2461
+ return await this.state.bgLogger().flush();
2462
+ }
2463
+ close(args) {
2464
+ return this.end(args);
2465
+ }
2466
+ };
2467
+ var Dataset = class extends ObjectFetcher {
2468
+ constructor(state, lazyMetadata, pinnedVersion, legacy) {
2469
+ const isLegacyDataset = legacy ?? DEFAULT_IS_LEGACY_DATASET;
2470
+ if (isLegacyDataset) {
2471
+ console.warn(
2472
+ `Records will be fetched from this dataset in the legacy format, with the "expected" field renamed to "output". Please update your code to use "expected", and use \`braintrust.initDataset()\` with \`{ useOutput: false }\`, which will become the default in a future version of Braintrust.`
2473
+ );
2474
+ }
2475
+ super(
2476
+ "dataset",
2477
+ pinnedVersion,
2478
+ (r) => ensureDatasetRecord(r, isLegacyDataset)
2479
+ );
2480
+ this.state = state;
2481
+ this.lazyMetadata = lazyMetadata;
2482
+ }
2483
+ lazyMetadata;
2484
+ get id() {
2485
+ return (async () => {
2486
+ return (await this.lazyMetadata.get()).dataset.id;
2487
+ })();
2488
+ }
2489
+ get name() {
2490
+ return (async () => {
2491
+ return (await this.lazyMetadata.get()).dataset.name;
2492
+ })();
2493
+ }
2494
+ get project() {
2495
+ return (async () => {
2496
+ return (await this.lazyMetadata.get()).project;
2497
+ })();
2498
+ }
2499
+ async getState() {
2500
+ await this.lazyMetadata.get();
2501
+ return this.state;
2502
+ }
2503
+ /**
2504
+ * Insert a single record to the dataset. The record will be batched and uploaded behind the scenes. If you pass in an `id`,
2505
+ * and a record with that `id` already exists, it will be overwritten (upsert).
2506
+ *
2507
+ * @param event The event to log.
2508
+ * @param event.input The argument that uniquely define an input case (an arbitrary, JSON serializable object).
2509
+ * @param event.expected The output of your application, including post-processing (an arbitrary, JSON serializable object).
2510
+ * @param event.tags (Optional) a list of strings that you can use to filter and group records later.
2511
+ * @param event.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
2512
+ * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
2513
+ * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
2514
+ * JSON-serializable type, but its keys must be strings.
2515
+ * @param event.id (Optional) a unique identifier for the event. If you don't provide one, Braintrust will generate one for you.
2516
+ * @param event.output: (Deprecated) The output of your application. Use `expected` instead.
2517
+ * @returns The `id` of the logged record.
2518
+ */
2519
+ insert({
2520
+ input,
2521
+ expected,
2522
+ metadata,
2523
+ tags,
2524
+ id,
2525
+ output
2526
+ }) {
2527
+ if (metadata !== void 0) {
2528
+ for (const key of Object.keys(metadata)) {
2529
+ if (typeof key !== "string") {
2530
+ throw new Error("metadata keys must be strings");
2531
+ }
2532
+ }
2533
+ }
2534
+ if (expected && output) {
2535
+ throw new Error(
2536
+ "Only one of expected or output (deprecated) can be specified. Prefer expected."
2537
+ );
2538
+ }
2539
+ if (tags) {
2540
+ validateTags(tags);
2541
+ }
2542
+ const rowId = id || uuidv4();
2543
+ const args = new LazyValue(async () => ({
2544
+ id: rowId,
2545
+ input,
2546
+ expected: expected === void 0 ? output : expected,
2547
+ tags,
2548
+ dataset_id: await this.id,
2549
+ created: (/* @__PURE__ */ new Date()).toISOString(),
2550
+ metadata
2551
+ }));
2552
+ this.state.bgLogger().log([args]);
2553
+ return rowId;
2554
+ }
2555
+ delete(id) {
2556
+ const args = new LazyValue(async () => ({
2557
+ id,
2558
+ dataset_id: await this.id,
2559
+ created: (/* @__PURE__ */ new Date()).toISOString(),
2560
+ _object_delete: true
2561
+ }));
2562
+ this.state.bgLogger().log([args]);
2563
+ return id;
2564
+ }
2565
+ /**
2566
+ * Summarize the dataset, including high level metrics about its size and other metadata.
2567
+ * @param summarizeData Whether to summarize the data. If false, only the metadata will be returned.
2568
+ * @returns `DatasetSummary`
2569
+ * @returns A summary of the dataset.
2570
+ */
2571
+ async summarize(options = {}) {
2572
+ let { summarizeData = true } = options || {};
2573
+ await this.flush();
2574
+ const state = await this.getState();
2575
+ const projectUrl = `${state.appPublicUrl}/app/${encodeURIComponent(
2576
+ state.orgName
2577
+ )}/p/${encodeURIComponent((await this.project).name)}`;
2578
+ const datasetUrl = `${projectUrl}/datasets/${encodeURIComponent(
2579
+ await this.name
2580
+ )}`;
2581
+ let dataSummary = void 0;
2582
+ if (summarizeData) {
2583
+ dataSummary = await state.logConn().get_json(
2584
+ "dataset-summary",
2585
+ {
2586
+ dataset_id: await this.id
2587
+ },
2588
+ 3
2589
+ );
2590
+ }
2591
+ return {
2592
+ projectName: (await this.project).name,
2593
+ datasetName: await this.name,
2594
+ projectUrl,
2595
+ datasetUrl,
2596
+ dataSummary
2597
+ };
2598
+ }
2599
+ /**
2600
+ * Flush any pending rows to the server.
2601
+ */
2602
+ async flush() {
2603
+ return await this.state.bgLogger().flush();
2604
+ }
2605
+ /**
2606
+ * This function is deprecated. You can simply remove it from your code.
2607
+ */
2608
+ async close() {
2609
+ console.warn(
2610
+ "close is deprecated and will be removed in a future version of braintrust. It is now a no-op and can be removed"
2611
+ );
2612
+ return this.id;
2613
+ }
2614
+ };
2615
+ var Prompt = class {
2616
+ constructor(metadata, defaults, noTrace) {
2617
+ this.metadata = metadata;
2618
+ this.defaults = defaults;
2619
+ this.noTrace = noTrace;
2620
+ }
2621
+ get id() {
2622
+ return this.metadata.id;
2623
+ }
2624
+ get projectId() {
2625
+ return this.metadata.project_id;
2626
+ }
2627
+ get name() {
2628
+ return "name" in this.metadata ? this.metadata.name : `Playground function ${this.metadata.id}`;
2629
+ }
2630
+ get slug() {
2631
+ return "slug" in this.metadata ? this.metadata.slug : this.metadata.id;
2632
+ }
2633
+ get prompt() {
2634
+ return this.metadata.prompt_data?.prompt;
2635
+ }
2636
+ get version() {
2637
+ return this.metadata[TRANSACTION_ID_FIELD];
2638
+ }
2639
+ get options() {
2640
+ return this.metadata.prompt_data?.options || {};
2641
+ }
2642
+ /**
2643
+ * Build the prompt with the given formatting options. The args you pass in will
2644
+ * be forwarded to the mustache template that defines the prompt and rendered with
2645
+ * the `mustache-js` library.
2646
+ *
2647
+ * @param buildArgs Args to forward along to the prompt template.
2648
+ */
2649
+ build(buildArgs, options = {}) {
2650
+ return this.runBuild(buildArgs, {
2651
+ flavor: options.flavor ?? "chat"
2652
+ });
2653
+ }
2654
+ runBuild(buildArgs, options) {
2655
+ const { flavor } = options;
2656
+ const params = {
2657
+ ...this.defaults,
2658
+ ...Object.fromEntries(
2659
+ Object.entries(this.options.params || {}).filter(
2660
+ ([k, _v]) => !BRAINTRUST_PARAMS.includes(k)
2661
+ )
2662
+ ),
2663
+ ...!isEmpty(this.options.model) ? {
2664
+ model: this.options.model
2665
+ } : {}
2666
+ };
2667
+ if (!("model" in params) || isEmpty(params.model)) {
2668
+ throw new Error(
2669
+ "No model specified. Either specify it in the prompt or as a default"
2670
+ );
2671
+ }
2672
+ const spanInfo = this.noTrace ? {} : {
2673
+ span_info: {
2674
+ metadata: {
2675
+ prompt: {
2676
+ variables: buildArgs,
2677
+ id: this.id,
2678
+ project_id: this.projectId,
2679
+ version: this.version,
2680
+ ..."prompt_session_id" in this.metadata ? { prompt_session_id: this.metadata.prompt_session_id } : {}
2681
+ }
2682
+ }
2683
+ }
2684
+ };
2685
+ const prompt = this.prompt;
2686
+ if (!prompt) {
2687
+ throw new Error("Empty prompt");
2688
+ }
2689
+ const dictArgParsed = z.record(z.unknown()).safeParse(buildArgs);
2690
+ const variables = {
2691
+ input: buildArgs,
2692
+ ...dictArgParsed.success ? dictArgParsed.data : {}
2693
+ };
2694
+ if (flavor === "chat") {
2695
+ if (prompt.type !== "chat") {
2696
+ throw new Error(
2697
+ "Prompt is a completion prompt. Use buildCompletion() instead"
2698
+ );
2699
+ }
2700
+ const render = (template) => Mustache.render(template, variables, void 0, {
2701
+ escape: (v) => typeof v === "string" ? v : JSON.stringify(v)
2702
+ });
2703
+ const messages = (prompt.messages || []).map((m) => ({
2704
+ ...m,
2705
+ ..."content" in m ? {
2706
+ content: typeof m.content === "string" ? render(m.content) : JSON.parse(render(JSON.stringify(m.content)))
2707
+ } : {}
2708
+ }));
2709
+ return {
2710
+ ...params,
2711
+ ...spanInfo,
2712
+ messages,
2713
+ ...prompt.tools ? {
2714
+ tools: toolsSchema.parse(
2715
+ JSON.parse(Mustache.render(prompt.tools, variables))
2716
+ )
2717
+ } : void 0
2718
+ };
2719
+ } else if (flavor === "completion") {
2720
+ if (prompt.type !== "completion") {
2721
+ throw new Error("Prompt is a chat prompt. Use buildChat() instead");
2722
+ }
2723
+ return {
2724
+ ...params,
2725
+ ...spanInfo,
2726
+ prompt: Mustache.render(prompt.content, variables)
2727
+ };
2728
+ } else {
2729
+ throw new Error("never!");
2730
+ }
2731
+ }
2732
+ };
2733
+
2734
+ // src/browser-config.ts
2735
+ var browserConfigured = false;
2736
+ function configureBrowser() {
2737
+ if (browserConfigured) {
2738
+ return;
2739
+ }
2740
+ try {
2741
+ if (typeof AsyncLocalStorage !== "undefined") {
2742
+ isomorph_default.newAsyncLocalStorage = () => new AsyncLocalStorage();
2743
+ }
2744
+ } catch {
2745
+ }
2746
+ isomorph_default.getEnv = (name) => {
2747
+ if (typeof process === "undefined" || typeof process.env === "undefined") {
2748
+ return void 0;
2749
+ }
2750
+ return process.env[name];
2751
+ };
2752
+ _internalSetInitialState();
2753
+ browserConfigured = true;
2754
+ }
2755
+
2756
+ // src/functions/invoke.ts
2757
+ import {
2758
+ INVOKE_API_VERSION
2759
+ } from "@braintrust/core/typespecs";
2760
+ async function invoke(args) {
2761
+ const {
2762
+ orgName,
2763
+ apiKey,
2764
+ appUrl,
2765
+ forceLogin,
2766
+ fetch,
2767
+ arg,
2768
+ parent: parentArg,
2769
+ state: stateArg,
2770
+ stream,
2771
+ schema,
2772
+ ...functionId
2773
+ } = args;
2774
+ const state = stateArg ?? _internalGetGlobalState();
2775
+ await state.login({
2776
+ orgName,
2777
+ apiKey,
2778
+ appUrl,
2779
+ forceLogin
2780
+ });
2781
+ const parent = parentArg ? typeof parentArg === "string" ? parentArg : await parentArg.export() : await currentSpan().export();
2782
+ const request = {
2783
+ ...functionId,
2784
+ arg,
2785
+ parent,
2786
+ stream,
2787
+ api_version: INVOKE_API_VERSION
2788
+ };
2789
+ const resp = await state.proxyConn().post(`function/invoke`, request, {
2790
+ headers: {
2791
+ Accept: stream ? "text/event-stream" : "application/json"
2792
+ }
2793
+ });
2794
+ if (stream) {
2795
+ if (!resp.body) {
2796
+ throw new Error("Received empty stream body");
2797
+ }
2798
+ return new BraintrustStream(resp.body);
2799
+ } else {
2800
+ const data = await resp.json();
2801
+ return schema ? schema.parse(data) : data;
2802
+ }
2803
+ }
2804
+
2805
+ // src/wrappers/oai.ts
2806
+ import { SpanTypeAttribute as SpanTypeAttribute2 } from "@braintrust/core";
2807
+ import { mergeDicts as mergeDicts2 } from "@braintrust/core";
2808
+ function wrapOpenAI(openai) {
2809
+ if (openai?.chat?.completions?.create) {
2810
+ return wrapOpenAIv4(openai);
2811
+ } else {
2812
+ console.warn("Unsupported OpenAI library (potentially v3). Not wrapping.");
2813
+ return openai;
2814
+ }
2815
+ }
2816
+ globalThis.__inherited_braintrust_wrap_openai = wrapOpenAI;
2817
+ function wrapOpenAIv4(openai) {
2818
+ let completionProxy = new Proxy(openai.chat.completions, {
2819
+ get(target, name, receiver) {
2820
+ const baseVal = Reflect.get(target, name, receiver);
2821
+ if (name === "create") {
2822
+ return wrapChatCompletion(baseVal.bind(target));
2823
+ }
2824
+ return baseVal;
2825
+ }
2826
+ });
2827
+ let chatProxy = new Proxy(openai.chat, {
2828
+ get(target, name, receiver) {
2829
+ if (name === "completions") {
2830
+ return completionProxy;
2831
+ }
2832
+ return Reflect.get(target, name, receiver);
2833
+ }
2834
+ });
2835
+ let embeddingProxy = new Proxy(openai.embeddings, {
2836
+ get(target, name, receiver) {
2837
+ const baseVal = Reflect.get(target, name, receiver);
2838
+ if (name === "create") {
2839
+ return wrapEmbeddings(baseVal.bind(target));
2840
+ }
2841
+ return baseVal;
2842
+ }
2843
+ });
2844
+ let betaProxy;
2845
+ if (openai.beta?.chat?.completions?.stream) {
2846
+ let betaChatCompletionProxy = new Proxy(openai?.beta?.chat.completions, {
2847
+ get(target, name, receiver) {
2848
+ const baseVal = Reflect.get(target, name, receiver);
2849
+ if (name === "stream") {
2850
+ return wrapBetaChatCompletion(baseVal.bind(target));
2851
+ }
2852
+ return baseVal;
2853
+ }
2854
+ });
2855
+ let betaChatProxy = new Proxy(openai.beta.chat, {
2856
+ get(target, name, receiver) {
2857
+ if (name === "completions") {
2858
+ return betaChatCompletionProxy;
2859
+ }
2860
+ return Reflect.get(target, name, receiver);
2861
+ }
2862
+ });
2863
+ betaProxy = new Proxy(openai.beta, {
2864
+ get(target, name, receiver) {
2865
+ if (name === "chat") {
2866
+ return betaChatProxy;
2867
+ }
2868
+ return Reflect.get(target, name, receiver);
2869
+ }
2870
+ });
2871
+ }
2872
+ let proxy = new Proxy(openai, {
2873
+ get(target, name, receiver) {
2874
+ if (name === "chat") {
2875
+ return chatProxy;
2876
+ }
2877
+ if (name === "embeddings") {
2878
+ return embeddingProxy;
2879
+ }
2880
+ if (name === "beta" && betaProxy) {
2881
+ return betaProxy;
2882
+ }
2883
+ return Reflect.get(target, name, receiver);
2884
+ }
2885
+ });
2886
+ return proxy;
2887
+ }
2888
+ function wrapBetaChatCompletion(completion) {
2889
+ return (allParams) => {
2890
+ const { span_info: _, ...params } = allParams;
2891
+ const span = startSpan(
2892
+ mergeDicts2(
2893
+ {
2894
+ name: "Chat Completion",
2895
+ spanAttributes: {
2896
+ type: SpanTypeAttribute2.LLM
2897
+ }
2898
+ },
2899
+ parseChatCompletionParams(allParams)
2900
+ )
2901
+ );
2902
+ const startTime = getCurrentUnixTimestamp();
2903
+ const ret = completion(params);
2904
+ let first = true;
2905
+ ret.on("chunk", (_chunk) => {
2906
+ if (first) {
2907
+ const now2 = getCurrentUnixTimestamp();
2908
+ span.log({
2909
+ metrics: {
2910
+ time_to_first_token: now2 - startTime
2911
+ }
2912
+ });
2913
+ first = false;
2914
+ }
2915
+ });
2916
+ ret.on("chatCompletion", (completion2) => {
2917
+ span.log({
2918
+ output: completion2.choices
2919
+ });
2920
+ });
2921
+ ret.on("end", () => {
2922
+ span.end();
2923
+ });
2924
+ return ret;
2925
+ };
2926
+ }
2927
+ var LEGACY_CACHED_HEADER = "x-cached";
2928
+ var X_CACHED_HEADER = "x-bt-cached";
2929
+ function parseCachedHeader(value) {
2930
+ return isEmpty(value) ? void 0 : ["true", "hit"].includes(value.toLowerCase()) ? 1 : 0;
2931
+ }
2932
+ function logHeaders(response, span) {
2933
+ const cachedHeader = response.headers.get(X_CACHED_HEADER);
2934
+ if (isEmpty(cachedHeader)) {
2935
+ const legacyCacheHeader = response.headers.get(LEGACY_CACHED_HEADER);
2936
+ if (!isEmpty(legacyCacheHeader)) {
2937
+ span.log({
2938
+ metrics: {
2939
+ cached: parseCachedHeader(legacyCacheHeader)
2940
+ }
2941
+ });
2942
+ }
2943
+ } else {
2944
+ span.log({
2945
+ metrics: {
2946
+ cached: parseCachedHeader(cachedHeader)
2947
+ }
2948
+ });
2949
+ }
2950
+ }
2951
+ function wrapChatCompletion(completion) {
2952
+ return async (allParams, options) => {
2953
+ const { span_info: _, ...params } = allParams;
2954
+ const span = startSpan(
2955
+ mergeDicts2(
2956
+ {
2957
+ name: "Chat Completion",
2958
+ spanAttributes: {
2959
+ type: SpanTypeAttribute2.LLM
2960
+ }
2961
+ },
2962
+ parseChatCompletionParams(allParams)
2963
+ )
2964
+ );
2965
+ const startTime = getCurrentUnixTimestamp();
2966
+ if (params.stream) {
2967
+ const { data: ret, response } = await completion(
2968
+ // We could get rid of this type coercion if we could somehow enforce
2969
+ // that `P extends ChatParams` BUT does not have the property
2970
+ // `span_info`.
2971
+ params,
2972
+ options
2973
+ ).withResponse();
2974
+ logHeaders(response, span);
2975
+ const wrapperStream = new WrapperStream(span, startTime, ret.iterator());
2976
+ ret.iterator = () => wrapperStream[Symbol.asyncIterator]();
2977
+ return ret;
2978
+ } else {
2979
+ try {
2980
+ const { data: ret, response } = await completion(
2981
+ params,
2982
+ options
2983
+ ).withResponse();
2984
+ logHeaders(response, span);
2985
+ const { messages, ...rest } = params;
2986
+ span.log({
2987
+ input: messages,
2988
+ metadata: {
2989
+ ...rest
2990
+ },
2991
+ output: ret.choices,
2992
+ metrics: {
2993
+ time_to_first_token: getCurrentUnixTimestamp() - startTime,
2994
+ tokens: ret.usage?.total_tokens,
2995
+ prompt_tokens: ret.usage?.prompt_tokens,
2996
+ completion_tokens: ret.usage?.completion_tokens
2997
+ }
2998
+ });
2999
+ return ret;
3000
+ } finally {
3001
+ span.end();
3002
+ }
3003
+ }
3004
+ };
3005
+ }
3006
+ function parseChatCompletionParams(allParams) {
3007
+ const { span_info, ...params } = allParams;
3008
+ const { metadata: spanInfoMetadata, ...spanInfoRest } = span_info ?? {};
3009
+ let ret = {
3010
+ ...spanInfoRest,
3011
+ event: {
3012
+ metadata: spanInfoMetadata
3013
+ }
3014
+ };
3015
+ const { messages, ...paramsRest } = params;
3016
+ return mergeDicts2(ret, { event: { input: messages, metadata: paramsRest } });
3017
+ }
3018
+ function wrapEmbeddings(create) {
3019
+ return async (allParams, options) => {
3020
+ const { span_info: _, ...params } = allParams;
3021
+ return traced(
3022
+ async (span) => {
3023
+ const { data: result, response } = await create(
3024
+ params,
3025
+ options
3026
+ ).withResponse();
3027
+ logHeaders(response, span);
3028
+ const embedding_length = result.data[0].embedding.length;
3029
+ span.log({
3030
+ // TODO: Add a flag to control whether to log the full embedding vector,
3031
+ // possibly w/ JSON compression.
3032
+ output: { embedding_length },
3033
+ metrics: {
3034
+ tokens: result.usage?.total_tokens,
3035
+ prompt_tokens: result.usage?.prompt_tokens
3036
+ }
3037
+ });
3038
+ return result;
3039
+ },
3040
+ mergeDicts2(
3041
+ {
3042
+ name: "Embedding",
3043
+ spanAttributes: {
3044
+ type: SpanTypeAttribute2.LLM
3045
+ }
3046
+ },
3047
+ parseEmbeddingParams(allParams)
3048
+ )
3049
+ );
3050
+ };
3051
+ }
3052
+ function parseEmbeddingParams(allParams) {
3053
+ const { span_info, ...params } = allParams;
3054
+ const { metadata: spanInfoMetadata, ...spanInfoRest } = span_info ?? {};
3055
+ let ret = {
3056
+ ...spanInfoRest,
3057
+ event: {
3058
+ metadata: spanInfoMetadata
3059
+ }
3060
+ };
3061
+ const { input, ...paramsRest } = params;
3062
+ return mergeDicts2(ret, { event: { input, metadata: paramsRest } });
3063
+ }
3064
+ function postprocessStreamingResults(allResults) {
3065
+ let role = void 0;
3066
+ let content = void 0;
3067
+ let tool_calls = void 0;
3068
+ let finish_reason = void 0;
3069
+ let metrics = {};
3070
+ for (const result of allResults) {
3071
+ if (result.usage) {
3072
+ metrics = {
3073
+ ...metrics,
3074
+ tokens: result.usage.total_tokens,
3075
+ prompt_tokens: result.usage.prompt_tokens,
3076
+ completion_tokens: result.usage.completion_tokens
3077
+ };
3078
+ }
3079
+ const delta = result.choices?.[0]?.delta;
3080
+ if (!delta) {
3081
+ continue;
3082
+ }
3083
+ if (!role && delta.role) {
3084
+ role = delta.role;
3085
+ }
3086
+ if (delta.finish_reason) {
3087
+ finish_reason = delta.finish_reason;
3088
+ }
3089
+ if (delta.content) {
3090
+ content = (content || "") + delta.content;
3091
+ }
3092
+ if (delta.tool_calls) {
3093
+ if (!tool_calls) {
3094
+ tool_calls = [
3095
+ {
3096
+ id: delta.tool_calls[0].id,
3097
+ type: delta.tool_calls[0].type,
3098
+ function: delta.tool_calls[0].function
3099
+ }
3100
+ ];
3101
+ } else {
3102
+ tool_calls[0].function.arguments += delta.tool_calls[0].function.arguments;
3103
+ }
3104
+ }
3105
+ }
3106
+ return {
3107
+ metrics,
3108
+ output: [
3109
+ {
3110
+ index: 0,
3111
+ message: {
3112
+ role,
3113
+ content,
3114
+ tool_calls
3115
+ },
3116
+ logprobs: null,
3117
+ finish_reason
3118
+ }
3119
+ ]
3120
+ };
3121
+ }
3122
+ var WrapperStream = class {
3123
+ span;
3124
+ iter;
3125
+ startTime;
3126
+ constructor(span, startTime, iter) {
3127
+ this.span = span;
3128
+ this.iter = iter;
3129
+ this.startTime = startTime;
3130
+ }
3131
+ async *[Symbol.asyncIterator]() {
3132
+ let first = true;
3133
+ let allResults = [];
3134
+ try {
3135
+ for await (const item of this.iter) {
3136
+ if (first) {
3137
+ const now2 = getCurrentUnixTimestamp();
3138
+ this.span.log({
3139
+ metrics: {
3140
+ time_to_first_token: now2 - this.startTime
3141
+ }
3142
+ });
3143
+ first = false;
3144
+ }
3145
+ allResults.push(item);
3146
+ yield item;
3147
+ }
3148
+ this.span.log({
3149
+ ...postprocessStreamingResults(allResults)
3150
+ });
3151
+ } finally {
3152
+ this.span.end();
3153
+ }
3154
+ }
3155
+ };
3156
+
3157
+ // src/browser.ts
3158
+ configureBrowser();
3159
+ export {
3160
+ BraintrustState,
3161
+ BraintrustStream,
3162
+ Dataset,
3163
+ Experiment,
3164
+ LEGACY_CACHED_HEADER,
3165
+ Logger,
3166
+ NOOP_SPAN,
3167
+ NoopSpan,
3168
+ Prompt,
3169
+ ReadonlyExperiment,
3170
+ SpanImpl,
3171
+ X_CACHED_HEADER,
3172
+ _internalGetGlobalState,
3173
+ _internalSetInitialState,
3174
+ createFinalValuePassThroughStream,
3175
+ currentExperiment,
3176
+ currentLogger,
3177
+ currentSpan,
3178
+ devNullWritableStream,
3179
+ flush,
3180
+ getSpanParentObject,
3181
+ init,
3182
+ initDataset,
3183
+ initExperiment,
3184
+ initLogger,
3185
+ invoke,
3186
+ loadPrompt,
3187
+ log,
3188
+ login,
3189
+ loginToState,
3190
+ newId,
3191
+ parseCachedHeader,
3192
+ setFetch,
3193
+ startSpan,
3194
+ summarize,
3195
+ traceable,
3196
+ traced,
3197
+ withDataset,
3198
+ withExperiment,
3199
+ withLogger,
3200
+ wrapOpenAI,
3201
+ wrapOpenAIv4,
3202
+ wrapTraced
3203
+ };