braintrust 0.0.97 → 0.0.99
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +349 -103
- package/dist/cli.js +307 -128
- package/dist/framework.d.ts +18 -18
- package/dist/gitutil.d.ts +2 -2
- package/dist/index.d.ts +1 -0
- package/dist/index.js +427 -146
- package/dist/isomorph.d.ts +2 -2
- package/dist/logger.d.ts +89 -134
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
package/dist/browser.js
CHANGED
|
@@ -12,7 +12,7 @@ var DefaultAsyncLocalStorage = class {
|
|
|
12
12
|
}
|
|
13
13
|
};
|
|
14
14
|
var iso = {
|
|
15
|
-
|
|
15
|
+
getRepoInfo: async (_settings) => void 0,
|
|
16
16
|
getPastNAncestors: async () => [],
|
|
17
17
|
getEnv: (_name) => void 0,
|
|
18
18
|
getCallerLocation: () => void 0,
|
|
@@ -70,9 +70,10 @@ function v4(options, buf, offset) {
|
|
|
70
70
|
}
|
|
71
71
|
var v4_default = v4;
|
|
72
72
|
|
|
73
|
-
// ../core/js/dist/index.mjs
|
|
73
|
+
// ../core/js/dist/main/index.mjs
|
|
74
74
|
var TRANSACTION_ID_FIELD = "_xact_id";
|
|
75
75
|
var IS_MERGE_FIELD = "_is_merge";
|
|
76
|
+
var MERGE_PATHS_FIELD = "_merge_paths";
|
|
76
77
|
var AUDIT_SOURCE_FIELD = "_audit_source";
|
|
77
78
|
var AUDIT_METADATA_FIELD = "_audit_metadata";
|
|
78
79
|
var VALID_SOURCES = ["app", "api", "external"];
|
|
@@ -131,6 +132,54 @@ function mergeRowBatch(rows) {
|
|
|
131
132
|
out.push(...Object.values(rowGroups));
|
|
132
133
|
return out;
|
|
133
134
|
}
|
|
135
|
+
var DEFAULT_IS_LEGACY_DATASET = true;
|
|
136
|
+
function ensureDatasetRecord(r, legacy) {
|
|
137
|
+
if (legacy) {
|
|
138
|
+
return ensureLegacyDatasetRecord(r);
|
|
139
|
+
} else {
|
|
140
|
+
return ensureNewDatasetRecord(r);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
function ensureLegacyDatasetRecord(r) {
|
|
144
|
+
if ("output" in r) {
|
|
145
|
+
return r;
|
|
146
|
+
}
|
|
147
|
+
const row = {
|
|
148
|
+
...r,
|
|
149
|
+
output: r.expected
|
|
150
|
+
};
|
|
151
|
+
delete row.expected;
|
|
152
|
+
return row;
|
|
153
|
+
}
|
|
154
|
+
function ensureNewDatasetRecord(r) {
|
|
155
|
+
if ("expected" in r) {
|
|
156
|
+
return r;
|
|
157
|
+
}
|
|
158
|
+
const row = {
|
|
159
|
+
...r,
|
|
160
|
+
expected: r.output
|
|
161
|
+
};
|
|
162
|
+
delete row.output;
|
|
163
|
+
return row;
|
|
164
|
+
}
|
|
165
|
+
function makeLegacyEvent(e) {
|
|
166
|
+
if (!("dataset_id" in e) || !("expected" in e)) {
|
|
167
|
+
return e;
|
|
168
|
+
}
|
|
169
|
+
const event = {
|
|
170
|
+
...e,
|
|
171
|
+
output: e.expected
|
|
172
|
+
};
|
|
173
|
+
delete event.expected;
|
|
174
|
+
if (MERGE_PATHS_FIELD in event) {
|
|
175
|
+
for (const path of event[MERGE_PATHS_FIELD] || []) {
|
|
176
|
+
if (path.length > 0 && path[0] === "expected") {
|
|
177
|
+
path[0] = "output";
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
return event;
|
|
182
|
+
}
|
|
134
183
|
var SpanTypeAttribute = /* @__PURE__ */ ((SpanTypeAttribute2) => {
|
|
135
184
|
SpanTypeAttribute2["LLM"] = "llm";
|
|
136
185
|
SpanTypeAttribute2["SCORE"] = "score";
|
|
@@ -184,10 +233,11 @@ function isEmpty(a) {
|
|
|
184
233
|
return a === void 0 || a === null;
|
|
185
234
|
}
|
|
186
235
|
var LazyValue = class {
|
|
236
|
+
callable;
|
|
237
|
+
value = {
|
|
238
|
+
hasComputed: false
|
|
239
|
+
};
|
|
187
240
|
constructor(callable) {
|
|
188
|
-
this.value = {
|
|
189
|
-
hasComputed: false
|
|
190
|
-
};
|
|
191
241
|
this.callable = callable;
|
|
192
242
|
}
|
|
193
243
|
async get() {
|
|
@@ -201,8 +251,11 @@ var LazyValue = class {
|
|
|
201
251
|
|
|
202
252
|
// src/logger.ts
|
|
203
253
|
var NoopSpan = class {
|
|
254
|
+
id;
|
|
255
|
+
span_id;
|
|
256
|
+
root_span_id;
|
|
257
|
+
kind = "span";
|
|
204
258
|
constructor() {
|
|
205
|
-
this.kind = "span";
|
|
206
259
|
this.id = "";
|
|
207
260
|
this.span_id = "";
|
|
208
261
|
this.root_span_id = "";
|
|
@@ -226,15 +279,22 @@ var NoopSpan = class {
|
|
|
226
279
|
};
|
|
227
280
|
var NOOP_SPAN = new NoopSpan();
|
|
228
281
|
var BraintrustState = class {
|
|
282
|
+
id;
|
|
283
|
+
currentExperiment;
|
|
284
|
+
// Note: the value of IsAsyncFlush doesn't really matter here, since we
|
|
285
|
+
// (safely) dynamically cast it whenever retrieving the logger.
|
|
286
|
+
currentLogger;
|
|
287
|
+
currentSpan;
|
|
288
|
+
appUrl = null;
|
|
289
|
+
loginToken = null;
|
|
290
|
+
orgId = null;
|
|
291
|
+
orgName = null;
|
|
292
|
+
logUrl = null;
|
|
293
|
+
loggedIn = false;
|
|
294
|
+
gitMetadataSettings;
|
|
295
|
+
_apiConn = null;
|
|
296
|
+
_logConn = null;
|
|
229
297
|
constructor() {
|
|
230
|
-
this.appUrl = null;
|
|
231
|
-
this.loginToken = null;
|
|
232
|
-
this.orgId = null;
|
|
233
|
-
this.orgName = null;
|
|
234
|
-
this.logUrl = null;
|
|
235
|
-
this.loggedIn = false;
|
|
236
|
-
this._apiConn = null;
|
|
237
|
-
this._logConn = null;
|
|
238
298
|
this.id = v4_default();
|
|
239
299
|
this.currentExperiment = void 0;
|
|
240
300
|
this.currentLogger = void 0;
|
|
@@ -281,6 +341,9 @@ function _internalSetInitialState() {
|
|
|
281
341
|
}
|
|
282
342
|
var _internalGetGlobalState = () => _state;
|
|
283
343
|
var FailedHTTPResponse = class extends Error {
|
|
344
|
+
status;
|
|
345
|
+
text;
|
|
346
|
+
data;
|
|
284
347
|
constructor(status, text, data = null) {
|
|
285
348
|
super(`${status}: ${text}`);
|
|
286
349
|
this.status = status;
|
|
@@ -300,6 +363,9 @@ async function checkResponse(resp) {
|
|
|
300
363
|
}
|
|
301
364
|
}
|
|
302
365
|
var HTTPConnection = class _HTTPConnection {
|
|
366
|
+
base_url;
|
|
367
|
+
token;
|
|
368
|
+
headers;
|
|
303
369
|
constructor(base_url) {
|
|
304
370
|
this.base_url = base_url;
|
|
305
371
|
this.token = null;
|
|
@@ -451,9 +517,13 @@ function logFeedbackImpl(bgLogger, parentIds, {
|
|
|
451
517
|
}
|
|
452
518
|
}
|
|
453
519
|
var Logger = class {
|
|
520
|
+
lazyMetadata;
|
|
521
|
+
logOptions;
|
|
522
|
+
bgLogger;
|
|
523
|
+
lastStartTime;
|
|
524
|
+
// For type identification.
|
|
525
|
+
kind = "logger";
|
|
454
526
|
constructor(lazyMetadata, logOptions = {}) {
|
|
455
|
-
// For type identification.
|
|
456
|
-
this.kind = "logger";
|
|
457
527
|
this.lazyMetadata = lazyMetadata;
|
|
458
528
|
this.logOptions = logOptions;
|
|
459
529
|
const logConn = new LazyValue(
|
|
@@ -487,9 +557,19 @@ var Logger = class {
|
|
|
487
557
|
* @param event.metadata: (Optional) a dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings.
|
|
488
558
|
* @param event.metrics: (Optional) a dictionary of metrics to log. The following keys are populated automatically: "start", "end".
|
|
489
559
|
* @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
|
|
560
|
+
* @param options Additional logging options
|
|
561
|
+
* @param options.allowLogConcurrentWithActiveSpan in rare cases where you need to log at the top level separately from an active span on the logger, set this to true.
|
|
490
562
|
* :returns: The `id` of the logged event.
|
|
491
563
|
*/
|
|
492
|
-
log(event) {
|
|
564
|
+
log(event, options) {
|
|
565
|
+
if (!options?.allowLogConcurrentWithActiveSpan) {
|
|
566
|
+
const checkCurrentSpan = currentSpan();
|
|
567
|
+
if (checkCurrentSpan instanceof SpanImpl && checkCurrentSpan.parentObject === this) {
|
|
568
|
+
throw new Error(
|
|
569
|
+
"Cannot run toplevel Logger.log method while there is an active span. To log to the span, use Span.log"
|
|
570
|
+
);
|
|
571
|
+
}
|
|
572
|
+
}
|
|
493
573
|
const span = this.startSpan({ startTime: this.lastStartTime, event });
|
|
494
574
|
this.lastStartTime = span.end();
|
|
495
575
|
const ret = span.id;
|
|
@@ -548,6 +628,7 @@ var Logger = class {
|
|
|
548
628
|
startSpan(args) {
|
|
549
629
|
const { name, ...argsRest } = args ?? {};
|
|
550
630
|
return new SpanImpl({
|
|
631
|
+
parentObject: this,
|
|
551
632
|
parentIds: new LazyValue(() => this.lazyParentIds()),
|
|
552
633
|
bgLogger: this.bgLogger,
|
|
553
634
|
name: name ?? "root",
|
|
@@ -596,16 +677,20 @@ var MaxRequestSize = 6 * 1024 * 1024;
|
|
|
596
677
|
function constructJsonArray(items) {
|
|
597
678
|
return `[${items.join(",")}]`;
|
|
598
679
|
}
|
|
680
|
+
function constructLogs3Data(items) {
|
|
681
|
+
return `{"rows": ${constructJsonArray(items)}, "api_version": 2}`;
|
|
682
|
+
}
|
|
599
683
|
var DefaultBatchSize = 100;
|
|
600
684
|
var NumRetries = 3;
|
|
601
685
|
function now() {
|
|
602
686
|
return (/* @__PURE__ */ new Date()).getTime();
|
|
603
687
|
}
|
|
604
688
|
var BackgroundLogger = class {
|
|
689
|
+
logConn;
|
|
690
|
+
items = [];
|
|
691
|
+
active_flush = Promise.resolve([]);
|
|
692
|
+
active_flush_resolved = true;
|
|
605
693
|
constructor(logConn) {
|
|
606
|
-
this.items = [];
|
|
607
|
-
this.active_flush = Promise.resolve([]);
|
|
608
|
-
this.active_flush_resolved = true;
|
|
609
694
|
this.logConn = logConn;
|
|
610
695
|
isomorph_default.processOn("beforeExit", async () => {
|
|
611
696
|
await this.flush();
|
|
@@ -654,11 +739,20 @@ var BackgroundLogger = class {
|
|
|
654
739
|
}
|
|
655
740
|
postPromises.push(
|
|
656
741
|
(async () => {
|
|
657
|
-
const
|
|
742
|
+
const dataStr = constructLogs3Data(items);
|
|
658
743
|
for (let i = 0; i < NumRetries; i++) {
|
|
659
744
|
const startTime = now();
|
|
660
745
|
try {
|
|
661
|
-
|
|
746
|
+
try {
|
|
747
|
+
return (await (await this.logConn.get()).post_json("logs3", dataStr)).ids.map((res) => res.id);
|
|
748
|
+
} catch (e) {
|
|
749
|
+
const legacyDataS = constructJsonArray(
|
|
750
|
+
items.map(
|
|
751
|
+
(r) => JSON.stringify(makeLegacyEvent(JSON.parse(r)))
|
|
752
|
+
)
|
|
753
|
+
);
|
|
754
|
+
return (await (await this.logConn.get()).post_json("logs", legacyDataS)).map((res) => res.id);
|
|
755
|
+
}
|
|
662
756
|
} catch (e) {
|
|
663
757
|
const retryingText = i + 1 === NumRetries ? "" : " Retrying";
|
|
664
758
|
const errMsg = (() => {
|
|
@@ -669,7 +763,7 @@ var BackgroundLogger = class {
|
|
|
669
763
|
}
|
|
670
764
|
})();
|
|
671
765
|
console.warn(
|
|
672
|
-
`log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${
|
|
766
|
+
`log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${dataStr.length}. Error: ${errMsg}.${retryingText}`
|
|
673
767
|
);
|
|
674
768
|
}
|
|
675
769
|
}
|
|
@@ -697,8 +791,21 @@ var BackgroundLogger = class {
|
|
|
697
791
|
}
|
|
698
792
|
}
|
|
699
793
|
};
|
|
700
|
-
function init(
|
|
794
|
+
function init(projectOrOptions, optionalOptions) {
|
|
795
|
+
const options = (() => {
|
|
796
|
+
if (typeof projectOrOptions === "string") {
|
|
797
|
+
return { ...optionalOptions, project: projectOrOptions };
|
|
798
|
+
} else {
|
|
799
|
+
if (optionalOptions !== void 0) {
|
|
800
|
+
throw new Error(
|
|
801
|
+
"Cannot specify options struct as both parameters. Must call either init(project, options) or init(options)."
|
|
802
|
+
);
|
|
803
|
+
}
|
|
804
|
+
return projectOrOptions;
|
|
805
|
+
}
|
|
806
|
+
})();
|
|
701
807
|
const {
|
|
808
|
+
project,
|
|
702
809
|
experiment,
|
|
703
810
|
description,
|
|
704
811
|
dataset,
|
|
@@ -710,47 +817,55 @@ function init(project, options = {}) {
|
|
|
710
817
|
apiKey,
|
|
711
818
|
orgName,
|
|
712
819
|
metadata,
|
|
713
|
-
gitMetadataSettings
|
|
714
|
-
|
|
820
|
+
gitMetadataSettings,
|
|
821
|
+
projectId,
|
|
822
|
+
baseExperimentId,
|
|
823
|
+
repoInfo
|
|
824
|
+
} = options;
|
|
715
825
|
if (open && update) {
|
|
716
826
|
throw new Error("Cannot open and update an experiment at the same time");
|
|
717
827
|
}
|
|
718
828
|
if (open || update) {
|
|
719
829
|
if (isEmpty(experiment)) {
|
|
720
830
|
const action = open ? "open" : "update";
|
|
721
|
-
throw new Error(
|
|
831
|
+
throw new Error(
|
|
832
|
+
`Cannot ${action} an experiment without specifying its name`
|
|
833
|
+
);
|
|
722
834
|
}
|
|
723
|
-
const lazyMetadata2 = new LazyValue(
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
return {
|
|
742
|
-
project: {
|
|
743
|
-
id: info.project_id,
|
|
744
|
-
name: "",
|
|
745
|
-
fullInfo: {}
|
|
746
|
-
},
|
|
747
|
-
experiment: {
|
|
748
|
-
id: info.id,
|
|
749
|
-
name: info.name,
|
|
750
|
-
fullInfo: info
|
|
835
|
+
const lazyMetadata2 = new LazyValue(
|
|
836
|
+
async () => {
|
|
837
|
+
await login({
|
|
838
|
+
orgName,
|
|
839
|
+
apiKey,
|
|
840
|
+
appUrl
|
|
841
|
+
});
|
|
842
|
+
const args = {
|
|
843
|
+
project_name: project,
|
|
844
|
+
project_id: projectId,
|
|
845
|
+
org_name: _state.orgName,
|
|
846
|
+
experiment_name: experiment
|
|
847
|
+
};
|
|
848
|
+
const response = await _state.apiConn().post_json("api/experiment/get", args);
|
|
849
|
+
if (response.length === 0) {
|
|
850
|
+
throw new Error(
|
|
851
|
+
`Experiment ${experiment} not found in project ${projectId ?? project}.`
|
|
852
|
+
);
|
|
751
853
|
}
|
|
752
|
-
|
|
753
|
-
|
|
854
|
+
const info = response[0];
|
|
855
|
+
return {
|
|
856
|
+
project: {
|
|
857
|
+
id: info.project_id,
|
|
858
|
+
name: "",
|
|
859
|
+
fullInfo: {}
|
|
860
|
+
},
|
|
861
|
+
experiment: {
|
|
862
|
+
id: info.id,
|
|
863
|
+
name: info.name,
|
|
864
|
+
fullInfo: info
|
|
865
|
+
}
|
|
866
|
+
};
|
|
867
|
+
}
|
|
868
|
+
);
|
|
754
869
|
if (open) {
|
|
755
870
|
return new ReadonlyExperiment(
|
|
756
871
|
lazyMetadata2
|
|
@@ -772,6 +887,7 @@ function init(project, options = {}) {
|
|
|
772
887
|
});
|
|
773
888
|
const args = {
|
|
774
889
|
project_name: project,
|
|
890
|
+
project_id: projectId,
|
|
775
891
|
org_id: _state.orgId
|
|
776
892
|
};
|
|
777
893
|
if (experiment) {
|
|
@@ -780,22 +896,29 @@ function init(project, options = {}) {
|
|
|
780
896
|
if (description) {
|
|
781
897
|
args["description"] = description;
|
|
782
898
|
}
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
899
|
+
const repoInfoArg = await (async () => {
|
|
900
|
+
if (repoInfo) {
|
|
901
|
+
return repoInfo;
|
|
786
902
|
}
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
)
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
903
|
+
let mergedGitMetadataSettings = {
|
|
904
|
+
..._state.gitMetadataSettings || {
|
|
905
|
+
collect: "all"
|
|
906
|
+
}
|
|
907
|
+
};
|
|
908
|
+
if (gitMetadataSettings) {
|
|
909
|
+
mergedGitMetadataSettings = mergeGitMetadataSettings(
|
|
910
|
+
mergedGitMetadataSettings,
|
|
911
|
+
gitMetadataSettings
|
|
912
|
+
);
|
|
913
|
+
}
|
|
914
|
+
return await isomorph_default.getRepoInfo(mergedGitMetadataSettings);
|
|
915
|
+
})();
|
|
916
|
+
if (repoInfoArg) {
|
|
917
|
+
args["repo_info"] = repoInfoArg;
|
|
797
918
|
}
|
|
798
|
-
if (
|
|
919
|
+
if (baseExperimentId) {
|
|
920
|
+
args["base_exp_id"] = baseExperimentId;
|
|
921
|
+
} else if (baseExperiment) {
|
|
799
922
|
args["base_experiment"] = baseExperiment;
|
|
800
923
|
} else {
|
|
801
924
|
args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
|
|
@@ -846,6 +969,21 @@ function init(project, options = {}) {
|
|
|
846
969
|
}
|
|
847
970
|
return ret;
|
|
848
971
|
}
|
|
972
|
+
function initExperiment(projectOrOptions, optionalOptions) {
|
|
973
|
+
const options = (() => {
|
|
974
|
+
if (typeof projectOrOptions === "string") {
|
|
975
|
+
return { ...optionalOptions, project: projectOrOptions };
|
|
976
|
+
} else {
|
|
977
|
+
if (optionalOptions !== void 0) {
|
|
978
|
+
throw new Error(
|
|
979
|
+
"Cannot specify options struct as both parameters. Must call either init(project, options) or init(options)."
|
|
980
|
+
);
|
|
981
|
+
}
|
|
982
|
+
return projectOrOptions;
|
|
983
|
+
}
|
|
984
|
+
})();
|
|
985
|
+
return init(options);
|
|
986
|
+
}
|
|
849
987
|
function withExperiment(project, callback, options = {}) {
|
|
850
988
|
console.warn(
|
|
851
989
|
"withExperiment is deprecated and will be removed in a future version of braintrust. Simply create the experiment with `init`."
|
|
@@ -860,8 +998,30 @@ function withLogger(callback, options = {}) {
|
|
|
860
998
|
const logger = initLogger(options);
|
|
861
999
|
return callback(logger);
|
|
862
1000
|
}
|
|
863
|
-
function initDataset(
|
|
864
|
-
const
|
|
1001
|
+
function initDataset(projectOrOptions, optionalOptions) {
|
|
1002
|
+
const options = (() => {
|
|
1003
|
+
if (typeof projectOrOptions === "string") {
|
|
1004
|
+
return { ...optionalOptions, project: projectOrOptions };
|
|
1005
|
+
} else {
|
|
1006
|
+
if (optionalOptions !== void 0) {
|
|
1007
|
+
throw new Error(
|
|
1008
|
+
"Cannot specify options struct as both parameters. Must call either initDataset(project, options) or initDataset(options)."
|
|
1009
|
+
);
|
|
1010
|
+
}
|
|
1011
|
+
return projectOrOptions;
|
|
1012
|
+
}
|
|
1013
|
+
})();
|
|
1014
|
+
const {
|
|
1015
|
+
project,
|
|
1016
|
+
dataset,
|
|
1017
|
+
description,
|
|
1018
|
+
version,
|
|
1019
|
+
appUrl,
|
|
1020
|
+
apiKey,
|
|
1021
|
+
orgName,
|
|
1022
|
+
projectId,
|
|
1023
|
+
useOutput: legacy
|
|
1024
|
+
} = options;
|
|
865
1025
|
const lazyMetadata = new LazyValue(
|
|
866
1026
|
async () => {
|
|
867
1027
|
await login({
|
|
@@ -872,6 +1032,7 @@ function initDataset(project, options = {}) {
|
|
|
872
1032
|
const args = {
|
|
873
1033
|
org_id: _state.orgId,
|
|
874
1034
|
project_name: project,
|
|
1035
|
+
project_id: projectId,
|
|
875
1036
|
dataset_name: dataset,
|
|
876
1037
|
description
|
|
877
1038
|
};
|
|
@@ -890,7 +1051,7 @@ function initDataset(project, options = {}) {
|
|
|
890
1051
|
};
|
|
891
1052
|
}
|
|
892
1053
|
);
|
|
893
|
-
return new Dataset(lazyMetadata, version);
|
|
1054
|
+
return new Dataset(lazyMetadata, version, legacy);
|
|
894
1055
|
}
|
|
895
1056
|
function withDataset(project, callback, options = {}) {
|
|
896
1057
|
console.warn(
|
|
@@ -960,15 +1121,30 @@ function initLogger(options = {}) {
|
|
|
960
1121
|
return ret;
|
|
961
1122
|
}
|
|
962
1123
|
async function login(options = {}) {
|
|
1124
|
+
let { forceLogin = false } = options || {};
|
|
1125
|
+
if (_state.loggedIn && !forceLogin) {
|
|
1126
|
+
let checkUpdatedParam2 = function(varname, arg, orig) {
|
|
1127
|
+
if (!isEmpty(arg) && !isEmpty(orig) && arg !== orig) {
|
|
1128
|
+
throw new Error(
|
|
1129
|
+
`Re-logging in with different ${varname} (${arg}) than original (${orig}). To force re-login, pass \`forceLogin: true\``
|
|
1130
|
+
);
|
|
1131
|
+
}
|
|
1132
|
+
};
|
|
1133
|
+
var checkUpdatedParam = checkUpdatedParam2;
|
|
1134
|
+
checkUpdatedParam2("appUrl", options.appUrl, _state.appUrl);
|
|
1135
|
+
checkUpdatedParam2(
|
|
1136
|
+
"apiKey",
|
|
1137
|
+
options.apiKey ? HTTPConnection.sanitize_token(options.apiKey) : void 0,
|
|
1138
|
+
_state.loginToken
|
|
1139
|
+
);
|
|
1140
|
+
checkUpdatedParam2("orgName", options.orgName, _state.orgName);
|
|
1141
|
+
return;
|
|
1142
|
+
}
|
|
963
1143
|
const {
|
|
964
1144
|
appUrl = isomorph_default.getEnv("BRAINTRUST_APP_URL") || "https://www.braintrustdata.com",
|
|
965
1145
|
apiKey = isomorph_default.getEnv("BRAINTRUST_API_KEY"),
|
|
966
1146
|
orgName = isomorph_default.getEnv("BRAINTRUST_ORG_NAME")
|
|
967
1147
|
} = options || {};
|
|
968
|
-
let { forceLogin = false } = options || {};
|
|
969
|
-
if (_state.loggedIn && !forceLogin) {
|
|
970
|
-
return;
|
|
971
|
-
}
|
|
972
1148
|
_state.resetLoginInfo();
|
|
973
1149
|
_state.appUrl = appUrl;
|
|
974
1150
|
let conn = null;
|
|
@@ -1177,11 +1353,12 @@ function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
|
|
|
1177
1353
|
return event;
|
|
1178
1354
|
}
|
|
1179
1355
|
var ObjectFetcher = class {
|
|
1180
|
-
constructor(objectType, pinnedVersion) {
|
|
1356
|
+
constructor(objectType, pinnedVersion, mutateRecord) {
|
|
1181
1357
|
this.objectType = objectType;
|
|
1182
1358
|
this.pinnedVersion = pinnedVersion;
|
|
1183
|
-
this.
|
|
1359
|
+
this.mutateRecord = mutateRecord;
|
|
1184
1360
|
}
|
|
1361
|
+
_fetchedData = void 0;
|
|
1185
1362
|
get id() {
|
|
1186
1363
|
throw new Error("ObjectFetcher subclasses must have an 'id' attribute");
|
|
1187
1364
|
}
|
|
@@ -1200,12 +1377,24 @@ var ObjectFetcher = class {
|
|
|
1200
1377
|
async fetchedData() {
|
|
1201
1378
|
if (this._fetchedData === void 0) {
|
|
1202
1379
|
const state = await this.getState();
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1380
|
+
let data = void 0;
|
|
1381
|
+
try {
|
|
1382
|
+
const resp = await state.logConn().get(`object3/${this.objectType}`, {
|
|
1383
|
+
id: await this.id,
|
|
1384
|
+
fmt: "json2",
|
|
1385
|
+
version: this.pinnedVersion,
|
|
1386
|
+
api_version: "2"
|
|
1387
|
+
});
|
|
1388
|
+
data = await resp.json();
|
|
1389
|
+
} catch (e) {
|
|
1390
|
+
const resp = await state.logConn().get(`object/${this.objectType}`, {
|
|
1391
|
+
id: await this.id,
|
|
1392
|
+
fmt: "json2",
|
|
1393
|
+
version: this.pinnedVersion
|
|
1394
|
+
});
|
|
1395
|
+
data = await resp.json();
|
|
1396
|
+
}
|
|
1397
|
+
this._fetchedData = this.mutateRecord ? data?.map(this.mutateRecord) : data;
|
|
1209
1398
|
}
|
|
1210
1399
|
return this._fetchedData || [];
|
|
1211
1400
|
}
|
|
@@ -1229,10 +1418,14 @@ var ObjectFetcher = class {
|
|
|
1229
1418
|
}
|
|
1230
1419
|
};
|
|
1231
1420
|
var Experiment = class extends ObjectFetcher {
|
|
1421
|
+
lazyMetadata;
|
|
1422
|
+
dataset;
|
|
1423
|
+
bgLogger;
|
|
1424
|
+
lastStartTime;
|
|
1425
|
+
// For type identification.
|
|
1426
|
+
kind = "experiment";
|
|
1232
1427
|
constructor(lazyMetadata, dataset) {
|
|
1233
1428
|
super("experiment", void 0);
|
|
1234
|
-
// For type identification.
|
|
1235
|
-
this.kind = "experiment";
|
|
1236
1429
|
this.lazyMetadata = lazyMetadata;
|
|
1237
1430
|
this.dataset = dataset;
|
|
1238
1431
|
const logConn = new LazyValue(
|
|
@@ -1273,9 +1466,19 @@ var Experiment = class extends ObjectFetcher {
|
|
|
1273
1466
|
* @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
|
|
1274
1467
|
* @param event.dataset_record_id: (Optional) the id of the dataset record that this event is associated with. This field is required if and only if the experiment is associated with a dataset.
|
|
1275
1468
|
* @param event.inputs: (Deprecated) the same as `input` (will be removed in a future version).
|
|
1469
|
+
* @param options Additional logging options
|
|
1470
|
+
* @param options.allowLogConcurrentWithActiveSpan in rare cases where you need to log at the top level separately from an active span on the experiment, set this to true.
|
|
1276
1471
|
* :returns: The `id` of the logged event.
|
|
1277
1472
|
*/
|
|
1278
|
-
log(event) {
|
|
1473
|
+
log(event, options) {
|
|
1474
|
+
if (!options?.allowLogConcurrentWithActiveSpan) {
|
|
1475
|
+
const checkCurrentSpan = currentSpan();
|
|
1476
|
+
if (checkCurrentSpan instanceof SpanImpl && checkCurrentSpan.parentObject === this) {
|
|
1477
|
+
throw new Error(
|
|
1478
|
+
"Cannot run toplevel Experiment.log method while there is an active span. To log to the span, use Span.log"
|
|
1479
|
+
);
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1279
1482
|
event = validateAndSanitizeExperimentLogFullArgs(event, !!this.dataset);
|
|
1280
1483
|
const span = this.startSpan({ startTime: this.lastStartTime, event });
|
|
1281
1484
|
this.lastStartTime = span.end();
|
|
@@ -1317,6 +1520,7 @@ var Experiment = class extends ObjectFetcher {
|
|
|
1317
1520
|
startSpan(args) {
|
|
1318
1521
|
const { name, ...argsRest } = args ?? {};
|
|
1319
1522
|
return new SpanImpl({
|
|
1523
|
+
parentObject: this,
|
|
1320
1524
|
parentIds: new LazyValue(() => this.lazyParentIds()),
|
|
1321
1525
|
bgLogger: this.bgLogger,
|
|
1322
1526
|
name: name ?? "root",
|
|
@@ -1454,20 +1658,38 @@ var ReadonlyExperiment = class extends ObjectFetcher {
|
|
|
1454
1658
|
if (record.root_span_id !== record.span_id) {
|
|
1455
1659
|
continue;
|
|
1456
1660
|
}
|
|
1457
|
-
const { output, expected } = record;
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1661
|
+
const { output, expected: expectedRecord } = record;
|
|
1662
|
+
const expected = expectedRecord ?? output;
|
|
1663
|
+
if (isEmpty(expected)) {
|
|
1664
|
+
yield {
|
|
1665
|
+
input: record.input
|
|
1666
|
+
};
|
|
1667
|
+
} else {
|
|
1668
|
+
yield {
|
|
1669
|
+
input: record.input,
|
|
1670
|
+
expected
|
|
1671
|
+
};
|
|
1672
|
+
}
|
|
1462
1673
|
}
|
|
1463
1674
|
}
|
|
1464
1675
|
};
|
|
1465
1676
|
var executionCounter = 0;
|
|
1466
1677
|
var SpanImpl = class _SpanImpl {
|
|
1678
|
+
bgLogger;
|
|
1679
|
+
// `internalData` contains fields that are not part of the "user-sanitized"
|
|
1680
|
+
// set of fields which we want to log in just one of the span rows.
|
|
1681
|
+
internalData;
|
|
1682
|
+
isMerge;
|
|
1683
|
+
loggedEndTime;
|
|
1684
|
+
// For internal use only.
|
|
1685
|
+
parentObject;
|
|
1686
|
+
// These fields are logged to every span row.
|
|
1687
|
+
parentIds;
|
|
1688
|
+
rowIds;
|
|
1689
|
+
kind = "span";
|
|
1467
1690
|
// root_experiment should only be specified for a root span. parent_span
|
|
1468
1691
|
// should only be specified for non-root spans.
|
|
1469
1692
|
constructor(args) {
|
|
1470
|
-
this.kind = "span";
|
|
1471
1693
|
this.loggedEndTime = void 0;
|
|
1472
1694
|
this.bgLogger = args.bgLogger;
|
|
1473
1695
|
const callerLocation = isomorph_default.getCallerLocation();
|
|
@@ -1495,6 +1717,7 @@ var SpanImpl = class _SpanImpl {
|
|
|
1495
1717
|
},
|
|
1496
1718
|
created: (/* @__PURE__ */ new Date()).toISOString()
|
|
1497
1719
|
};
|
|
1720
|
+
this.parentObject = args.parentObject;
|
|
1498
1721
|
this.parentIds = args.parentIds;
|
|
1499
1722
|
const id = args.event?.id ?? v4_default();
|
|
1500
1723
|
const span_id = v4_default();
|
|
@@ -1566,6 +1789,7 @@ var SpanImpl = class _SpanImpl {
|
|
|
1566
1789
|
}
|
|
1567
1790
|
startSpan(args) {
|
|
1568
1791
|
return new _SpanImpl({
|
|
1792
|
+
parentObject: this.parentObject,
|
|
1569
1793
|
parentIds: this.parentIds,
|
|
1570
1794
|
bgLogger: this.bgLogger,
|
|
1571
1795
|
parentSpanInfo: {
|
|
@@ -1591,8 +1815,20 @@ var SpanImpl = class _SpanImpl {
|
|
|
1591
1815
|
}
|
|
1592
1816
|
};
|
|
1593
1817
|
var Dataset = class extends ObjectFetcher {
|
|
1594
|
-
|
|
1595
|
-
|
|
1818
|
+
lazyMetadata;
|
|
1819
|
+
bgLogger;
|
|
1820
|
+
constructor(lazyMetadata, pinnedVersion, legacy) {
|
|
1821
|
+
const isLegacyDataset = legacy ?? DEFAULT_IS_LEGACY_DATASET;
|
|
1822
|
+
if (isLegacyDataset) {
|
|
1823
|
+
console.warn(
|
|
1824
|
+
`Records will be fetched from this dataset in the legacy format, with the "expected" field renamed to "output". Please update your code to use "expected", and use \`braintrust.initDataset()\` with \`{ useOutput: false }\`, which will become the default in a future version of Braintrust.`
|
|
1825
|
+
);
|
|
1826
|
+
}
|
|
1827
|
+
super(
|
|
1828
|
+
"dataset",
|
|
1829
|
+
pinnedVersion,
|
|
1830
|
+
(r) => ensureDatasetRecord(r, isLegacyDataset)
|
|
1831
|
+
);
|
|
1596
1832
|
this.lazyMetadata = lazyMetadata;
|
|
1597
1833
|
const logConn = new LazyValue(
|
|
1598
1834
|
() => this.getState().then((state) => state.logConn())
|
|
@@ -1624,19 +1860,21 @@ var Dataset = class extends ObjectFetcher {
|
|
|
1624
1860
|
*
|
|
1625
1861
|
* @param event The event to log.
|
|
1626
1862
|
* @param event.input The argument that uniquely define an input case (an arbitrary, JSON serializable object).
|
|
1627
|
-
* @param event.
|
|
1863
|
+
* @param event.expected The output of your application, including post-processing (an arbitrary, JSON serializable object).
|
|
1628
1864
|
* @param event.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
|
|
1629
1865
|
* about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
|
|
1630
1866
|
* `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
|
|
1631
1867
|
* JSON-serializable type, but its keys must be strings.
|
|
1632
1868
|
* @param event.id (Optional) a unique identifier for the event. If you don't provide one, Braintrust will generate one for you.
|
|
1869
|
+
* @param event.output: (Deprecated) The output of your application. Use `expected` instead.
|
|
1633
1870
|
* @returns The `id` of the logged record.
|
|
1634
1871
|
*/
|
|
1635
1872
|
insert({
|
|
1636
1873
|
input,
|
|
1637
|
-
|
|
1874
|
+
expected,
|
|
1638
1875
|
metadata,
|
|
1639
|
-
id
|
|
1876
|
+
id,
|
|
1877
|
+
output
|
|
1640
1878
|
}) {
|
|
1641
1879
|
if (metadata !== void 0) {
|
|
1642
1880
|
for (const key of Object.keys(metadata)) {
|
|
@@ -1645,11 +1883,16 @@ var Dataset = class extends ObjectFetcher {
|
|
|
1645
1883
|
}
|
|
1646
1884
|
}
|
|
1647
1885
|
}
|
|
1886
|
+
if (expected && output) {
|
|
1887
|
+
throw new Error(
|
|
1888
|
+
"Only one of expected or output (deprecated) can be specified. Prefer expected."
|
|
1889
|
+
);
|
|
1890
|
+
}
|
|
1648
1891
|
const rowId = id || v4_default();
|
|
1649
1892
|
const args = new LazyValue(async () => ({
|
|
1650
1893
|
id: rowId,
|
|
1651
|
-
|
|
1652
|
-
output,
|
|
1894
|
+
input,
|
|
1895
|
+
expected: expected === void 0 ? output : expected,
|
|
1653
1896
|
project_id: (await this.project).id,
|
|
1654
1897
|
dataset_id: await this.id,
|
|
1655
1898
|
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
@@ -1934,6 +2177,9 @@ function wrapEmbeddings(create) {
|
|
|
1934
2177
|
};
|
|
1935
2178
|
}
|
|
1936
2179
|
var WrapperStream = class {
|
|
2180
|
+
span;
|
|
2181
|
+
iter;
|
|
2182
|
+
startTime;
|
|
1937
2183
|
constructor(span, startTime, iter) {
|
|
1938
2184
|
this.span = span;
|
|
1939
2185
|
this.iter = iter;
|
|
@@ -1968,7 +2214,6 @@ var WrapperStream = class {
|
|
|
1968
2214
|
// src/browser.ts
|
|
1969
2215
|
configureBrowser();
|
|
1970
2216
|
export {
|
|
1971
|
-
Dataset,
|
|
1972
2217
|
Experiment,
|
|
1973
2218
|
Logger,
|
|
1974
2219
|
NOOP_SPAN,
|
|
@@ -1983,6 +2228,7 @@ export {
|
|
|
1983
2228
|
getSpanParentObject,
|
|
1984
2229
|
init,
|
|
1985
2230
|
initDataset,
|
|
2231
|
+
initExperiment,
|
|
1986
2232
|
initLogger,
|
|
1987
2233
|
log,
|
|
1988
2234
|
login,
|