braintrust 0.0.92 → 0.0.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +252 -199
- package/dist/cli.js +171 -128
- package/dist/index.js +255 -200
- package/dist/logger.d.ts +36 -25
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/dist/util.d.ts +6 -0
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -7681,7 +7681,9 @@ async function getBaseBranchAncestor(remote = void 0) {
|
|
|
7681
7681
|
if (git === null) {
|
|
7682
7682
|
throw new Error("Not in a git repo");
|
|
7683
7683
|
}
|
|
7684
|
-
const { remote: remoteName, branch: baseBranch } = await getBaseBranch(
|
|
7684
|
+
const { remote: remoteName, branch: baseBranch } = await getBaseBranch(
|
|
7685
|
+
remote
|
|
7686
|
+
);
|
|
7685
7687
|
const isDirty = (await git.diffSummary()).files.length > 0;
|
|
7686
7688
|
const head = isDirty ? "HEAD" : "HEAD^";
|
|
7687
7689
|
try {
|
|
@@ -7999,6 +8001,21 @@ function getCurrentUnixTimestamp() {
|
|
|
7999
8001
|
function isEmpty(a) {
|
|
8000
8002
|
return a === void 0 || a === null;
|
|
8001
8003
|
}
|
|
8004
|
+
var LazyValue = class {
|
|
8005
|
+
constructor(callable) {
|
|
8006
|
+
this.value = {
|
|
8007
|
+
hasComputed: false
|
|
8008
|
+
};
|
|
8009
|
+
this.callable = callable;
|
|
8010
|
+
}
|
|
8011
|
+
async get() {
|
|
8012
|
+
if (this.value.hasComputed) {
|
|
8013
|
+
return this.value.val;
|
|
8014
|
+
}
|
|
8015
|
+
this.value = { hasComputed: true, val: await this.callable() };
|
|
8016
|
+
return this.value.val;
|
|
8017
|
+
}
|
|
8018
|
+
};
|
|
8002
8019
|
|
|
8003
8020
|
// src/logger.ts
|
|
8004
8021
|
var NoopSpan = class {
|
|
@@ -8028,7 +8045,7 @@ var NoopSpan = class {
|
|
|
8028
8045
|
var NOOP_SPAN = new NoopSpan();
|
|
8029
8046
|
var BraintrustState = class {
|
|
8030
8047
|
constructor() {
|
|
8031
|
-
this.
|
|
8048
|
+
this.appUrl = null;
|
|
8032
8049
|
this.loginToken = null;
|
|
8033
8050
|
this.orgId = null;
|
|
8034
8051
|
this.orgName = null;
|
|
@@ -8044,7 +8061,7 @@ var BraintrustState = class {
|
|
|
8044
8061
|
globalThis.__inherited_braintrust_state = this;
|
|
8045
8062
|
}
|
|
8046
8063
|
resetLoginInfo() {
|
|
8047
|
-
this.
|
|
8064
|
+
this.appUrl = null;
|
|
8048
8065
|
this.loginToken = null;
|
|
8049
8066
|
this.orgId = null;
|
|
8050
8067
|
this.orgName = null;
|
|
@@ -8056,10 +8073,10 @@ var BraintrustState = class {
|
|
|
8056
8073
|
}
|
|
8057
8074
|
apiConn() {
|
|
8058
8075
|
if (!this._apiConn) {
|
|
8059
|
-
if (!this.
|
|
8060
|
-
throw new Error("Must initialize
|
|
8076
|
+
if (!this.appUrl) {
|
|
8077
|
+
throw new Error("Must initialize appUrl before requesting apiConn");
|
|
8061
8078
|
}
|
|
8062
|
-
this._apiConn = new HTTPConnection(this.
|
|
8079
|
+
this._apiConn = new HTTPConnection(this.appUrl);
|
|
8063
8080
|
}
|
|
8064
8081
|
return this._apiConn;
|
|
8065
8082
|
}
|
|
@@ -8213,25 +8230,25 @@ function logFeedbackImpl(bgLogger, parentIds, {
|
|
|
8213
8230
|
updateEvent = Object.fromEntries(
|
|
8214
8231
|
Object.entries(updateEvent).filter(([_, v]) => !isEmpty(v))
|
|
8215
8232
|
);
|
|
8216
|
-
const trueParentIds = (async () => {
|
|
8217
|
-
const { kind, ...ids } = await parentIds;
|
|
8233
|
+
const trueParentIds = new LazyValue(async () => {
|
|
8234
|
+
const { kind, ...ids } = await parentIds.get();
|
|
8218
8235
|
return ids;
|
|
8219
|
-
})
|
|
8236
|
+
});
|
|
8220
8237
|
if (Object.keys(updateEvent).length > 0) {
|
|
8221
|
-
const record = (async () => {
|
|
8238
|
+
const record = new LazyValue(async () => {
|
|
8222
8239
|
return {
|
|
8223
8240
|
id,
|
|
8224
8241
|
...updateEvent,
|
|
8225
|
-
...await trueParentIds,
|
|
8242
|
+
...await trueParentIds.get(),
|
|
8226
8243
|
[AUDIT_SOURCE_FIELD]: source,
|
|
8227
8244
|
[AUDIT_METADATA_FIELD]: metadata,
|
|
8228
8245
|
[IS_MERGE_FIELD]: true
|
|
8229
8246
|
};
|
|
8230
|
-
})
|
|
8247
|
+
});
|
|
8231
8248
|
bgLogger.log([record]);
|
|
8232
8249
|
}
|
|
8233
8250
|
if (!isEmpty(comment)) {
|
|
8234
|
-
const record = (async () => {
|
|
8251
|
+
const record = new LazyValue(async () => {
|
|
8235
8252
|
return {
|
|
8236
8253
|
id: v4_default(),
|
|
8237
8254
|
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
@@ -8243,11 +8260,11 @@ function logFeedbackImpl(bgLogger, parentIds, {
|
|
|
8243
8260
|
comment: {
|
|
8244
8261
|
text: comment
|
|
8245
8262
|
},
|
|
8246
|
-
...await trueParentIds,
|
|
8263
|
+
...await trueParentIds.get(),
|
|
8247
8264
|
[AUDIT_SOURCE_FIELD]: source,
|
|
8248
8265
|
[AUDIT_METADATA_FIELD]: metadata
|
|
8249
8266
|
};
|
|
8250
|
-
})
|
|
8267
|
+
});
|
|
8251
8268
|
bgLogger.log([record]);
|
|
8252
8269
|
}
|
|
8253
8270
|
}
|
|
@@ -8257,32 +8274,34 @@ var Logger = class {
|
|
|
8257
8274
|
this.kind = "logger";
|
|
8258
8275
|
this.lazyMetadata = lazyMetadata;
|
|
8259
8276
|
this.logOptions = logOptions;
|
|
8260
|
-
const logConn =
|
|
8277
|
+
const logConn = new LazyValue(
|
|
8278
|
+
() => this.getState().then((state) => state.logConn())
|
|
8279
|
+
);
|
|
8261
8280
|
this.bgLogger = new BackgroundLogger(logConn);
|
|
8262
8281
|
this.lastStartTime = getCurrentUnixTimestamp();
|
|
8263
8282
|
}
|
|
8264
8283
|
get org_id() {
|
|
8265
8284
|
return (async () => {
|
|
8266
|
-
return (await this.lazyMetadata).org_id;
|
|
8285
|
+
return (await this.lazyMetadata.get()).org_id;
|
|
8267
8286
|
})();
|
|
8268
8287
|
}
|
|
8269
8288
|
get project() {
|
|
8270
8289
|
return (async () => {
|
|
8271
|
-
return (await this.lazyMetadata).project;
|
|
8290
|
+
return (await this.lazyMetadata.get()).project;
|
|
8272
8291
|
})();
|
|
8273
8292
|
}
|
|
8274
8293
|
async getState() {
|
|
8275
|
-
await this.lazyMetadata;
|
|
8294
|
+
await this.lazyMetadata.get();
|
|
8276
8295
|
return _state;
|
|
8277
8296
|
}
|
|
8278
8297
|
/**
|
|
8279
8298
|
* Log a single event. The event will be batched and uploaded behind the scenes if `logOptions.asyncFlush` is true.
|
|
8280
8299
|
*
|
|
8281
8300
|
* @param event The event to log.
|
|
8282
|
-
* @param event.input:
|
|
8283
|
-
* @param event.output:
|
|
8284
|
-
* @param event.expected:
|
|
8285
|
-
* @param event.scores:
|
|
8301
|
+
* @param event.input: (Optional) the arguments that uniquely define a user input (an arbitrary, JSON serializable object).
|
|
8302
|
+
* @param event.output: (Optional) the output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question.
|
|
8303
|
+
* @param event.expected: (Optional) the ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models.
|
|
8304
|
+
* @param event.scores: (Optional) a dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs.
|
|
8286
8305
|
* @param event.metadata: (Optional) a dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings.
|
|
8287
8306
|
* @param event.metrics: (Optional) a dictionary of metrics to log. The following keys are populated automatically: "start", "end".
|
|
8288
8307
|
* @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
|
|
@@ -8338,14 +8357,16 @@ var Logger = class {
|
|
|
8338
8357
|
};
|
|
8339
8358
|
}
|
|
8340
8359
|
/**
|
|
8341
|
-
* Lower-level alternative to `traced
|
|
8360
|
+
* Lower-level alternative to `traced`. This allows you to start a span yourself, and can be useful in situations
|
|
8361
|
+
* where you cannot use callbacks. However, spans started with `startSpan` will not be marked as the "current span",
|
|
8362
|
+
* so `currentSpan()` and `traced()` will be no-ops. If you want to mark a span as current, use `traced` instead.
|
|
8342
8363
|
*
|
|
8343
8364
|
* See `traced` for full details.
|
|
8344
8365
|
*/
|
|
8345
8366
|
startSpan(args) {
|
|
8346
8367
|
const { name, ...argsRest } = args ?? {};
|
|
8347
8368
|
return new SpanImpl({
|
|
8348
|
-
parentIds: this.lazyParentIds(),
|
|
8369
|
+
parentIds: new LazyValue(() => this.lazyParentIds()),
|
|
8349
8370
|
bgLogger: this.bgLogger,
|
|
8350
8371
|
name: name ?? "root",
|
|
8351
8372
|
...argsRest
|
|
@@ -8363,7 +8384,11 @@ var Logger = class {
|
|
|
8363
8384
|
* @param event.source (Optional) the source of the feedback. Must be one of "external" (default), "app", or "api".
|
|
8364
8385
|
*/
|
|
8365
8386
|
logFeedback(event) {
|
|
8366
|
-
logFeedbackImpl(
|
|
8387
|
+
logFeedbackImpl(
|
|
8388
|
+
this.bgLogger,
|
|
8389
|
+
new LazyValue(() => this.lazyParentIds()),
|
|
8390
|
+
event
|
|
8391
|
+
);
|
|
8367
8392
|
}
|
|
8368
8393
|
/*
|
|
8369
8394
|
* Flush any pending logs to the server.
|
|
@@ -8413,9 +8438,20 @@ var BackgroundLogger = class {
|
|
|
8413
8438
|
}
|
|
8414
8439
|
async flush_once(batchSize = DefaultBatchSize) {
|
|
8415
8440
|
this.active_flush_resolved = false;
|
|
8416
|
-
const
|
|
8441
|
+
const itemLazyValues = this.items;
|
|
8417
8442
|
this.items = [];
|
|
8418
|
-
const allItems =
|
|
8443
|
+
const allItems = await (async () => {
|
|
8444
|
+
try {
|
|
8445
|
+
const itemPromises = itemLazyValues.map((x) => x.get());
|
|
8446
|
+
return mergeRowBatch(await Promise.all(itemPromises)).reverse();
|
|
8447
|
+
} catch (e) {
|
|
8448
|
+
console.warn(
|
|
8449
|
+
"Encountered error when constructing records to flush:\n",
|
|
8450
|
+
e
|
|
8451
|
+
);
|
|
8452
|
+
return [];
|
|
8453
|
+
}
|
|
8454
|
+
})();
|
|
8419
8455
|
let postPromises = [];
|
|
8420
8456
|
while (true) {
|
|
8421
8457
|
const items = [];
|
|
@@ -8440,9 +8476,7 @@ var BackgroundLogger = class {
|
|
|
8440
8476
|
for (let i = 0; i < NumRetries; i++) {
|
|
8441
8477
|
const startTime = now();
|
|
8442
8478
|
try {
|
|
8443
|
-
return (await (await this.logConn).post_json("logs", itemsS)).map(
|
|
8444
|
-
(res) => res.id
|
|
8445
|
-
);
|
|
8479
|
+
return (await (await this.logConn.get()).post_json("logs", itemsS)).map((res) => res.id);
|
|
8446
8480
|
} catch (e) {
|
|
8447
8481
|
const retryingText = i + 1 === NumRetries ? "" : " Retrying";
|
|
8448
8482
|
const errMsg = (() => {
|
|
@@ -8489,88 +8523,92 @@ function init(project, options = {}) {
|
|
|
8489
8523
|
baseExperiment,
|
|
8490
8524
|
isPublic,
|
|
8491
8525
|
update,
|
|
8492
|
-
|
|
8526
|
+
appUrl,
|
|
8493
8527
|
apiKey,
|
|
8494
8528
|
orgName,
|
|
8495
8529
|
metadata,
|
|
8496
8530
|
gitMetadataSettings
|
|
8497
8531
|
} = options || {};
|
|
8498
|
-
const lazyMetadata =
|
|
8499
|
-
|
|
8500
|
-
|
|
8501
|
-
|
|
8502
|
-
|
|
8503
|
-
|
|
8504
|
-
|
|
8505
|
-
|
|
8506
|
-
|
|
8507
|
-
|
|
8508
|
-
|
|
8509
|
-
|
|
8510
|
-
|
|
8511
|
-
if (description) {
|
|
8512
|
-
args["description"] = description;
|
|
8513
|
-
}
|
|
8514
|
-
if (update) {
|
|
8515
|
-
args["update"] = update;
|
|
8516
|
-
}
|
|
8517
|
-
let mergedGitMetadataSettings = {
|
|
8518
|
-
..._state.gitMetadataSettings || {
|
|
8519
|
-
collect: "all"
|
|
8532
|
+
const lazyMetadata = new LazyValue(
|
|
8533
|
+
async () => {
|
|
8534
|
+
await login({
|
|
8535
|
+
orgName,
|
|
8536
|
+
apiKey,
|
|
8537
|
+
appUrl
|
|
8538
|
+
});
|
|
8539
|
+
const args = {
|
|
8540
|
+
project_name: project,
|
|
8541
|
+
org_id: _state.orgId
|
|
8542
|
+
};
|
|
8543
|
+
if (experiment) {
|
|
8544
|
+
args["experiment_name"] = experiment;
|
|
8520
8545
|
}
|
|
8521
|
-
|
|
8522
|
-
|
|
8523
|
-
|
|
8524
|
-
|
|
8525
|
-
|
|
8526
|
-
|
|
8527
|
-
|
|
8528
|
-
|
|
8529
|
-
|
|
8530
|
-
args["repo_info"] = repoStatus2;
|
|
8531
|
-
}
|
|
8532
|
-
if (baseExperiment) {
|
|
8533
|
-
args["base_experiment"] = baseExperiment;
|
|
8534
|
-
} else {
|
|
8535
|
-
args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
|
|
8536
|
-
}
|
|
8537
|
-
if (dataset !== void 0) {
|
|
8538
|
-
args["dataset_id"] = dataset.id;
|
|
8539
|
-
args["dataset_version"] = await dataset.version();
|
|
8540
|
-
}
|
|
8541
|
-
if (isPublic !== void 0) {
|
|
8542
|
-
args["public"] = isPublic;
|
|
8543
|
-
}
|
|
8544
|
-
if (metadata) {
|
|
8545
|
-
args["metadata"] = metadata;
|
|
8546
|
-
}
|
|
8547
|
-
let response = null;
|
|
8548
|
-
while (true) {
|
|
8549
|
-
try {
|
|
8550
|
-
response = await _state.apiConn().post_json("api/experiment/register", args);
|
|
8551
|
-
break;
|
|
8552
|
-
} catch (e) {
|
|
8553
|
-
if (args["base_experiment"] && `${"data" in e && e.data}`.includes("base experiment")) {
|
|
8554
|
-
console.warn(`Base experiment ${args["base_experiment"]} not found.`);
|
|
8555
|
-
delete args["base_experiment"];
|
|
8556
|
-
} else {
|
|
8557
|
-
throw e;
|
|
8546
|
+
if (description) {
|
|
8547
|
+
args["description"] = description;
|
|
8548
|
+
}
|
|
8549
|
+
if (update) {
|
|
8550
|
+
args["update"] = update;
|
|
8551
|
+
}
|
|
8552
|
+
let mergedGitMetadataSettings = {
|
|
8553
|
+
..._state.gitMetadataSettings || {
|
|
8554
|
+
collect: "all"
|
|
8558
8555
|
}
|
|
8556
|
+
};
|
|
8557
|
+
if (gitMetadataSettings) {
|
|
8558
|
+
mergedGitMetadataSettings = mergeGitMetadataSettings(
|
|
8559
|
+
mergedGitMetadataSettings,
|
|
8560
|
+
gitMetadataSettings
|
|
8561
|
+
);
|
|
8559
8562
|
}
|
|
8560
|
-
|
|
8561
|
-
|
|
8562
|
-
|
|
8563
|
-
id: response.project.id,
|
|
8564
|
-
name: response.project.name,
|
|
8565
|
-
fullInfo: response.project
|
|
8566
|
-
},
|
|
8567
|
-
experiment: {
|
|
8568
|
-
id: response.experiment.id,
|
|
8569
|
-
name: response.experiment.name,
|
|
8570
|
-
fullInfo: response.experiment
|
|
8563
|
+
const repoStatus2 = await isomorph_default.getRepoStatus(gitMetadataSettings);
|
|
8564
|
+
if (repoStatus2) {
|
|
8565
|
+
args["repo_info"] = repoStatus2;
|
|
8571
8566
|
}
|
|
8572
|
-
|
|
8573
|
-
|
|
8567
|
+
if (baseExperiment) {
|
|
8568
|
+
args["base_experiment"] = baseExperiment;
|
|
8569
|
+
} else {
|
|
8570
|
+
args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
|
|
8571
|
+
}
|
|
8572
|
+
if (dataset !== void 0) {
|
|
8573
|
+
args["dataset_id"] = dataset.id;
|
|
8574
|
+
args["dataset_version"] = await dataset.version();
|
|
8575
|
+
}
|
|
8576
|
+
if (isPublic !== void 0) {
|
|
8577
|
+
args["public"] = isPublic;
|
|
8578
|
+
}
|
|
8579
|
+
if (metadata) {
|
|
8580
|
+
args["metadata"] = metadata;
|
|
8581
|
+
}
|
|
8582
|
+
let response = null;
|
|
8583
|
+
while (true) {
|
|
8584
|
+
try {
|
|
8585
|
+
response = await _state.apiConn().post_json("api/experiment/register", args);
|
|
8586
|
+
break;
|
|
8587
|
+
} catch (e) {
|
|
8588
|
+
if (args["base_experiment"] && `${"data" in e && e.data}`.includes("base experiment")) {
|
|
8589
|
+
console.warn(
|
|
8590
|
+
`Base experiment ${args["base_experiment"]} not found.`
|
|
8591
|
+
);
|
|
8592
|
+
delete args["base_experiment"];
|
|
8593
|
+
} else {
|
|
8594
|
+
throw e;
|
|
8595
|
+
}
|
|
8596
|
+
}
|
|
8597
|
+
}
|
|
8598
|
+
return {
|
|
8599
|
+
project: {
|
|
8600
|
+
id: response.project.id,
|
|
8601
|
+
name: response.project.name,
|
|
8602
|
+
fullInfo: response.project
|
|
8603
|
+
},
|
|
8604
|
+
experiment: {
|
|
8605
|
+
id: response.experiment.id,
|
|
8606
|
+
name: response.experiment.name,
|
|
8607
|
+
fullInfo: response.experiment
|
|
8608
|
+
}
|
|
8609
|
+
};
|
|
8610
|
+
}
|
|
8611
|
+
);
|
|
8574
8612
|
const ret = new Experiment(lazyMetadata, dataset);
|
|
8575
8613
|
if (options.setCurrent ?? true) {
|
|
8576
8614
|
_state.currentExperiment = ret;
|
|
@@ -8592,33 +8630,35 @@ function withLogger(callback, options = {}) {
|
|
|
8592
8630
|
return callback(logger);
|
|
8593
8631
|
}
|
|
8594
8632
|
function initDataset(project, options = {}) {
|
|
8595
|
-
const { dataset, description, version,
|
|
8596
|
-
const lazyMetadata =
|
|
8597
|
-
|
|
8598
|
-
|
|
8599
|
-
|
|
8600
|
-
|
|
8601
|
-
|
|
8602
|
-
|
|
8603
|
-
|
|
8604
|
-
|
|
8605
|
-
|
|
8606
|
-
|
|
8607
|
-
|
|
8608
|
-
|
|
8609
|
-
|
|
8610
|
-
|
|
8611
|
-
|
|
8612
|
-
|
|
8613
|
-
|
|
8614
|
-
|
|
8615
|
-
|
|
8616
|
-
|
|
8617
|
-
|
|
8618
|
-
|
|
8619
|
-
|
|
8620
|
-
|
|
8621
|
-
|
|
8633
|
+
const { dataset, description, version, appUrl, apiKey, orgName } = options || {};
|
|
8634
|
+
const lazyMetadata = new LazyValue(
|
|
8635
|
+
async () => {
|
|
8636
|
+
await login({
|
|
8637
|
+
orgName,
|
|
8638
|
+
apiKey,
|
|
8639
|
+
appUrl
|
|
8640
|
+
});
|
|
8641
|
+
const args = {
|
|
8642
|
+
org_id: _state.orgId,
|
|
8643
|
+
project_name: project,
|
|
8644
|
+
dataset_name: dataset,
|
|
8645
|
+
description
|
|
8646
|
+
};
|
|
8647
|
+
const response = await _state.apiConn().post_json("api/dataset/register", args);
|
|
8648
|
+
return {
|
|
8649
|
+
project: {
|
|
8650
|
+
id: response.project.id,
|
|
8651
|
+
name: response.project.name,
|
|
8652
|
+
fullInfo: response.project
|
|
8653
|
+
},
|
|
8654
|
+
dataset: {
|
|
8655
|
+
id: response.dataset.id,
|
|
8656
|
+
name: response.dataset.name,
|
|
8657
|
+
fullInfo: response.dataset
|
|
8658
|
+
}
|
|
8659
|
+
};
|
|
8660
|
+
}
|
|
8661
|
+
);
|
|
8622
8662
|
return new Dataset(lazyMetadata, version);
|
|
8623
8663
|
}
|
|
8624
8664
|
function withDataset(project, callback, options = {}) {
|
|
@@ -8633,51 +8673,53 @@ function initLogger(options = {}) {
|
|
|
8633
8673
|
projectName,
|
|
8634
8674
|
projectId,
|
|
8635
8675
|
asyncFlush,
|
|
8636
|
-
|
|
8676
|
+
appUrl,
|
|
8637
8677
|
apiKey,
|
|
8638
8678
|
orgName,
|
|
8639
8679
|
forceLogin
|
|
8640
8680
|
} = options || {};
|
|
8641
|
-
const lazyMetadata =
|
|
8642
|
-
|
|
8643
|
-
|
|
8644
|
-
|
|
8645
|
-
|
|
8646
|
-
|
|
8647
|
-
|
|
8648
|
-
const org_id = _state.orgId;
|
|
8649
|
-
if (projectId === void 0) {
|
|
8650
|
-
const response = await _state.apiConn().post_json("api/project/register", {
|
|
8651
|
-
project_name: projectName || GLOBAL_PROJECT,
|
|
8652
|
-
org_id
|
|
8653
|
-
});
|
|
8654
|
-
return {
|
|
8655
|
-
org_id,
|
|
8656
|
-
project: {
|
|
8657
|
-
id: response.project.id,
|
|
8658
|
-
name: response.project.name,
|
|
8659
|
-
fullInfo: response.project
|
|
8660
|
-
}
|
|
8661
|
-
};
|
|
8662
|
-
} else if (projectName === void 0) {
|
|
8663
|
-
const response = await _state.apiConn().get_json("api/project", {
|
|
8664
|
-
id: projectId
|
|
8681
|
+
const lazyMetadata = new LazyValue(
|
|
8682
|
+
async () => {
|
|
8683
|
+
await login({
|
|
8684
|
+
orgName,
|
|
8685
|
+
apiKey,
|
|
8686
|
+
appUrl,
|
|
8687
|
+
forceLogin
|
|
8665
8688
|
});
|
|
8666
|
-
|
|
8667
|
-
|
|
8668
|
-
project
|
|
8669
|
-
|
|
8670
|
-
|
|
8671
|
-
|
|
8672
|
-
|
|
8673
|
-
|
|
8674
|
-
|
|
8675
|
-
|
|
8676
|
-
|
|
8677
|
-
|
|
8678
|
-
|
|
8689
|
+
const org_id = _state.orgId;
|
|
8690
|
+
if (projectId === void 0) {
|
|
8691
|
+
const response = await _state.apiConn().post_json("api/project/register", {
|
|
8692
|
+
project_name: projectName || GLOBAL_PROJECT,
|
|
8693
|
+
org_id
|
|
8694
|
+
});
|
|
8695
|
+
return {
|
|
8696
|
+
org_id,
|
|
8697
|
+
project: {
|
|
8698
|
+
id: response.project.id,
|
|
8699
|
+
name: response.project.name,
|
|
8700
|
+
fullInfo: response.project
|
|
8701
|
+
}
|
|
8702
|
+
};
|
|
8703
|
+
} else if (projectName === void 0) {
|
|
8704
|
+
const response = await _state.apiConn().get_json("api/project", {
|
|
8705
|
+
id: projectId
|
|
8706
|
+
});
|
|
8707
|
+
return {
|
|
8708
|
+
org_id,
|
|
8709
|
+
project: {
|
|
8710
|
+
id: projectId,
|
|
8711
|
+
name: response.name,
|
|
8712
|
+
fullInfo: response.project
|
|
8713
|
+
}
|
|
8714
|
+
};
|
|
8715
|
+
} else {
|
|
8716
|
+
return {
|
|
8717
|
+
org_id,
|
|
8718
|
+
project: { id: projectId, name: projectName, fullInfo: {} }
|
|
8719
|
+
};
|
|
8720
|
+
}
|
|
8679
8721
|
}
|
|
8680
|
-
|
|
8722
|
+
);
|
|
8681
8723
|
const ret = new Logger(lazyMetadata, {
|
|
8682
8724
|
asyncFlush
|
|
8683
8725
|
});
|
|
@@ -8688,7 +8730,7 @@ function initLogger(options = {}) {
|
|
|
8688
8730
|
}
|
|
8689
8731
|
async function login(options = {}) {
|
|
8690
8732
|
const {
|
|
8691
|
-
|
|
8733
|
+
appUrl = isomorph_default.getEnv("BRAINTRUST_APP_URL") || "https://www.braintrustdata.com",
|
|
8692
8734
|
apiKey = isomorph_default.getEnv("BRAINTRUST_API_KEY"),
|
|
8693
8735
|
orgName = isomorph_default.getEnv("BRAINTRUST_ORG_NAME")
|
|
8694
8736
|
} = options || {};
|
|
@@ -8697,11 +8739,11 @@ async function login(options = {}) {
|
|
|
8697
8739
|
return;
|
|
8698
8740
|
}
|
|
8699
8741
|
_state.resetLoginInfo();
|
|
8700
|
-
_state.
|
|
8742
|
+
_state.appUrl = appUrl;
|
|
8701
8743
|
let conn = null;
|
|
8702
8744
|
if (apiKey !== void 0) {
|
|
8703
8745
|
const resp = await checkResponse(
|
|
8704
|
-
await fetch(_urljoin(_state.
|
|
8746
|
+
await fetch(_urljoin(_state.appUrl, `/api/apikey/login`), {
|
|
8705
8747
|
method: "POST",
|
|
8706
8748
|
headers: {
|
|
8707
8749
|
"Content-Type": "application/json"
|
|
@@ -8818,7 +8860,7 @@ function _check_org_info(org_info, org_name) {
|
|
|
8818
8860
|
if (org_name === void 0 || org.name === org_name) {
|
|
8819
8861
|
_state.orgId = org.id;
|
|
8820
8862
|
_state.orgName = org.name;
|
|
8821
|
-
_state.logUrl = isomorph_default.getEnv("
|
|
8863
|
+
_state.logUrl = isomorph_default.getEnv("BRAINTRUST_API_URL") ?? org.api_url;
|
|
8822
8864
|
_state.gitMetadataSettings = org.git_metadata || void 0;
|
|
8823
8865
|
break;
|
|
8824
8866
|
}
|
|
@@ -8888,6 +8930,9 @@ function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
|
|
|
8888
8930
|
"Exactly one of input or inputs (deprecated) must be specified. Prefer input."
|
|
8889
8931
|
);
|
|
8890
8932
|
}
|
|
8933
|
+
if (!event.output) {
|
|
8934
|
+
throw new Error("output must be specified");
|
|
8935
|
+
}
|
|
8891
8936
|
if (!event.scores) {
|
|
8892
8937
|
throw new Error("scores must be specified");
|
|
8893
8938
|
}
|
|
@@ -8906,27 +8951,29 @@ var Experiment = class {
|
|
|
8906
8951
|
this.kind = "experiment";
|
|
8907
8952
|
this.lazyMetadata = lazyMetadata;
|
|
8908
8953
|
this.dataset = dataset;
|
|
8909
|
-
const logConn =
|
|
8954
|
+
const logConn = new LazyValue(
|
|
8955
|
+
() => this.getState().then((state) => state.logConn())
|
|
8956
|
+
);
|
|
8910
8957
|
this.bgLogger = new BackgroundLogger(logConn);
|
|
8911
8958
|
this.lastStartTime = getCurrentUnixTimestamp();
|
|
8912
8959
|
}
|
|
8913
8960
|
get id() {
|
|
8914
8961
|
return (async () => {
|
|
8915
|
-
return (await this.lazyMetadata).experiment.id;
|
|
8962
|
+
return (await this.lazyMetadata.get()).experiment.id;
|
|
8916
8963
|
})();
|
|
8917
8964
|
}
|
|
8918
8965
|
get name() {
|
|
8919
8966
|
return (async () => {
|
|
8920
|
-
return (await this.lazyMetadata).experiment.name;
|
|
8967
|
+
return (await this.lazyMetadata.get()).experiment.name;
|
|
8921
8968
|
})();
|
|
8922
8969
|
}
|
|
8923
8970
|
get project() {
|
|
8924
8971
|
return (async () => {
|
|
8925
|
-
return (await this.lazyMetadata).project;
|
|
8972
|
+
return (await this.lazyMetadata.get()).project;
|
|
8926
8973
|
})();
|
|
8927
8974
|
}
|
|
8928
8975
|
async getState() {
|
|
8929
|
-
await this.lazyMetadata;
|
|
8976
|
+
await this.lazyMetadata.get();
|
|
8930
8977
|
return _state;
|
|
8931
8978
|
}
|
|
8932
8979
|
/**
|
|
@@ -8935,7 +8982,7 @@ var Experiment = class {
|
|
|
8935
8982
|
* @param event The event to log.
|
|
8936
8983
|
* @param event.input: The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on, Braintrust will use the `input` to know whether two test cases are the same between experiments, so they should not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the `input` should be identical.
|
|
8937
8984
|
* @param event.output: The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question.
|
|
8938
|
-
* @param event.expected: The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models.
|
|
8985
|
+
* @param event.expected: (Optional) The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models.
|
|
8939
8986
|
* @param event.scores: A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
|
|
8940
8987
|
* @param event.metadata: (Optional) a dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings.
|
|
8941
8988
|
* @param event.metrics: (Optional) a dictionary of metrics to log. The following keys are populated automatically: "start", "end".
|
|
@@ -8977,14 +9024,16 @@ var Experiment = class {
|
|
|
8977
9024
|
};
|
|
8978
9025
|
}
|
|
8979
9026
|
/**
|
|
8980
|
-
* Lower-level alternative to `traced
|
|
9027
|
+
* Lower-level alternative to `traced`. This allows you to start a span yourself, and can be useful in situations
|
|
9028
|
+
* where you cannot use callbacks. However, spans started with `startSpan` will not be marked as the "current span",
|
|
9029
|
+
* so `currentSpan()` and `traced()` will be no-ops. If you want to mark a span as current, use `traced` instead.
|
|
8981
9030
|
*
|
|
8982
9031
|
* See `traced` for full details.
|
|
8983
9032
|
*/
|
|
8984
9033
|
startSpan(args) {
|
|
8985
9034
|
const { name, ...argsRest } = args ?? {};
|
|
8986
9035
|
return new SpanImpl({
|
|
8987
|
-
parentIds: this.lazyParentIds(),
|
|
9036
|
+
parentIds: new LazyValue(() => this.lazyParentIds()),
|
|
8988
9037
|
bgLogger: this.bgLogger,
|
|
8989
9038
|
name: name ?? "root",
|
|
8990
9039
|
...argsRest
|
|
@@ -9002,7 +9051,7 @@ var Experiment = class {
|
|
|
9002
9051
|
let { summarizeScores = true, comparisonExperimentId = void 0 } = options || {};
|
|
9003
9052
|
await this.bgLogger.flush();
|
|
9004
9053
|
const state = await this.getState();
|
|
9005
|
-
const projectUrl = `${state.
|
|
9054
|
+
const projectUrl = `${state.appUrl}/app/${encodeURIComponent(
|
|
9006
9055
|
state.orgName
|
|
9007
9056
|
)}/p/${encodeURIComponent((await this.project).name)}`;
|
|
9008
9057
|
const experimentUrl = `${projectUrl}/${encodeURIComponent(
|
|
@@ -9058,7 +9107,11 @@ var Experiment = class {
|
|
|
9058
9107
|
* @param event.source (Optional) the source of the feedback. Must be one of "external" (default), "app", or "api".
|
|
9059
9108
|
*/
|
|
9060
9109
|
logFeedback(event) {
|
|
9061
|
-
logFeedbackImpl(
|
|
9110
|
+
logFeedbackImpl(
|
|
9111
|
+
this.bgLogger,
|
|
9112
|
+
new LazyValue(() => this.lazyParentIds()),
|
|
9113
|
+
event
|
|
9114
|
+
);
|
|
9062
9115
|
}
|
|
9063
9116
|
/**
|
|
9064
9117
|
* Flush any pending rows to the server.
|
|
@@ -9139,18 +9192,18 @@ var SpanImpl = class _SpanImpl {
|
|
|
9139
9192
|
if (sanitizedAndInternalData.metrics?.end) {
|
|
9140
9193
|
this.loggedEndTime = sanitizedAndInternalData.metrics?.end;
|
|
9141
9194
|
}
|
|
9142
|
-
const parentIds = (async () => {
|
|
9143
|
-
const { kind, ...ids } = await this.parentIds;
|
|
9195
|
+
const parentIds = new LazyValue(async () => {
|
|
9196
|
+
const { kind, ...ids } = await this.parentIds.get();
|
|
9144
9197
|
return ids;
|
|
9145
|
-
})
|
|
9146
|
-
const record = (async () => {
|
|
9198
|
+
});
|
|
9199
|
+
const record = new LazyValue(async () => {
|
|
9147
9200
|
return {
|
|
9148
9201
|
...sanitizedAndInternalData,
|
|
9149
9202
|
...this.rowIds,
|
|
9150
|
-
...await parentIds,
|
|
9203
|
+
...await parentIds.get(),
|
|
9151
9204
|
[IS_MERGE_FIELD]: this.isMerge
|
|
9152
9205
|
};
|
|
9153
|
-
})
|
|
9206
|
+
});
|
|
9154
9207
|
this.bgLogger.log([record]);
|
|
9155
9208
|
}
|
|
9156
9209
|
logFeedback(event) {
|
|
@@ -9204,26 +9257,28 @@ var Dataset = class {
|
|
|
9204
9257
|
this._fetchedData = void 0;
|
|
9205
9258
|
this.lazyMetadata = lazyMetadata;
|
|
9206
9259
|
this.pinnedVersion = pinnedVersion;
|
|
9207
|
-
const logConn =
|
|
9260
|
+
const logConn = new LazyValue(
|
|
9261
|
+
() => this.getState().then((state) => state.logConn())
|
|
9262
|
+
);
|
|
9208
9263
|
this.bgLogger = new BackgroundLogger(logConn);
|
|
9209
9264
|
}
|
|
9210
9265
|
get id() {
|
|
9211
9266
|
return (async () => {
|
|
9212
|
-
return (await this.lazyMetadata).dataset.id;
|
|
9267
|
+
return (await this.lazyMetadata.get()).dataset.id;
|
|
9213
9268
|
})();
|
|
9214
9269
|
}
|
|
9215
9270
|
get name() {
|
|
9216
9271
|
return (async () => {
|
|
9217
|
-
return (await this.lazyMetadata).dataset.name;
|
|
9272
|
+
return (await this.lazyMetadata.get()).dataset.name;
|
|
9218
9273
|
})();
|
|
9219
9274
|
}
|
|
9220
9275
|
get project() {
|
|
9221
9276
|
return (async () => {
|
|
9222
|
-
return (await this.lazyMetadata).project;
|
|
9277
|
+
return (await this.lazyMetadata.get()).project;
|
|
9223
9278
|
})();
|
|
9224
9279
|
}
|
|
9225
9280
|
async getState() {
|
|
9226
|
-
await this.lazyMetadata;
|
|
9281
|
+
await this.lazyMetadata.get();
|
|
9227
9282
|
return _state;
|
|
9228
9283
|
}
|
|
9229
9284
|
/**
|
|
@@ -9254,7 +9309,7 @@ var Dataset = class {
|
|
|
9254
9309
|
}
|
|
9255
9310
|
}
|
|
9256
9311
|
const rowId = id || v4_default();
|
|
9257
|
-
const args = (async () => ({
|
|
9312
|
+
const args = new LazyValue(async () => ({
|
|
9258
9313
|
id: rowId,
|
|
9259
9314
|
inputs: input,
|
|
9260
9315
|
output,
|
|
@@ -9262,18 +9317,18 @@ var Dataset = class {
|
|
|
9262
9317
|
dataset_id: await this.id,
|
|
9263
9318
|
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9264
9319
|
metadata
|
|
9265
|
-
}))
|
|
9320
|
+
}));
|
|
9266
9321
|
this.bgLogger.log([args]);
|
|
9267
9322
|
return rowId;
|
|
9268
9323
|
}
|
|
9269
9324
|
delete(id) {
|
|
9270
|
-
const args = (async () => ({
|
|
9325
|
+
const args = new LazyValue(async () => ({
|
|
9271
9326
|
id,
|
|
9272
9327
|
project_id: (await this.project).id,
|
|
9273
9328
|
dataset_id: await this.id,
|
|
9274
9329
|
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9275
9330
|
_object_delete: true
|
|
9276
|
-
}))
|
|
9331
|
+
}));
|
|
9277
9332
|
this.bgLogger.log([args]);
|
|
9278
9333
|
return id;
|
|
9279
9334
|
}
|
|
@@ -9287,7 +9342,7 @@ var Dataset = class {
|
|
|
9287
9342
|
let { summarizeData = true } = options || {};
|
|
9288
9343
|
await this.bgLogger.flush();
|
|
9289
9344
|
const state = await this.getState();
|
|
9290
|
-
const projectUrl = `${state.
|
|
9345
|
+
const projectUrl = `${state.appUrl}/app/${encodeURIComponent(
|
|
9291
9346
|
state.orgName
|
|
9292
9347
|
)}/p/${encodeURIComponent((await this.project).name)}`;
|
|
9293
9348
|
const datasetUrl = `${projectUrl}/d/${encodeURIComponent(await this.name)}`;
|